From aa3bc364dddf5a847015438660b1e982f5e6fcb9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 14:08:01 +0000 Subject: [PATCH 1/9] Read the expressions the model actually writes, and stop two skills taking questions that are not theirs Co-authored-by: Sebastian --- README.md | 10 ++++- src/eval/scenarios.ts | 31 ++++++++++++++++ src/skills/current-date/SKILL.md | 7 ++++ src/skills/library.test.ts | 6 ++- src/skills/lookup-term/SKILL.md | 5 +++ src/skills/route.test.ts | 28 ++++++++++++++ src/skills/summarize-url/SKILL.md | 6 ++- src/tools/calculator.test.ts | 62 ++++++++++++++++++++++++++++++- src/tools/calculator.ts | 57 ++++++++++++++++++++++++++-- 9 files changed, 205 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 27d6390..b2c4aa6 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,8 @@ It removed a deployment compromise too. The Pages build used to unset `VITE_AGEN The calculator deliberately avoids `eval`. Expressions come from model output, which is attacker-influenceable as soon as the model has read an untrusted page. +**It reads what the model writes, though.** A refused expression spends a tool round and sends the model back to the mental arithmetic this tool exists to replace, and of eighteen expressions a 0.8B model plausibly produces, ten were refused: `98,765 * 4,321`, `18% of 2450`, `(17 * 23) / 4 =`, `2 x 3`, `$1200 * 1.19`. Each has one arithmetic meaning, so each is now normalised before parsing — thousands separators and the decimal comma, `×` `·` `÷` `−` and `x` between numbers, a trailing equals sign, currency symbols, and a percentage written as `18% of` / `18 Prozent von`. Ambiguity is still refused rather than guessed: `18% off 2450` is a discount, not a percentage of the number beside it, and a unit conversion is not arithmetic at all. Where it does refuse, the message now says so in a way the next round can act on. + ### What leaves the browser Inference does not: prompts, reasoning, and replies never leave the GPU, and neither do [memories](#memory), which are written to IndexedDB in this browser and read back into a prompt that goes no further than the GPU either. Tools are the exception, and always were. A `web_search` call sends the query to the chosen provider, a `read_page` call sends the URL to the reader, and a `weather` call sends the place name to Open-Meteo's geocoder and its coordinates to the two forecast services — the difference now is that these go direct, with no server of ours in the path to log them. @@ -421,12 +423,18 @@ A skill's triggers are a claim on a class of request, and the way that claim goe | _Wie spät ist es?_ | nothing | The clock had no German shape at all | | _Who is that?_ / _Wer ist das?_ | `research` | `who is` / `wer ist` treated a pronoun as a person to look up | | _Was ist los?_ | `research` | `los` was grouped with _gerade_ / _heute_, and it is a greeting | +| _Was ist heute für ein Tag?_ | `research` | `was ist heute` is a trigger, and it outranks any keyword | +| _Was bedeutet TLDR?_ | `summarize` | `tldr` names the request, except when it _is_ the question | The first four, and the two pronoun cases at the bottom, are the same mistake: a word that _appears in_ a kind of request was mistaken for the request itself. The fix is to match the shape instead — `what('s| is) the (date|time)` anchored at the end of the message rather than the word `today`, `how much is a` rather than `how much is`, an interrogative alongside the year rather than the year alone, `who is` only when the next word is not a pronoun. Anchoring also buys an honest refusal. `current_time` reads the user's own clock and no other, so its triggers end in `(?!\s+in\b)`: _what time is it_ routes, _what time is it in Tokyo_ deliberately routes nowhere, because answering it with the local hour would be wrong rather than approximate. A keyword cannot express that, which is why these German shapes are triggers rather than index entries. -The last two are the cost of a keyword-only second language, and they were the commonest question there is. Where a German phrasing has a shape worth matching it is now written out; the index still catches the rest. +_Was ist Stripe?_ and _Wie spät ist es?_ are the cost of a keyword-only second language, and they were the commonest question there is. Where a German phrasing has a shape worth matching it is now written out; the index still catches the rest. + +**A trigger outranks every keyword, wherever it lives.** Stage 1 runs across the whole catalogue before stage 2 runs at all, so `was ist heute` in `research-question` — priority 10 — was taking _Was ist heute für ein Tag?_ to a search engine while `welcher tag` sat unread in `current-date`'s index at priority 25. Priority orders the triggers among themselves and does nothing for a keyword, which means a German phrasing left to the index is only safe until some other skill writes a trigger that happens to cover it. Both German date shapes are triggers now for that reason. + +The last one is the same word doing two jobs. _tldr_ asks for a summary, and _was bedeutet TLDR_ asks what the word means — one needs a page and the other needs a search, so the trigger now stands down when the abbreviation is what is being asked about, and `lookup-term` picks the question up. A definition question is a bare-name lookup with the words the other way round, which is why it belongs to the skill that searches for the term verbatim rather than to one of its own. ## Finding the right skill diff --git a/src/eval/scenarios.ts b/src/eval/scenarios.ts index 67d90ca..c8c2566 100644 --- a/src/eval/scenarios.ts +++ b/src/eval/scenarios.ts @@ -184,6 +184,25 @@ export const SCENARIOS: Scenario[] = [ expectTool: 'current_time', accept: matches(/\d{1,2}[:.]\d{2}|\buhr\b/i), }, + { + id: 'time-german-weekday', + category: 'time', + // The commonest German way to ask, and it used to reach `web_search`: + // research-question's `was ist heute` is a trigger, and a trigger is matched + // before the keyword that would have found the clock. + prompt: 'Was ist heute für ein Tag?', + expectTool: 'current_time', + accept: matches(/montag|dienstag|mittwoch|donnerstag|freitag|samstag|sonntag|\d{1,2}\.\s*\w+/i), + }, + { + id: 'arith-percent-sign', + category: 'arithmetic', + // Written with the sign rather than the word, which is how the percentage + // reaches the calculator as `18% von 2450` — an expression it used to refuse. + prompt: 'Wie viel sind 18% von 2450?', + expectTool: 'calculator', + accept: (answer) => hasNumber(answer, '441'), + }, { id: 'recall-favourite-colour', category: 'recall', @@ -307,6 +326,18 @@ export const SCENARIOS: Scenario[] = [ accept: matches(/payment|zahlung|checkout|billing|fintech|bezahl/i), online: true, }, + { + id: 'lookup-abbreviation', + category: 'lookup', + // An abbreviation is a bare name asked about the other way round. This went + // to `summarize-url`, which answered a question about a word by asking which + // page was meant. + prompt: 'Was bedeutet TLDR?', + expectTool: 'web_search', + acceptCall: keepsTermIntact('tldr'), + accept: matches(/zusammenfass|kurzfass|kurz|lang|summar|too long/i), + online: true, + }, { id: 'web-price-not-arithmetic', category: 'web', diff --git a/src/skills/current-date/SKILL.md b/src/skills/current-date/SKILL.md index af8ca40..0300973 100644 --- a/src/skills/current-date/SKILL.md +++ b/src/skills/current-date/SKILL.md @@ -32,6 +32,13 @@ jarvis: - '\bwie sp(ä|ae)t ist es\b(?!\s+in\b)' - '\bwie viel uhr ist es\b(?!\s+in\b)' - '\bist (heute|es) (montag|dienstag|mittwoch|donnerstag|freitag|samstag|sonntag)\b' + # The commonest way to ask in German, and it has to be a trigger rather than + # a keyword: triggers are matched across the whole catalogue before any + # keyword is, so `was ist heute` in research-question was answering *Was ist + # heute für ein Tag?* with a web search while `welcher tag` sat unread. + - '\bwas ist heute für ein(en)?\s+(tag|wochentag|datum)\b' + - '\bwelcher (tag|wochentag) ist (heute|gerade|jetzt)\b' + - '\bwelches datum ist (heute|gerade|jetzt)\b' exemplars: - user: What year is it? steps: diff --git a/src/skills/library.test.ts b/src/skills/library.test.ts index d74fb86..6f4d428 100644 --- a/src/skills/library.test.ts +++ b/src/skills/library.test.ts @@ -160,7 +160,11 @@ describe('current-date', () => { ['Ist heute Freitag?', 'trigger'], ['Welches Jahr ist gerade?', 'search'], ['Welches Datum haben wir?', 'search'], - ['Welcher Tag ist heute?', 'search'], + // Now a trigger rather than an index hit, because a trigger anywhere in the + // catalogue is matched before any keyword: research-question's `was ist + // heute` was taking this whole shape of question off the clock. + ['Welcher Tag ist heute?', 'trigger'], + ['Was ist heute für ein Tag?', 'trigger'], ['what day is it tomorrow', 'trigger'], ['time right now please', 'search'], ['Kannst du die Uhrzeit sagen?', 'search'], diff --git a/src/skills/lookup-term/SKILL.md b/src/skills/lookup-term/SKILL.md index 8b8e317..52857df 100644 --- a/src/skills/lookup-term/SKILL.md +++ b/src/skills/lookup-term/SKILL.md @@ -24,6 +24,11 @@ jarvis: # *Was ist 1inch?* used to route nowhere at all. - '^\s*(was|wer)\s+ist\s+(?!(das|dies|es|los|passiert|dein|deine|mein|meine)\b)[^\s?]{1,24}\s*\??\s*$' - '\b(was|wer)\s+ist\s+(?=\S*\d)(?=\S*[a-z])\S+' + # An abbreviation is the same question as a bare name, asked the other way + # round, and it used to route nowhere: *was bedeutet TLDR* went to the page + # summariser and *what does TLDR mean* to no skill at all. + - "^\\s*(what|whats)('?s)?\\s+does\\s+[^\\s?]{1,24}\\s+mean\\s*\\??\\s*$" + - '^\s*was\s+(bedeutet|hei(ß|ss)t)\s+[^\s?]{1,24}\s*\??\s*$' exemplars: - user: What is 1Password? steps: diff --git a/src/skills/route.test.ts b/src/skills/route.test.ts index 969b6a2..c21b520 100644 --- a/src/skills/route.test.ts +++ b/src/skills/route.test.ts @@ -91,6 +91,13 @@ describe('routing by trigger', () => { ['Was kostet ein iPhone?', 'research-question'], ['Wie spät ist es?', 'current-date'], ['Ist heute Montag?', 'current-date'], + // `was ist heute` is a research-question trigger, and a trigger anywhere in + // the catalogue is matched before any keyword is — so the date question was + // being searched for on the web while `welcher tag` waited in the index. + ['Was ist heute für ein Tag?', 'current-date'], + ['Welcher Wochentag ist heute?', 'current-date'], + ['Was bedeutet TLDR?', 'lookup-term'], + ['Was heißt IBAN?', 'lookup-term'], ['Wie viel ist 7 mal 8?', 'arithmetic'], ['Wurzel aus 144', 'arithmetic'], ['Berechne 18 Prozent von 2450', 'arithmetic'], @@ -114,6 +121,27 @@ describe('routing by trigger', () => { expect(reason(message)).toBe('trigger') }) + it.each([ + // An abbreviation is a bare name asked about the other way round, so it + // belongs to the skill that searches for the term verbatim. *Was bedeutet + // TLDR?* reached the page summariser instead, which asked which page was + // meant; the English shape reached no skill at all. + ['What does TLDR mean?', 'lookup-term'], + ['what does IBAN mean', 'lookup-term'], + ])('routes the definition question %j to %s', (message, expected) => { + expect(routed(message)).toBe(expected) + expect(reason(message)).toBe('trigger') + }) + + it.each([ + // `tldr` still asks for a summary wherever it is not the subject. + ['tldr https://example.com/post', 'summarize-url'], + ['tl;dr please', 'summarize-url'], + ['Give me a tldr of the article', 'summarize-url'], + ])('keeps %j with %s', (message, expected) => { + expect(routed(message)).toBe(expected) + }) + it('leaves a linked weather site to summarize-url', () => { // `weather` and `forecast` both appear in the URL, and a search would // answer about somewhere else entirely. diff --git a/src/skills/summarize-url/SKILL.md b/src/skills/summarize-url/SKILL.md index a3672aa..e64e4c5 100644 --- a/src/skills/summarize-url/SKILL.md +++ b/src/skills/summarize-url/SKILL.md @@ -18,7 +18,11 @@ jarvis: - was steht auf der seite triggers: - 'https?://\S+' - - '\b(summari[sz]e|tl;?dr|what does .{0,20}(page|article|link) say)\b' + - '\b(summari[sz]e|what does .{0,20}(page|article|link) say)\b' + # `tldr` is a request to shorten something, except when it is the thing being + # asked about: *Was bedeutet TLDR?* reached here and was answered with "which + # page do you mean?", for a question needing no page at all. + - '(? { it('respects operator precedence', () => { @@ -37,4 +37,64 @@ describe('evaluateExpression', () => { it('rejects trailing garbage', () => { expect(() => evaluateExpression('1 + 1 oops')).toThrow(/Unknown function or constant|trailing/) }) + + it('says what the tool is for when it refuses', () => { + // The message goes back into the model's context as the whole result of a + // spent round, so it is the only chance to redirect the next attempt. + expect(() => evaluateExpression('5 miles in km')).toThrow(/arithmetic only/) + }) +}) + +/** + * Everything here was refused before, and every one of them is something the + * model actually writes. A refusal costs the round and sends it back to mental + * arithmetic, so being read correctly is the difference between an exact answer + * and a confident wrong one. + */ +describe('expressions as the model writes them', () => { + it.each([ + ['98,765 * 4,321', 426763565], + ['1,234,567 + 1', 1234568], + // The decimal comma, which is the same character in the other locale. + ['18,5 * 2', 37], + ['(17 * 23) / 4 =', 97.75], + ['2 x 3', 6], + ['7·8', 56], + ['15 ÷ 3', 5], + ['10 −4', 6], + ['$1200 * 1.19', 1428], + ['18% of 2450', 441], + ['18 percent of 2450', 441], + ['18 Prozent von 2450', 441], + ['2450 * 18%', 441], + // The addition must stay outside the fraction. + ['20% of 300 + 50', 110], + ])('reads %j as %d', (expression, expected) => { + expect(evaluateExpression(expression)).toBeCloseTo(expected, 6) + }) + + it('leaves the modulo operator alone', () => { + // `%` is documented as modulo, and only a percentage with no operand after + // it is read as one hundredth. + expect(evaluateExpression('12 % 5')).toBe(2) + expect(evaluateExpression('12 % (2 + 3)')).toBe(2) + }) + + it.each([ + // A discount is not a percentage of the number beside it, and guessing + // which would be a wrong answer rather than a refusal. + '18% off 2450', + // Unit conversion is not arithmetic, and no amount of reading helps. + '5 miles in km', + '32 fahrenheit in celsius', + ])('still refuses %j', (expression) => { + expect(() => evaluateExpression(expression)).toThrow() + }) +}) + +describe('normalizeExpression', () => { + it('leaves an expression the parser already reads untouched', () => { + expect(normalizeExpression('(2 + 3) * 4')).toBe('(2 + 3) * 4') + expect(normalizeExpression('sqrt(16) ^ 2')).toBe('sqrt(16) ^ 2') + }) }) diff --git a/src/tools/calculator.ts b/src/tools/calculator.ts index d7216da..4852b76 100644 --- a/src/tools/calculator.ts +++ b/src/tools/calculator.ts @@ -20,7 +20,56 @@ const FUNCTIONS: Record number> = { const CONSTANTS: Record = { pi: Math.PI, e: Math.E } -export function evaluateExpression(input: string): number { +/** + * What the model writes, mapped onto what the parser reads. + * + * A rejected expression is not a neutral outcome: the round is spent, and the + * model that gets `Unexpected trailing input` back usually answers from its own + * arithmetic instead — which is the failure this tool exists to prevent. Of + * eighteen expressions a 0.8B model plausibly produces, ten were refused, and + * none of them for a reason the user would recognise: `98,765 * 4,321`, + * `18% of 2450`, `(17 * 23) / 4 =`, `2 x 3`, `$1200 * 1.19`. + * + * So the shapes below are read rather than refused. Every one of them has a + * single arithmetic meaning; anything genuinely ambiguous is left to fail. + */ +export function normalizeExpression(input: string): string { + return ( + input + .trim() + // The model often echoes the question's own equals sign. + .replace(/\s*=\s*\??\s*$/, '') + // Currency is a unit, and this tool has none: the number is the argument. + .replace(/[$€£¥]/g, '') + // Operators as they are typed or pasted rather than as ASCII. + .replace(/[×⋅·∗]/g, '*') + .replace(/[÷∕]/g, '/') + .replace(/[−–]/g, '-') + // `x` between two numbers is a multiplication sign. It cannot be anything + // else here, since the parser has no variables to confuse it with. + .replace(/(\d)\s*[xX]\s*(?=[\d(.])/g, '$1*') + // A comma in groups of exactly three digits is a thousands separator; one + // followed by one or two is the decimal comma a German-speaking model + // writes. `2,450` is read as two thousand four hundred and fifty on that + // rule, which is what it means in the far commoner of the two locales. + .replace(/(\d)(?:,(\d{3}))+(?![\d,])/g, (match) => match.replace(/,/g, '')) + .replace(/(\d),(\d{1,2})(?![\d,])/g, '$1.$2') + // A percentage of something, in either language. Parenthesised because + // `20% of 300 + 50` must not become `20/100*300 + 50`'s cousin with the + // addition pulled inside the fraction. + .replace(/(\d+(?:\.\d+)?)\s*(?:%|percent|per cent|prozent)\s+(?:of|von)\s+/gi, '($1 / 100) * ') + // A trailing percentage — `2450 * 18%`. Only where no operand follows, so + // `12 % 5` is still the modulo the tool documents. + .replace(/(\d+(?:\.\d+)?)\s*%(?!\s*[\d(a-z])/gi, '($1 / 100)') + .trim() + ) +} + +/** Appended to the two failures a model can usually act on. */ +const ARITHMETIC_ONLY = 'This tool evaluates arithmetic only, so write the sum out in numbers.' + +export function evaluateExpression(rawInput: string): number { + const input = normalizeExpression(rawInput) let position = 0 const skipSpace = (): void => { @@ -94,7 +143,7 @@ export function evaluateExpression(input: string): number { position += word[0].length if (name in CONSTANTS) return CONSTANTS[name]! const fn = FUNCTIONS[name] - if (!fn) throw new Error(`Unknown function or constant: ${word[0]}`) + if (!fn) throw new Error(`Unknown function or constant: ${word[0]}. ${ARITHMETIC_ONLY}`) if (!consume('(')) throw new Error(`Expected "(" after ${name}`) const argument = parseExpression() if (!consume(')')) throw new Error('Missing closing parenthesis') @@ -106,7 +155,9 @@ export function evaluateExpression(input: string): number { const result = parseExpression() skipSpace() - if (position !== input.length) throw new Error(`Unexpected trailing input: "${input.slice(position)}"`) + if (position !== input.length) { + throw new Error(`Unexpected trailing input: "${input.slice(position)}". ${ARITHMETIC_ONLY}`) + } if (!Number.isFinite(result)) throw new Error('Result is not a finite number') return result } From abfae53778752ef54e526e865320ed17856546df Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 17:54:34 +0000 Subject: [PATCH 2/9] Give the model a converter, since a conversion is neither arithmetic nor a search Co-authored-by: Sebastian --- README.md | 23 +- src/agent/review.test.ts | 10 + src/agent/review.ts | 21 +- src/eval/scenarios.ts | 39 ++- src/skills/convert-units/SKILL.md | 52 ++++ src/skills/library.test.ts | 52 +++- src/skills/route.test.ts | 8 +- src/tools/builtins.ts | 23 +- src/tools/units.test.ts | 98 +++++++ src/tools/units.ts | 444 ++++++++++++++++++++++++++++++ 10 files changed, 750 insertions(+), 20 deletions(-) create mode 100644 src/skills/convert-units/SKILL.md create mode 100644 src/tools/units.test.ts create mode 100644 src/tools/units.ts diff --git a/README.md b/README.md index b2c4aa6..ee46fc3 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,7 @@ Three details make the app work from a repository sub-path rather than a domain | `web_search` | Full web search with no key; Wikipedia, LangSearch or Jina instead. | | `read_page` | Fetches a URL and returns its readable text. | | `calculator` | Exact arithmetic via a hand-written parser. | +| `convert` | Units: length, mass, temperature, volume, speed, area, data, time. | | `current_time` | Local date, time, and timezone. | | `weather` | Current conditions and a three-day outlook, from several forecasts. | | `memory` | Saves, lists, corrects and deletes what it remembers about you. | @@ -379,7 +380,7 @@ An exemplar can hold several steps, which is how a workflow gets taught. Split a Two further things a skill does. It **narrows the tool list** to what it declares, because tool-calling accuracy falls as the number of visible tools grows. And it can **override the reasoning budget** per skill. -Seven ship: `arithmetic`, `current-date`, `summarize-url`, `lookup-term`, `research-question`, `weather` and `memory`. +Eight ship: `arithmetic`, `convert-units`, `current-date`, `summarize-url`, `lookup-term`, `research-question`, `weather` and `memory`. ### Which skill, and when @@ -395,6 +396,16 @@ Two collisions are pinned by tests. Its priority sits above `current-date`, so _ The exemplars carry the part prose cannot. One quotes a reading whose sources are 3.4 °C apart and calls the temperature approximate; the other answers _will it rain tomorrow_ off the dated line rather than the `Now` line. Both are behaviours a 0.8B model gets wrong from a description and right from an example. +### Why `convert-units` exists + +_What is 32 fahrenheit in celsius_ was the one question in this app with no tool behind it at all. `lookup-term` excludes it by hand, because searching a measurement verbatim answers nothing. The calculator refuses it, correctly, since a conversion is not arithmetic. So it fell to the model, and unit arithmetic is exactly what a 0.8B model gets wrong while sounding certain — the failure the calculator exists to prevent, in the one place the calculator could not help. + +The [`convert` tool](#tools) is a finite table: length, mass, temperature, volume, speed, area, data and duration, with every spelling the question might use in either language, since the model passes the user's own word through as the argument. Three things are deliberately absent. **A conversion needing a second quantity fails** rather than guessing — grams to cups depends on what is in the cup, and an answer that looks exact and is not would be worse than the round it costs to say so. **A month and a year are not durations here**, because neither has a fixed length. **Bits are not units here either**: `Mb` and `MB` differ by a factor of eight and by one letter's case, which model output cannot be trusted to preserve. + +The skill sits above `arithmetic` in priority, because a conversion looks like a sum to a model holding a calculator. Its trigger for the `5 miles in km` shape anchors the **target** on a unit it knows, so _20 minutes to Berlin_ is not read as a conversion into a city; the reverse phrasings — _how many ounces is 200 grams_, _wie viele Zentimeter sind 3 Zoll_ — are triggers of their own. + +The answer check follows the tool: a converted number is held to the same standard as a calculated one, so a reply that quotes 8 kilometres for a result of 8.04672 is corrected, and one that rounds it to 8.05 is not. + ### Why `lookup-term` exists Asked _what is 1inch_, the model searched for **`1 inch to measurement in centimeters`**. It split the token, decided on its own that the question was about unit conversion, and searched for that instead — so the results never got the chance to mention that 1inch is a DEX aggregator. @@ -480,11 +491,11 @@ A skill fires on some requests. This runs on all of them. Between the model settling on an answer and that answer reaching the screen, `src/agent/review.ts` reads it back against what the turn actually produced — the results the tools returned, and the URLs already in the conversation. Three things are checked: -| Check | Fires when | -| ----------------- | --------------------------------------------------------------------------- | -| `wrong-number` | The calculator returned a value the answer states nowhere, at any precision | -| `invented-source` | The answer cites a URL that no tool returned and nobody supplied | -| `missing-source` | Tools returned sources and the answer cites none | +| Check | Fires when | +| ----------------- | -------------------------------------------------------------------- | +| `wrong-number` | `calculator` or `convert` returned a value the answer states nowhere | +| `invented-source` | The answer cites a URL that no tool returned and nobody supplied | +| `missing-source` | Tools returned sources and the answer cites none | A failed check costs one further generation. The model is handed its own draft and told what to change — _The calculator returned 6748 \* 9 = 60732. Give that number, exactly as it came back._ — and the correction replaces the draft only if it leaves fewer problems behind. Otherwise the draft stands. That gate is the important half: the correction comes from the same 0.8B model, so a mechanism that could not tell an improvement from a regression would be a coin toss on every reply. diff --git a/src/agent/review.test.ts b/src/agent/review.test.ts index b7c6654..e683fac 100644 --- a/src/agent/review.test.ts +++ b/src/agent/review.test.ts @@ -61,6 +61,16 @@ describe('reviewAnswer', () => { expect(checks('I could not work that out.', failed)).toEqual([]) }) + it('holds a conversion to the same standard, unit and all', () => { + // `convert` writes its result as `5 mi = 8.04672 km`, so the number is not + // the last thing on the line and used to be read as no number at all. + const converted = evidence({ toolResults: [{ tool: 'convert', result: '5 mi = 8.04672 km' }] }) + + expect(checks('5 miles is 8.05 km.', converted)).toEqual([]) + expect(checks('5 miles is roughly 8 kilometres.', converted)).toEqual(['wrong-number']) + expect(reviewAnswer('About 8 km.', converted)[0]?.instruction).toContain('The conversion returned') + }) + it('reports one correction however many sums were dropped', () => { const two = evidence({ toolResults: [ diff --git a/src/agent/review.ts b/src/agent/review.ts index e39460b..a9738fb 100644 --- a/src/agent/review.ts +++ b/src/agent/review.ts @@ -86,17 +86,23 @@ function isGrounded(cited: Located, known: Located[]): boolean { ) } -/** The shape `calculator` returns: `expression = value`. */ -const CALCULATION = /^(.+) = (-?\d[\d.]*(?:e[+-]?\d+)?)$/i +/** + * The shape both computing tools return: `expression = value`, with a unit after + * the value where `convert` produced it — `5 mi = 8.04672 km`. + */ +const CALCULATION = /^(.+) = (-?\d[\d.]*(?:e[+-]?\d+)?)(?:\s+\S{1,10})?$/i + +/** The tools whose result is a number the answer is then expected to state. */ +const COMPUTED = new Set(['calculator', 'convert']) -function calculations(evidence: ReviewEvidence): { expression: string; value: number }[] { - const found: { expression: string; value: number }[] = [] +function calculations(evidence: ReviewEvidence): { tool: string; expression: string; value: number }[] { + const found: { tool: string; expression: string; value: number }[] = [] for (const { tool, result } of evidence.toolResults) { - if (tool !== 'calculator') continue + if (!COMPUTED.has(tool)) continue const match = CALCULATION.exec(result.trim()) if (!match?.[1] || !match[2]) continue const value = Number(match[2]) - if (Number.isFinite(value)) found.push({ expression: match[1], value }) + if (Number.isFinite(value)) found.push({ tool, expression: match[1], value }) } return found } @@ -155,9 +161,10 @@ export function reviewAnswer(answer: string, evidence: ReviewEvidence): ReviewFi // result has the same fix however many sums it dropped. const missed = calculations(evidence).find(({ value }) => !statesNumber(draft, value)) if (missed) { + const source = missed.tool === 'convert' ? 'The conversion' : 'The calculator' findings.push({ check: 'wrong-number', - instruction: `The calculator returned ${missed.expression} = ${missed.value}. Give that number, exactly as it came back.`, + instruction: `${source} returned ${missed.expression} = ${missed.value}. Give that number, exactly as it came back.`, }) } diff --git a/src/eval/scenarios.ts b/src/eval/scenarios.ts index c8c2566..1114549 100644 --- a/src/eval/scenarios.ts +++ b/src/eval/scenarios.ts @@ -15,7 +15,8 @@ import type { MemoryKind } from '@/memory/types' -export type Category = 'arithmetic' | 'time' | 'recall' | 'memory' | 'no-tool' | 'web' | 'lookup' | 'weather' +export type Category = + 'arithmetic' | 'convert' | 'time' | 'recall' | 'memory' | 'no-tool' | 'web' | 'lookup' | 'weather' export interface Invocation { name: string @@ -150,6 +151,42 @@ export const SCENARIOS: Scenario[] = [ expectTool: 'calculator', accept: (answer) => hasNumber(answer, '1048576'), }, + { + id: 'convert-temperature', + category: 'convert', + // The prompt that had no tool at all: a conversion is not arithmetic, so the + // calculator refused it and the model answered from its own head. + prompt: 'What is 32 fahrenheit in celsius?', + expectTool: 'convert', + accept: (answer) => hasNumber(answer, '0'), + }, + { + id: 'convert-german-distance', + category: 'convert', + prompt: 'Wie viel sind 5 Meilen in Kilometer?', + expectTool: 'convert', + // Whether the units survived as the user wrote them is the half a tool name + // cannot see: `5` and `km` reaching the tool as `5 miles` and `Berlin` is a + // correct tool call with an answer-destroying argument. + acceptCall: (calls) => + calls.some( + (call) => + call.name === 'convert' && + /me?ile/i.test(String(call.arguments.from ?? call.arguments.value ?? '')) && + /km|kilometer/i.test(String(call.arguments.to ?? '')), + ), + accept: (answer) => hasNumber(answer, '8'), + }, + { + id: 'convert-mass-not-volume', + category: 'convert', + // The tool refuses this, because how much a cup of something weighs depends + // on what is in it. What is measured here is whether the model relays the + // refusal instead of inventing the number it was denied. + prompt: 'How many cups is 200 grams?', + expectTool: 'convert', + accept: matches(/depends|cannot|can't|not possible|what|which|ingredient|density|volume/i), + }, { id: 'time-current-year', category: 'time', diff --git a/src/skills/convert-units/SKILL.md b/src/skills/convert-units/SKILL.md new file mode 100644 index 0000000..bc1751a --- /dev/null +++ b/src/skills/convert-units/SKILL.md @@ -0,0 +1,52 @@ +--- +name: convert-units +description: Converts a quantity into different units with the convert tool. Use when the user asks what a length, mass, temperature, volume, speed, area, amount of data or duration is in other units. +jarvis: + # Above `arithmetic`, because a conversion looks like a sum to a model that has + # a calculator and no converter — and the calculator refuses it, correctly. + priority: 32 + tools: + - convert + keywords: + - in celsius + - in fahrenheit + - in inches + - in kilometers + - how many ounces + - how many pounds + - umgerechnet + - in kilometer + - wie viele gramm + - wie viele zentimeter + triggers: + - '\b(convert|umrechnen|rechne .{0,20}um)\b' + # `N in `, with the target anchored at the end so that + # *20 minutes to Berlin* is not read as a conversion into a city. + - "\\b\\d+(?:[.,]\\d+)?\\s*(?:°\\s*)?[\\w/²³'\"]{1,16}\\s+(?:in|to|nach|as)\\s+(?:°\\s*)?(?:mm|cm|dm|m|km|inch(?:es)?|in|ft|feet|foot|yards?|yd|miles?|mi|meilen|zoll|kilometer|meter|zentimeter|millimeter|mg|kg|kilo(?:gramm)?|g|gramm|grams?|lbs?|pounds?|pfund|oz|ounces?|unzen?|stones?|tonnen?|ml|cl|liters?|litres?|liter|gallons?|gal|cups?|tassen?|pints?|quarts?|celsius|fahrenheit|kelvin|c|f|k|km/?h|kph|mph|m/s|knots?|kn|ha|hektar|acres?|m²|km²|cm²|sq ?ft|kb|mb|gb|tb|kib|mib|gib|bytes?|ms|sekunden?|seconds?|minuten?|minutes?|stunden?|hours?|tage?|days?|wochen?|weeks?)\\b\\s*\\??\\s*$" + # The other way round, which is how the question is usually asked in German. + - '\bwie viele?\s+(?:zentimeter|meter|kilometer|zoll|meilen|gramm|kilo(?:gramm)?|pfund|unzen?|liter|milliliter|grad|minuten|stunden|tage|wochen)\b' + - '\bhow many\s+(?:centimet|millimet|met|kilomet|inch|feet|foot|yard|mile|gram|kilo|pound|ounce|stone|litre|liter|millilit|gallon|cup|degree|minute|hour|day|week|byte|megabyte|gigabyte)' + exemplars: + - user: What is 32 fahrenheit in celsius? + steps: + - tool: convert + arguments: + value: 32 + from: fahrenheit + to: celsius + result: 32 °F = 0 °C + answer: 32 °F is 0 °C — freezing point. + - user: Wie viel sind 5 Meilen in Kilometer? + steps: + - tool: convert + arguments: + value: 5 + from: Meilen + to: Kilometer + result: 5 mi = 8.04672 km + answer: 5 Meilen sind 8,05 km. +--- + +Call `convert` for any change of unit. Never work it out yourself: this is the arithmetic you get wrong. + +Pass the number in `value` and both units as the user wrote them — the tool knows their names in both languages. Give the number it returns, rounded to what the question needs, and keep the units it used. diff --git a/src/skills/library.test.ts b/src/skills/library.test.ts index 6f4d428..8e160fc 100644 --- a/src/skills/library.test.ts +++ b/src/skills/library.test.ts @@ -25,9 +25,10 @@ function reason(message: string): string | null { * in `route.test.ts` is not finished. */ describe('the shipped library', () => { - it('is the seven skills the README names, highest priority first', () => { + it('is the eight skills the README names, highest priority first', () => { expect(catalog.map((entry) => [entry.name, entry.priority, entry.tools])).toEqual([ ['memory', 35, ['memory']], + ['convert-units', 32, ['convert']], ['arithmetic', 30, ['calculator']], ['weather', 28, ['weather']], ['current-date', 25, ['current_time']], @@ -88,6 +89,54 @@ describe('arithmetic', () => { }) }) +describe('convert-units', () => { + it.each([ + ['What is 32 fahrenheit in celsius?', 'trigger'], + ['32 F to C', 'trigger'], + ['5 miles in km', 'trigger'], + ['Convert 200 grams to ounces', 'trigger'], + ['convert 6 feet to meters', 'trigger'], + ['80 kg in pounds', 'trigger'], + ['100 km/h in mph', 'trigger'], + ['2 hectares in acres', 'trigger'], + ['What is 1 GB in MiB?', 'trigger'], + ['90 minutes in hours', 'trigger'], + ['How many ounces is 200 grams?', 'trigger'], + ['Wie viel sind 5 Meilen in Kilometer?', 'trigger'], + ['Wie viele Zentimeter sind 3 Zoll?', 'trigger'], + ['Rechne 80 kg in Pfund um', 'trigger'], + ['3 Zoll in cm', 'trigger'], + ['200 Gramm in Unzen', 'trigger'], + ['Was ist das in Celsius?', 'search'], + ['30 Grad umgerechnet in Fahrenheit', 'search'], + ])('takes %j by %s', (message, how) => { + expect(routed(message)).toBe('convert-units') + expect(reason(message)).toBe(how) + }) + + it.each([ + // A destination is not a unit, and the round spent finding that out is the + // reason the target of a conversion has to be a unit this tool knows. + '20 minutes to Berlin', + 'Wie komme ich von Berlin nach München?', + '3 hours in Lisbon', + ])('leaves %j alone', (message) => { + expect(routed(message)).not.toBe('convert-units') + }) + + it.each([ + // Arithmetic is arithmetic even where it mentions a percentage or a power, + // and it sits directly below this skill in priority. + ['How much is 18 percent of 2450?', 'arithmetic'], + ['What is 98765 * 4321?', 'arithmetic'], + ['What is 2 to the power of 20?', 'arithmetic'], + // A temperature question about the world, not about a scale. + ['Wie warm ist es in München?', 'weather'], + ])('does not take %j from %s', (message, expected) => { + expect(routed(message)).toBe(expected) + }) +}) + describe('weather', () => { it.each([ ["What's the weather in Berlin?", 'trigger'], @@ -336,7 +385,6 @@ describe('priority and near misses', () => { 'I was born in 2024', 'I currently live in Berlin', 'Was machst du heute?', - 'What is 32 fahrenheit in celsius', 'What time is it in Tokyo?', 'Wie spät ist es in Tokio?', 'Wie spät ist es in Berlin?', diff --git a/src/skills/route.test.ts b/src/skills/route.test.ts index c21b520..56b7cad 100644 --- a/src/skills/route.test.ts +++ b/src/skills/route.test.ts @@ -154,6 +154,11 @@ describe('routing by trigger', () => { ['What is 2 to the power of 20?', 'arithmetic'], ['What is 98765 * 4321?', 'arithmetic'], ['What is the date today?', 'current-date'], + // A conversion, not a name: the digit-bearing token lookup-term matches has + // to carry letters too, or `1inch` and `32` are the same shape to it. This + // used to reach no skill at all, which left the one question with no tool + // behind it to the model's own arithmetic. + ['What is 32 fahrenheit in celsius', 'convert-units'], ])('does not let lookup-term steal %j from %s', (message, expected) => { expect(routed(message)).toBe(expected) }) @@ -200,9 +205,6 @@ describe('routing nothing at all', () => { // `heute` was a current-date keyword, which turned every mention of today // into a question about the date. 'Was machst du heute?', - // A conversion, not a name: the digit-bearing token lookup-term matches has - // to carry letters too, or `1inch` and `32` are the same shape to it. - 'What is 32 fahrenheit in celsius', // `current_time` reads the user's own clock and no other, so a question // about somewhere else must not reach it and answer with the wrong hour. 'What time is it in Tokyo?', diff --git a/src/tools/builtins.ts b/src/tools/builtins.ts index f7f8a4a..47a31bd 100644 --- a/src/tools/builtins.ts +++ b/src/tools/builtins.ts @@ -1,6 +1,7 @@ import { evaluateExpression } from './calculator' import { memory } from './memory' import { defineTool, type Tool } from './types' +import { convertQuantity } from './units' import { DEFAULT_WEB_ACCESS, readPage, searchWeb, type SearchProvider, type WebAccessConfig } from './web' import { weatherReport } from './weather' @@ -110,6 +111,26 @@ export const weather = defineTool( }, ) +/** + * The one thing on this list the model was previously left to do in its head. + * A conversion is not arithmetic, so the calculator refuses it, and searching + * for it verbatim answers nothing — see `src/tools/units.ts`. + */ +export const convert = defineTool( + 'convert', + 'Convert a quantity into different units: length, mass, temperature, volume, speed, area, data or duration. Use whenever the user asks what something is in other units.', + { + type: 'object', + properties: { + value: { type: 'string', description: 'The number to convert. For example: 32' }, + from: { type: 'string', description: 'The unit it is given in. For example: fahrenheit' }, + to: { type: 'string', description: 'The unit it is wanted in. For example: celsius' }, + }, + required: ['value', 'from', 'to'], + }, + async (args) => convertQuantity(args), +) + export const currentTime = defineTool( 'current_time', "Return the user's current date, time and timezone. Use whenever the answer depends on today's date.", @@ -131,7 +152,7 @@ export const currentTime = defineTool( * who asked not to be remembered. */ export function createBuiltinTools(config: WebAccessConfig, options: { memory?: boolean } = {}): Tool[] { - const tools = [createWebSearch(config), createReadPage(config), calculator, currentTime, weather] + const tools = [createWebSearch(config), createReadPage(config), calculator, convert, currentTime, weather] return options.memory === false ? tools : [...tools, memory] } diff --git a/src/tools/units.test.ts b/src/tools/units.test.ts new file mode 100644 index 0000000..bb583ab --- /dev/null +++ b/src/tools/units.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from 'vitest' +import { convertQuantity, formatQuantity, readConversionRequest } from './units' + +describe('convertQuantity', () => { + it.each([ + // The prompt this tool exists for: no skill routed it, the calculator + // refused it, and the model did it in its head. + [{ value: '32', from: 'fahrenheit', to: 'celsius' }, '32 °F = 0 °C'], + [{ value: '100', from: 'celsius', to: 'fahrenheit' }, '100 °C = 212 °F'], + [{ value: '0', from: 'celsius', to: 'kelvin' }, '0 °C = 273.15 K'], + [{ value: '5', from: 'miles', to: 'km' }, '5 mi = 8.04672 km'], + [{ value: '200', from: 'grams', to: 'ounces' }, '200 g = 7.05479 oz'], + [{ value: '1', from: 'inch', to: 'cm' }, '1 in = 2.54 cm'], + [{ value: '80', from: 'kg', to: 'lbs' }, '80 kg = 176.37 lb'], + [{ value: '100', from: 'km/h', to: 'mph' }, '100 km/h = 62.1371 mph'], + [{ value: '2', from: 'hectares', to: 'acres' }, '2 ha = 4.94211 acre'], + [{ value: '1', from: 'GB', to: 'MiB' }, '1 GB = 953.674 MiB'], + [{ value: '90', from: 'minutes', to: 'hours' }, '90 min = 1.5 h'], + [{ value: '1.5', from: 'liters', to: 'cups' }, '1.5 l = 6.34013 cup'], + ])('converts %o to %s', (args, expected) => { + expect(convertQuantity(args)).toBe(expected) + }) + + it.each([ + // German, because the model passes the word the user wrote straight through. + [{ value: '5', from: 'Meilen', to: 'Kilometer' }, '5 mi = 8.04672 km'], + [{ value: '250', from: 'Gramm', to: 'Pfund' }, '250 g = 0.551156 lb'], + [{ value: '30', from: 'Grad Celsius', to: 'Fahrenheit' }, '30 °C = 86 °F'], + [{ value: '2', from: 'Zoll', to: 'cm' }, '2 in = 5.08 cm'], + ])('reads the German %o as %s', (args, expected) => { + expect(convertQuantity(args)).toBe(expected) + }) + + it('accepts the degree sign and the word both', () => { + expect(convertQuantity({ value: '32', from: '°F', to: '°C' })).toBe('32 °F = 0 °C') + expect(convertQuantity({ value: '32', from: 'degrees fahrenheit', to: 'celsius' })).toBe('32 °F = 0 °C') + }) + + it('refuses a conversion that depends on something it was not given', () => { + // Grams to cups needs to know what is in the cup. An answer here would look + // exact and be wrong, which is worse than the round it costs to say no. + expect(() => convertQuantity({ value: '200', from: 'grams', to: 'cups' })).toThrow( + /mass.*volume|volume.*mass/, + ) + }) + + it('names the unit it could not place', () => { + expect(() => convertQuantity({ value: '5', from: 'smoots', to: 'm' })).toThrow(/Unknown unit "smoots"/) + }) + + it('refuses a value that is not a number', () => { + expect(() => convertQuantity({ value: 'some', from: 'kg', to: 'lb' })).toThrow(/must be a number/) + }) +}) + +/** + * The schema asks for three arguments, and a 0.8B model routinely sends the + * whole phrase in one of them. Each shape below was worth reading rather than + * rejecting: a rejection spends the tool round and teaches nothing. + */ +describe('readConversionRequest', () => { + it.each([ + [{ value: '5 miles to km' }, { value: 5, from: 'miles', to: 'km' }], + [{ value: '32 fahrenheit in celsius' }, { value: 32, from: 'fahrenheit', to: 'celsius' }], + [{ value: '5 Meilen nach Kilometer' }, { value: 5, from: 'Meilen', to: 'Kilometer' }], + [ + { value: '5 miles', to: 'km' }, + { value: 5, from: 'miles', to: 'km' }, + ], + [ + { value: '5', from: '5 miles', to: 'km' }, + { value: 5, from: 'miles', to: 'km' }, + ], + [ + { value: '1,5', from: 'm', to: 'cm' }, + { value: 1.5, from: 'm', to: 'cm' }, + ], + ])('reads %o as %o', (args, expected) => { + expect(readConversionRequest(args)).toEqual(expected) + }) + + it('converts what it read back out of a phrase', () => { + expect(convertQuantity({ value: '5 miles to km' })).toBe('5 mi = 8.04672 km') + expect(convertQuantity({ value: '32 fahrenheit in celsius' })).toBe('32 °F = 0 °C') + }) +}) + +describe('formatQuantity', () => { + it('keeps six significant digits and drops the rest', () => { + expect(formatQuantity(8.046719999999999)).toBe('8.04672') + expect(formatQuantity(212)).toBe('212') + expect(formatQuantity(0.5)).toBe('0.5') + }) + + it('falls back to exponent notation where a decimal would be unreadable', () => { + expect(formatQuantity(0.00000123)).toBe('1.2300e-6') + }) +}) diff --git a/src/tools/units.ts b/src/tools/units.ts new file mode 100644 index 0000000..ad74b37 --- /dev/null +++ b/src/tools/units.ts @@ -0,0 +1,444 @@ +/** + * Unit conversion, because nothing else here can do it. + * + * `What is 32 fahrenheit in celsius` reaches no skill on purpose — searching for + * it verbatim answers nothing — and the calculator refuses it, correctly, since + * a conversion is not arithmetic. So the model was left to do it in its head, + * which is the one thing this app has measured it doing badly and confidently. + * + * The table is deliberately finite. Every unit here is one somebody asks about + * in a sentence; anything whose conversion depends on a second quantity is left + * out rather than guessed at, which is why grams to cups fails and fuel + * consumption is absent altogether. + */ + +export type Dimension = 'length' | 'mass' | 'temperature' | 'volume' | 'speed' | 'area' | 'data' | 'time' + +interface Unit { + dimension: Dimension + /** How the result names it, which is not always how the user wrote it. */ + label: string + /** Multiplier onto the dimension's base unit. Temperature carries none. */ + factor: number +} + +/** + * The base of each dimension is the SI unit where there is one: metre, kilogram, + * litre, metre per second, square metre, byte, second. Temperature is converted + * through degrees Celsius by hand, since a scale with an offset has no factor. + */ +const UNITS: Record = { + // Length, base metre. + mm: { dimension: 'length', label: 'mm', factor: 0.001 }, + cm: { dimension: 'length', label: 'cm', factor: 0.01 }, + dm: { dimension: 'length', label: 'dm', factor: 0.1 }, + m: { dimension: 'length', label: 'm', factor: 1 }, + km: { dimension: 'length', label: 'km', factor: 1000 }, + inch: { dimension: 'length', label: 'in', factor: 0.0254 }, + ft: { dimension: 'length', label: 'ft', factor: 0.3048 }, + yd: { dimension: 'length', label: 'yd', factor: 0.9144 }, + mi: { dimension: 'length', label: 'mi', factor: 1609.344 }, + nmi: { dimension: 'length', label: 'nmi', factor: 1852 }, + + // Mass, base kilogram. A stone is 14 pounds and is still how a British + // question states a body weight. + mg: { dimension: 'mass', label: 'mg', factor: 0.000001 }, + g: { dimension: 'mass', label: 'g', factor: 0.001 }, + kg: { dimension: 'mass', label: 'kg', factor: 1 }, + t: { dimension: 'mass', label: 't', factor: 1000 }, + oz: { dimension: 'mass', label: 'oz', factor: 0.028349523125 }, + lb: { dimension: 'mass', label: 'lb', factor: 0.45359237 }, + st: { dimension: 'mass', label: 'st', factor: 6.35029318 }, + + // Temperature. `factor` is unused and set to 1 so the type stays one shape. + celsius: { dimension: 'temperature', label: '°C', factor: 1 }, + fahrenheit: { dimension: 'temperature', label: '°F', factor: 1 }, + kelvin: { dimension: 'temperature', label: 'K', factor: 1 }, + + // Volume, base litre. The US gallon and its subdivisions, because the + // questions that use them are written in American recipes and forecasts; the + // imperial gallon differs by a fifth and would be a wrong answer, so it is + // named separately rather than merged. + ml: { dimension: 'volume', label: 'ml', factor: 0.001 }, + cl: { dimension: 'volume', label: 'cl', factor: 0.01 }, + l: { dimension: 'volume', label: 'l', factor: 1 }, + m3: { dimension: 'volume', label: 'm³', factor: 1000 }, + floz: { dimension: 'volume', label: 'fl oz', factor: 0.0295735295625 }, + cup: { dimension: 'volume', label: 'cup', factor: 0.2365882365 }, + pt: { dimension: 'volume', label: 'pt', factor: 0.473176473 }, + qt: { dimension: 'volume', label: 'qt', factor: 0.946352946 }, + gal: { dimension: 'volume', label: 'gal', factor: 3.785411784 }, + impgal: { dimension: 'volume', label: 'imp gal', factor: 4.54609 }, + tbsp: { dimension: 'volume', label: 'tbsp', factor: 0.01478676478125 }, + tsp: { dimension: 'volume', label: 'tsp', factor: 0.00492892159375 }, + + // Speed, base metre per second. + mps: { dimension: 'speed', label: 'm/s', factor: 1 }, + kmh: { dimension: 'speed', label: 'km/h', factor: 1 / 3.6 }, + mph: { dimension: 'speed', label: 'mph', factor: 0.44704 }, + kn: { dimension: 'speed', label: 'kn', factor: 0.514444444444 }, + + // Area, base square metre. + cm2: { dimension: 'area', label: 'cm²', factor: 0.0001 }, + m2: { dimension: 'area', label: 'm²', factor: 1 }, + km2: { dimension: 'area', label: 'km²', factor: 1000000 }, + ha: { dimension: 'area', label: 'ha', factor: 10000 }, + acre: { dimension: 'area', label: 'acre', factor: 4046.8564224 }, + sqft: { dimension: 'area', label: 'sq ft', factor: 0.09290304 }, + sqmi: { dimension: 'area', label: 'sq mi', factor: 2589988.110336 }, + + // Data, base byte. Decimal and binary both, because a disk is sold in one and + // reported in the other, and that gap is exactly what gets asked about. Bits + // are left out: `Mb` and `MB` differ by a factor of eight and by one letter's + // case, which the model's own output cannot be trusted to preserve. + byte: { dimension: 'data', label: 'bytes', factor: 1 }, + kb: { dimension: 'data', label: 'kB', factor: 1000 }, + mb: { dimension: 'data', label: 'MB', factor: 1000000 }, + gb: { dimension: 'data', label: 'GB', factor: 1000000000 }, + tb: { dimension: 'data', label: 'TB', factor: 1000000000000 }, + kib: { dimension: 'data', label: 'KiB', factor: 1024 }, + mib: { dimension: 'data', label: 'MiB', factor: 1048576 }, + gib: { dimension: 'data', label: 'GiB', factor: 1073741824 }, + tib: { dimension: 'data', label: 'TiB', factor: 1099511627776 }, + + // Duration, base second. It stops at a week: a month is not a fixed length + // and a year is only one to four decimal places, so both would be answers + // that look exact and are not. + ms: { dimension: 'time', label: 'ms', factor: 0.001 }, + s: { dimension: 'time', label: 's', factor: 1 }, + min: { dimension: 'time', label: 'min', factor: 60 }, + h: { dimension: 'time', label: 'h', factor: 3600 }, + d: { dimension: 'time', label: 'd', factor: 86400 }, + week: { dimension: 'time', label: 'weeks', factor: 604800 }, +} + +/** + * Every spelling the model or the user might arrive with, mapped to a unit id. + * + * German is in here for the same reason the skills have German triggers: the app + * answers in the language it was asked in, and a 0.8B model passes the word the + * user wrote straight through as the argument. + */ +const ALIASES: Record = { + // Length + millimeter: 'mm', + millimetre: 'mm', + millimeters: 'mm', + zentimeter: 'cm', + centimeter: 'cm', + centimetre: 'cm', + centimeters: 'cm', + centimetres: 'cm', + dezimeter: 'dm', + meter: 'm', + metre: 'm', + meters: 'm', + metres: 'm', + kilometer: 'km', + kilometre: 'km', + kilometers: 'km', + kilometres: 'km', + kilometern: 'km', + in: 'inch', + '"': 'inch', + zoll: 'inch', + inches: 'inch', + foot: 'ft', + feet: 'ft', + fuss: 'ft', + fuß: 'ft', + "'": 'ft', + yard: 'yd', + yards: 'yd', + mile: 'mi', + miles: 'mi', + meile: 'mi', + meilen: 'mi', + 'nautical mile': 'nmi', + 'nautical miles': 'nmi', + seemeile: 'nmi', + + // Mass + milligram: 'mg', + milligramm: 'mg', + gram: 'g', + gramm: 'g', + grams: 'g', + gramme: 'g', + grammes: 'g', + kilo: 'kg', + kilogram: 'kg', + kilogramm: 'kg', + kilograms: 'kg', + kilogrammes: 'kg', + tonne: 't', + tonnes: 't', + tonnen: 't', + ton: 't', + ounce: 'oz', + ounces: 'oz', + unze: 'oz', + unzen: 'oz', + lbs: 'lb', + pound: 'lb', + pounds: 'lb', + pfund: 'lb', + stone: 'st', + stones: 'st', + + // Temperature + c: 'celsius', + '°c': 'celsius', + celcius: 'celsius', + grad: 'celsius', + 'grad celsius': 'celsius', + f: 'fahrenheit', + '°f': 'fahrenheit', + 'grad fahrenheit': 'fahrenheit', + k: 'kelvin', + + // Volume + milliliter: 'ml', + millilitre: 'ml', + centiliter: 'cl', + liter: 'l', + litre: 'l', + liters: 'l', + litres: 'l', + 'cubic meter': 'm3', + 'cubic metre': 'm3', + kubikmeter: 'm3', + 'fluid ounce': 'floz', + 'fluid ounces': 'floz', + 'fl oz': 'floz', + cups: 'cup', + tasse: 'cup', + tassen: 'cup', + pint: 'pt', + pints: 'pt', + quart: 'qt', + quarts: 'qt', + gallon: 'gal', + gallons: 'gal', + gallone: 'gal', + gallonen: 'gal', + 'imperial gallon': 'impgal', + 'imperial gallons': 'impgal', + tablespoon: 'tbsp', + tablespoons: 'tbsp', + esslöffel: 'tbsp', + teaspoon: 'tsp', + teaspoons: 'tsp', + teelöffel: 'tsp', + + // Speed + 'm/s': 'mps', + 'meters per second': 'mps', + 'km/h': 'kmh', + kph: 'kmh', + 'kilometers per hour': 'kmh', + 'kilometer pro stunde': 'kmh', + stundenkilometer: 'kmh', + 'miles per hour': 'mph', + 'meilen pro stunde': 'mph', + knot: 'kn', + knots: 'kn', + knoten: 'kn', + + // Area + cm2: 'cm2', + 'cm²': 'cm2', + 'm²': 'm2', + 'square meter': 'm2', + 'square metre': 'm2', + 'square meters': 'm2', + quadratmeter: 'm2', + 'km²': 'km2', + 'square kilometer': 'km2', + quadratkilometer: 'km2', + hectare: 'ha', + hectares: 'ha', + hektar: 'ha', + acres: 'acre', + 'square foot': 'sqft', + 'square feet': 'sqft', + 'sq ft': 'sqft', + ft2: 'sqft', + 'square mile': 'sqmi', + 'square miles': 'sqmi', + quadratmeilen: 'sqmi', + + // Data + b: 'byte', + bytes: 'byte', + kilobyte: 'kb', + kilobytes: 'kb', + megabyte: 'mb', + megabytes: 'mb', + gigabyte: 'gb', + gigabytes: 'gb', + terabyte: 'tb', + terabytes: 'tb', + kibibyte: 'kib', + mebibyte: 'mib', + gibibyte: 'gib', + + // Duration + millisecond: 'ms', + milliseconds: 'ms', + millisekunden: 'ms', + sec: 's', + secs: 's', + second: 's', + seconds: 's', + sekunde: 's', + sekunden: 's', + mins: 'min', + minute: 'min', + minutes: 'min', + minuten: 'min', + hr: 'h', + hrs: 'h', + hour: 'h', + hours: 'h', + stunde: 'h', + stunden: 'h', + day: 'd', + days: 'd', + tag: 'd', + tage: 'd', + weeks: 'week', + woche: 'week', + wochen: 'week', +} + +/** What a dimension is called when the failure has to name it. */ +const DIMENSION_NAMES: Record = { + length: 'a length', + mass: 'a mass', + temperature: 'a temperature', + volume: 'a volume', + speed: 'a speed', + area: 'an area', + data: 'an amount of data', + time: 'a duration', +} + +function resolve(raw: string): { id: string; unit: Unit } | null { + const cleaned = raw + .trim() + .toLowerCase() + // `°C`, `° C` and `degrees C` all name the same scale. + .replace(/degrees?\s+/g, '') + .replace(/°\s*/g, '°') + .replace(/\.$/, '') + const id = cleaned in UNITS ? cleaned : (ALIASES[cleaned] ?? ALIASES[cleaned.replace(/^°/, '')]) + const unit = id ? UNITS[id] : undefined + return id && unit ? { id, unit } : null +} + +/** Through degrees Celsius, since a scale with an offset has no factor. */ +function toCelsius(value: number, from: string): number { + if (from === 'fahrenheit') return ((value - 32) * 5) / 9 + if (from === 'kelvin') return value - 273.15 + return value +} + +function fromCelsius(value: number, to: string): number { + if (to === 'fahrenheit') return (value * 9) / 5 + 32 + if (to === 'kelvin') return value + 273.15 + return value +} + +/** + * Six significant digits, trailing zeros dropped. + * + * Enough that a conversion is not silently rounded into a different answer, few + * enough that `8.04672 km` does not arrive as `8.046719999999999`. + */ +export function formatQuantity(value: number): string { + if (!Number.isFinite(value)) throw new Error('Result is not a finite number') + const magnitude = Math.abs(value) + if (magnitude !== 0 && (magnitude < 0.0001 || magnitude >= 1e15)) { + return value.toExponential(4).replace(/e\+?/, 'e') + } + return String(Number(value.toPrecision(6))) +} + +export interface ConversionRequest { + value: number + from: string + to: string +} + +/** + * Reads a conversion out of whatever the model passed. + * + * The schema asks for three arguments and the exemplars show three, but a 0.8B + * model also writes the whole phrase into one of them — `value: "5 miles to + * km"`, or `from: "5 miles"`. Refusing that spends the round on a rejection + * instead of an answer, and the phrase says exactly what was meant. + */ +export function readConversionRequest(args: Record): ConversionRequest { + const raw = (key: string): string => String(args[key] ?? '').trim() + const parts = [raw('value'), raw('from'), raw('to')].filter(Boolean).join(' ') + const explicitFrom = raw('from') + const explicitTo = raw('to') + + // A number in `value` and units either side of it, wherever they arrived. + const phrase = + /(-?\d+(?:[.,]\d+)?)\s*([^\d]*?)\s*(?:\b(?:in|to|nach|as|into|umrechnen)\b|=|→)\s*(.+)$/i.exec(parts) + + if (phrase?.[1] && phrase[3]) { + const from = (phrase[2] ?? '').trim() + return { + value: Number(phrase[1].replace(',', '.')), + from: from || explicitFrom, + to: phrase[3].trim(), + } + } + + const NUMBER = /-?\d+(?:[.,]\d+)?/ + const value = raw('value') + const number = NUMBER.exec(value || parts) + if (!number) throw new Error('value must be a number, for example 32') + + return { + value: Number(number[0].replace(',', '.')), + // `from` may carry the number it came with — `from: "5 miles"` — or be + // missing entirely because the unit went into `value`. + from: explicitFrom.replace(NUMBER, '').trim() || value.replace(NUMBER, '').trim(), + to: explicitTo, + } +} + +/** + * Converts, or says plainly why it will not. + * + * A refusal is fed back into the model's context as the whole result of a spent + * round, so each one names what went wrong in terms the next attempt can act + * on. Guessing would be worse: grams to cups depends on what is in the cup, and + * an answer that looks exact and is not is the failure this tool exists to stop. + */ +export function convertQuantity(args: Record): string { + const request = readConversionRequest(args) + if (!Number.isFinite(request.value)) throw new Error('value must be a number, for example 32') + + const from = resolve(request.from) + const to = resolve(request.to) + if (!from) throw new Error(`Unknown unit "${request.from || '(none)'}"`) + if (!to) throw new Error(`Unknown unit "${request.to || '(none)'}"`) + + if (from.unit.dimension !== to.unit.dimension) { + throw new Error( + `${from.unit.label} is ${DIMENSION_NAMES[from.unit.dimension]} and ${to.unit.label} is ${DIMENSION_NAMES[to.unit.dimension]}, so one cannot be converted into the other.`, + ) + } + + const converted = + from.unit.dimension === 'temperature' + ? fromCelsius(toCelsius(request.value, from.id), to.id) + : (request.value * from.unit.factor) / to.unit.factor + + return `${formatQuantity(request.value)} ${from.unit.label} = ${formatQuantity(converted)} ${to.unit.label}` +} From d74e9788ccb1370adc80ed8dfc6aa8ab2c131154 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 17:58:38 +0000 Subject: [PATCH 3/9] Send a figure, a date and an attribution to the web, where nothing was checking them Co-authored-by: Sebastian --- README.md | 18 +++++++- src/eval/scenarios.ts | 20 +++++++++ src/skills/library.test.ts | 61 ++++++++++++++++++++++++++- src/skills/research-question/SKILL.md | 17 ++++++++ src/skills/weather/SKILL.md | 7 ++- 5 files changed, 118 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ee46fc3..d5da6bd 100644 --- a/README.md +++ b/README.md @@ -414,10 +414,26 @@ Nothing in the prompt caused this and no skill was firing; the model simply pref So `lookup-term` triggers on the shape of the question — `what is `, `was ist `, or a subject whose token mixes letters with digits — and teaches by example that the query is the user's word, unaltered, and that what the term _means_ is something the results decide rather than the model. Its second exemplar runs search then `read_page`, which is the "check what actually came back" half of the same lesson. -Two shapes are excluded by hand, because both look exactly like a name to a pattern that counts tokens. A token of bare digits is a measurement rather than a project, so _what is 32 fahrenheit in celsius_ is left alone: searching for it verbatim answers nothing, and it was the one prompt this skill reliably stole. And _what is that?_ is a pronoun, not a product. +Two shapes are excluded by hand, because both look exactly like a name to a pattern that counts tokens. A token of bare digits is a measurement rather than a project, so _what is 32 fahrenheit in celsius_ is left alone: searching for it verbatim answers nothing, and it now belongs to [`convert-units`](#why-convert-units-exists) instead. And _what is that?_ is a pronoun, not a product. The eval scores this directly: scenarios may assert on the arguments a tool was called with, not just its name, and the harness reports that as a separate **Right args** column. +### What `research-question` had been missing + +Routing ordinary questions through `route` turned up a gap larger than any collision: three quarters of the questions people actually ask reached no skill at all. Some of those belong to nobody — _write me a rhyme_, _what is the capital of France_ — but one group did not, and it is the group where this model is least safe on its own: + +| Asked | Answered by | +| ---------------------------------- | ---------------------------- | +| _How many people live in Tokyo?_ | whatever the weights recall | +| _Wie alt ist Angela Merkel?_ | a number, stated confidently | +| _When was the Eiffel Tower built?_ | a year, stated confidently | +| _Who wrote Dune?_ / _Wer hat …?_ | an attribution, uncited | +| _Wie hoch ist der Eiffelturm?_ | a figure with no source | + +A figure, a date and an attribution are exactly what a 0.8B model produces plausibly and unverifiably, so all three shapes are now `research-question` triggers, in both languages — the skill already knew how to search, open the best result and cite it, and had simply never been asked. Each shape excludes the version of itself that is about the user or the assistant, because _how old are you_ and _when is my flight_ are not on the web. + +Growing this skill also came at another one's expense, which is the mechanism worth remembering: `chance of rain` and `regenwahrscheinlichkeit` were weather **keywords**, and `how high is` / `wie hoch ist` is a research **trigger**, so the forecast question moved to a search engine the moment the trigger was written. Both are triggers on the weather skill now. A keyword cannot defend a question against a trigger, wherever that trigger lives and however low its priority. + Skills are bundled at build time rather than fetched, so no part of routing depends on a request that could fail. ### Narrowing what a skill claims diff --git a/src/eval/scenarios.ts b/src/eval/scenarios.ts index 1114549..383cf92 100644 --- a/src/eval/scenarios.ts +++ b/src/eval/scenarios.ts @@ -385,6 +385,26 @@ export const SCENARIOS: Scenario[] = [ accept: (answer) => /\d/.test(answer), online: true, }, + { + id: 'web-unsupported-figure', + category: 'web', + // A population is a number the model will happily state from memory, and the + // shape reached no skill at all until `how many` became a trigger. + prompt: 'How many people live in Tokyo?', + expectTool: 'web_search', + // Any figure with the millions in it. Tokyo is 14 million in the city and 37 + // in the metropolitan area, and both are correct answers to this question. + accept: matches(/\b(1[34]|3[6-8])(\.\d+)?\s*(million|m\b)|\b(1[34]|3[6-8])[.,]\d{3}[.,]\d{3}/i), + online: true, + }, + { + id: 'web-date-built', + category: 'web', + prompt: 'When was the Eiffel Tower built?', + expectTool: 'web_search', + accept: matches(/188[7-9]|1889/), + online: true, + }, { id: 'web-news-not-clock', category: 'web', diff --git a/src/skills/library.test.ts b/src/skills/library.test.ts index 8e160fc..539b2b5 100644 --- a/src/skills/library.test.ts +++ b/src/skills/library.test.ts @@ -170,10 +170,13 @@ describe('weather', () => { ['Temperatur draussen', 'trigger'], ['how warm will it be', 'search'], ['how cold does it get', 'search'], - ['chance of rain later', 'search'], ['Wie warm wird es morgen in Rom?', 'search'], ['wie kalt wird die Nacht', 'search'], - ['Wie hoch ist die Regenwahrscheinlichkeit?', 'search'], + // Both were keywords until research-question grew a `how high is` shape. + // A keyword cannot defend a question against a trigger, wherever the + // trigger lives and however low its skill's priority is. + ['chance of rain later', 'trigger'], + ['Wie hoch ist die Regenwahrscheinlichkeit?', 'trigger'], ])('takes %j by %s', (message, how) => { expect(routed(message)).toBe('weather') expect(reason(message)).toBe(how) @@ -315,6 +318,60 @@ describe('research-question', () => { expect(routed(message)).toBe('research-question') expect(reason(message)).toBe(how) }) + + it.each([ + // A figure, a date or an attribution. All of these reached no skill at all, + // which is the state in which a 0.8B model answers from memory and states a + // number nobody can check. + ['How many people live in Tokyo?', 'trigger'], + ['How many countries are in Africa?', 'trigger'], + ['How old is Angela Merkel?', 'trigger'], + ['How tall is the Burj Khalifa?', 'trigger'], + ['How fast is a cheetah?', 'trigger'], + ['When was the Eiffel Tower built?', 'trigger'], + ['When will the next election be?', 'trigger'], + ['Who wrote Dune?', 'trigger'], + ['Who invented the telephone?', 'trigger'], + ['Who founded Stripe?', 'trigger'], + ['Wie alt ist Angela Merkel?', 'trigger'], + ['Wie viele Einwohner hat Deutschland?', 'trigger'], + ['Wie hoch ist der Eiffelturm?', 'trigger'], + ['Wie schwer ist ein Blauwal?', 'trigger'], + ['Wann wurde die Mauer gebaut?', 'trigger'], + ['Wer hat das Telefon erfunden?', 'trigger'], + ['Wer hat Dune geschrieben?', 'trigger'], + ['Wie viele Einwohner hat Wien im Vergleich zu Graz?', 'trigger'], + ])('takes the unsupported fact %j by %s', (message, how) => { + expect(routed(message)).toBe('research-question') + expect(reason(message)).toBe(how) + }) + + it.each([ + // The same shapes asked about the user or the assistant. Neither is on the + // web, and searching for either is the wrong kind of answer. + 'How old are you?', + 'Wie alt bist du?', + 'When is my flight?', + 'Wann ist mein Termin?', + 'How many do I have?', + 'Wie viele Notizen habe ich?', + ])('leaves %j alone', (message) => { + expect(routed(message)).not.toBe('research-question') + }) + + it.each([ + // Every one of these is a question the new shapes above pass through: the + // skill that owns it has a higher priority, or matches a longer shape. + ['How many ounces is 200 grams?', 'convert-units'], + ['Wie viele Zentimeter sind 3 Zoll?', 'convert-units'], + ['How much is 18 percent of 2450?', 'arithmetic'], + ['Wie viel Uhr ist es?', 'current-date'], + ['How high is the chance of rain tomorrow?', 'weather'], + ['Wie hoch ist die Regenwahrscheinlichkeit?', 'weather'], + ['Wie warm ist es in München?', 'weather'], + ])('does not take %j from %s', (message, expected) => { + expect(routed(message)).toBe(expected) + }) }) describe('memory', () => { diff --git a/src/skills/research-question/SKILL.md b/src/skills/research-question/SKILL.md index 6226e53..a865679 100644 --- a/src/skills/research-question/SKILL.md +++ b/src/skills/research-question/SKILL.md @@ -11,11 +11,14 @@ jarvis: - find out - search the web - who won + - population of + - how many people - suche im netz - schau nach - finde heraus - wer hat gewonnen - aktuelle nachrichten + - wie viele einwohner triggers: - '\b(latest|current|recent|news|today.s)\b' # `who is` without an exclusion also takes *Who is that?*, which is a @@ -26,6 +29,20 @@ jarvis: # being sent to a search engine. An interrogative has to be there too. - '\b(what|which|who|when|where|why|how)\b[^.?!]{0,60}\b(20[2-9]\d)\b' - '\b(look up|search for|find out|google)\b' + # A figure, a date or an attribution: the three shapes where a 0.8B model + # produces something plausible and unchecked, and the three that reached no + # skill at all. Each excludes the version of itself that is about the user or + # about the assistant, neither of which is on the web. + - '\bhow many\b(?!\s+(?:tools?|memor))(?!.{0,20}\b(?:do|did|have) (?:i|we)\b)' + - '\bhow (old|tall|high|long|deep|heavy|big|far|fast) (is|was|are|were)\b(?!\s+(?:you|i|we|my)\b)' + - '\bwhen (was|were|did|is|will)\b(?!\s+(?:my|i|we|you)\b)' + - '\bwho (wrote|invented|founded|discovered|created|directed|built|painted|composed|owns)\b' + # German. `wie viele` in front of a unit belongs to `convert-units`, which + # outranks this skill, so what is left here is a count of something real. + - '\bwie viele?\b(?!\s+(?:uhr|erinnerung))(?!.{0,20}\bhabe ich\b)' + - '\bwie (alt|hoch|gro(ß|ss)|lang|schwer|tief|weit|schnell) (ist|war|sind|waren)\b(?!\s+(?:du|sie|ich|wir|mein)\b)' + - '\bwann (wurde|war|ist|sind|hat|kommt)\b(?!\s+(?:mein|ich|wir|du)\b)' + - '\bwer hat\s+(?:das\s+|die\s+|den\s+)?\S+\s*(geschrieben|erfunden|gegründet|gebaut|entdeckt|komponiert|gemalt)\b' # A price is looked up, never worked out. `arithmetic` used to take these on # the strength of the words `how much is` alone. - '\bhow much (does|do|did) .{0,40} cost\b' diff --git a/src/skills/weather/SKILL.md b/src/skills/weather/SKILL.md index 254ba52..6f4ca23 100644 --- a/src/skills/weather/SKILL.md +++ b/src/skills/weather/SKILL.md @@ -10,10 +10,8 @@ jarvis: keywords: - how warm - how cold - - chance of rain - wie warm - wie kalt - - regenwahrscheinlichkeit triggers: # `\b` would fire on weather.com and on /forecast, handing a linked page to a # weather lookup when `summarize-url` should have read it. @@ -22,6 +20,11 @@ jarvis: - '\bis it (raining|snowing|sunny|windy|humid)\b' - '\btemperature (in|outside|here|today|tonight|right now)\b' - '\b(will it|going to) (rain|snow)\b' + # Both were keywords, which is a weaker claim than it looks: a trigger + # anywhere in the catalogue is matched before any keyword, and + # research-question's `how high is` / `wie hoch ist` now covers this shape. + - '\bchance of (rain|snow|showers)\b' + - '\bregenwahrscheinlichkeit\b' # German, because the app answers in the language it is asked in and the # compound words a German question uses would not survive a word boundary: # Wettervorhersage and Unwetter both have to match. From f1dc9678db8c1c886e86573f3f8fa5e5be4cd90f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 18:06:46 +0000 Subject: [PATCH 4/9] Read the part of a page the question is about, not the part the site put first Co-authored-by: Sebastian --- README.md | 4 + src/agent/loop.test.ts | 25 ++- src/agent/loop.ts | 8 +- src/tools/builtins.ts | 24 +-- src/tools/extract.test.ts | 171 +++++++++++++++++++ src/tools/extract.ts | 336 ++++++++++++++++++++++++++++++++++++++ src/tools/types.ts | 16 +- 7 files changed, 564 insertions(+), 20 deletions(-) create mode 100644 src/tools/extract.test.ts create mode 100644 src/tools/extract.ts diff --git a/README.md b/README.md index d5da6bd..2131077 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,10 @@ A browser may only read a response whose origin opts in with CORS headers, which **`read_page`** goes through `r.jina.ai`, which reflects the requesting origin, needs no account, and returns extracted markdown rather than raw HTML. Anonymous use is capped at 20 requests per minute per IP; a Jina key raises that and is optional. +**What of that page reaches the model is chosen, not truncated.** The cap is 8,000 characters, roughly 2,000 tokens and by far the largest thing in this model's context — and it used to be the first 8,000, which is a bet that the answer was printed at the top. It usually is not. What is at the top is the navigation, the cookie notice and a list of related links, and all of it competed with the answer for a 0.8B model's attention. So `src/tools/extract.ts` strips the furniture and then, only if the page is still too long, keeps the passages the question is about: the page is split into blocks with each heading glued to the paragraph beneath it, every block is scored by the question's terms weighted by how rare each is **in that page**, and whatever budget is left over goes on the paragraphs either side of the best match and on the page's own opening. Skipped passages are marked with `[…]`, so the model can tell it is not reading a whole page. + +The question comes from the agent loop, not from the model: `runAgent` passes the user's turn to every tool as context, because asking the model to fill in a fourth argument would spend tool-calling accuracy on something already known. With no question to go on — or one that shares no word with the page — it falls back to the head, which is where this started. A short page is returned whole either way, so most reads are unaffected except for losing their navigation. + **`web_search`** has a provider choice under **Tools → Web access**: | Provider | Key | Covers | diff --git a/src/agent/loop.test.ts b/src/agent/loop.test.ts index 38e6f3b..c0489ab 100644 --- a/src/agent/loop.test.ts +++ b/src/agent/loop.test.ts @@ -62,7 +62,7 @@ describe('runAgent', () => { const result = await runAgent(fakeClient([toolCall, 'done2 + 2 = 4']), turns, [calculator], hooks) - expect(execute).toHaveBeenCalledWith({ expression: '2+2' }) + expect(execute).toHaveBeenCalledWith({ expression: '2+2' }, { question: 'hi' }) expect(result.content).toBe('2 + 2 = 4') // The first round had no visible content, but its reasoning must not become an answer. expect(hooks.onRoundEnd).toHaveBeenNthCalledWith( @@ -71,6 +71,29 @@ describe('runAgent', () => { ) }) + it('hands a tool the question, from the turn the user actually wrote', async () => { + // `read_page` decides which part of a long page is worth the context from + // this. It has to be the user's turn and not the last `user` turn in the + // conversation, which by the wind-down round is a prompt the loop wrote. + const execute = vi.fn(async () => 'page') + const reader = defineTool('read_page', 'read', { type: 'object', properties: {} }, execute) + const asked = [ + { role: 'user' as const, content: 'What is 1inch?' }, + { role: 'assistant' as const, content: 'A DEX aggregator.' }, + { role: 'user' as const, content: 'What does it charge?' }, + ] + + await runAgent( + fakeClient([toolCall('read_page', 'url', 'https://example.com'), 'doneNothing.']), + asked, + [reader], + callbacks(), + { review: false }, + ) + + expect(execute).toHaveBeenCalledWith({ url: 'https://example.com' }, { question: 'What does it charge?' }) + }) + it('stops calling tools after the round budget rather than looping', async () => { const search = defineTool( 'web_search', diff --git a/src/agent/loop.ts b/src/agent/loop.ts index bea2da0..8ef97e9 100644 --- a/src/agent/loop.ts +++ b/src/agent/loop.ts @@ -130,6 +130,12 @@ export async function runAgent( const strategy = options.strategy ?? DEFAULT_STRATEGY const checking = options.review ?? true const evidence = collectEvidence(turns) + /** + * Read once, before the loop adds turns of its own: the wind-down prompt and + * a correction request are both `user` turns, and neither is what the user + * asked. Tools that can use the question get this one. + */ + const question = turns.findLast((turn) => turn.role === 'user')?.content ?? '' let last: AgentResult = { content: '', @@ -233,7 +239,7 @@ export async function runAgent( } try { - const result = await tool.execute(call.arguments) + const result = await tool.execute(call.arguments, { question }) executed.set(fingerprint, result) callbacks.onToolEnd(id, { result, durationMs: performance.now() - startedAt }) conversation.push({ role: 'tool', content: result }) diff --git a/src/tools/builtins.ts b/src/tools/builtins.ts index 47a31bd..e871263 100644 --- a/src/tools/builtins.ts +++ b/src/tools/builtins.ts @@ -1,4 +1,5 @@ import { evaluateExpression } from './calculator' +import { pageExtract } from './extract' import { memory } from './memory' import { defineTool, type Tool } from './types' import { convertQuantity } from './units' @@ -42,21 +43,6 @@ function createWebSearch(config: WebAccessConfig): Tool { ) } -/** - * Tool results are fed straight back into the context, and long ones are not a - * neutral cost: measured across several models, function-calling accuracy falls - * by between 7% and 91% as tool responses grow (arXiv:2505.10570). An unbounded - * page would be by far the largest thing in a 0.8B model's context. - * - * Roughly 2,000 tokens, which leaves room for the prompt and the answer. - */ -const MAX_PAGE_CHARS = 8000 - -function truncate(text: string): string { - if (text.length <= MAX_PAGE_CHARS) return text - return `${text.slice(0, MAX_PAGE_CHARS)}\n\n[Truncated: the page continues beyond this point.]` -} - function createReadPage(config: WebAccessConfig): Tool { return defineTool( 'read_page', @@ -68,11 +54,15 @@ function createReadPage(config: WebAccessConfig): Tool { }, required: ['url'], }, - async (args) => { + // The question comes from the agent loop rather than from the model: what + // part of a long page is worth the context is decided in `extract.ts`, and + // asking the model for it would spend a tool argument on something already + // known. Without one, a page too long to fit falls back to its head. + async (args, context) => { const url = String(args.url ?? '').trim() if (!url) throw new Error('url must not be empty') const page = await readPage(url, config) - return `# ${page.title}\nSource: ${page.url}\n\n${truncate(page.text)}` + return `# ${page.title}\nSource: ${page.url}\n\n${pageExtract(page.text, context?.question ?? '')}` }, ) } diff --git a/src/tools/extract.test.ts b/src/tools/extract.test.ts new file mode 100644 index 0000000..729f9bd --- /dev/null +++ b/src/tools/extract.test.ts @@ -0,0 +1,171 @@ +import { describe, expect, it } from 'vitest' +import { MAX_PAGE_CHARS, pageExtract, queryTerms, rankPassages, splitPassages } from './extract' + +/** A page in the shape the reader returns one: chrome, then an article. */ +const PAGE = `![Image 1: logo](https://example.com/logo.png) + +[Home](https://example.com/) +[News](https://example.com/news) +[About](https://example.com/about) + +Cookies + +We use cookies. Accept all + +# The Eiffel Tower + +## Construction + +Construction began in January 1887 and the tower was completed in March 1889 for the World's Fair. + +## Visiting + +The lift runs from 09:30 until 23:00 in summer, and tickets cost 29.40 euro for the top floor. + +## Related articles + +[The Statue of Liberty](https://example.com/liberty) + +Advertisement` + +describe('splitPassages', () => { + it('drops the site and keeps the page', () => { + const passages = splitPassages(PAGE) + + expect(passages.join('\n')).not.toMatch(/logo|Home|Accept all|Advertisement|Statue of Liberty/) + expect(passages.some((passage) => passage.includes('1887'))).toBe(true) + }) + + it('keeps a heading with the paragraph under it', () => { + // A heading alone matches a question and answers nothing; attached, it is the + // context for the paragraph that does answer it. + const construction = splitPassages(PAGE).find((passage) => passage.includes('1887')) + + expect(construction).toContain('## Construction') + }) + + it('keeps a sentence about cookies in an article about cookies', () => { + // The chrome list only applies to short lines, or a page about cookie law + // would lose its subject. + const passages = splitPassages( + 'The 2002 directive made cookies a matter of consent across the European Union, which changed how sites are built.', + ) + + expect(passages).toHaveLength(1) + }) + + it('drops a navigation strip repeated on every screen', () => { + expect(splitPassages('Menu\n\nMenu\n\nMenu')).toEqual([]) + }) +}) + +describe('rankPassages', () => { + const passages = splitPassages(PAGE) + + it('puts the passage about the question first', () => { + const [best] = rankPassages(passages, 'When was the Eiffel Tower built?') + + expect(best?.text).toContain('1887') + }) + + it('ranks a different question onto a different passage', () => { + const [best] = rankPassages(passages, 'What do tickets cost?') + + expect(best?.text).toContain('29.40') + }) + + it('scores nothing when the page and the question share no word', () => { + expect(rankPassages(passages, 'Wie ist das Wetter in Hamburg?')).toEqual([]) + }) + + it('ignores a term the whole page uses', () => { + // Inverse document frequency, measured over this page: a word in every + // passage cannot distinguish between them, whatever the question asks. + const repeated = ['Alpha carries the word tower.', 'Beta carries the word tower and 1887.'] + const [best] = rankPassages(repeated, 'tower 1887') + + expect(best?.text).toContain('Beta') + }) +}) + +describe('pageExtract', () => { + it('returns a short page whole, minus its furniture', () => { + const extracted = pageExtract(PAGE, 'When was it built?') + + expect(extracted).toContain('1887') + expect(extracted).toContain('29.40') + expect(extracted).not.toContain('Accept all') + expect(extracted).not.toContain('Shortened') + }) + + /** + * The case this exists for: the answer is past the cap, so a head-first + * truncation cannot see it however large the cap is. + */ + describe('a page longer than the budget', () => { + const filler = Array.from( + { length: 200 }, + (_, index) => `Paragraph ${index} of general history, describing the period at some length.`, + ).join('\n\n') + const buried = `${filler}\n\nThe tower was completed in March 1889 and cost 7.8 million francs.\n\n${filler}` + + it('finds the answer the old truncation cut off', () => { + const extracted = pageExtract(buried, 'What did the tower cost?') + + expect(buried.slice(0, MAX_PAGE_CHARS)).not.toContain('7.8 million') + expect(extracted).toContain('7.8 million') + expect(extracted.length).toBeLessThanOrEqual(MAX_PAGE_CHARS) + }) + + it('says that it left something out', () => { + expect(pageExtract(buried, 'What did the tower cost?')).toContain('[Shortened') + }) + + it('marks where passages were skipped', () => { + // Two answers, far apart, with several thousand characters of history in + // between: the gap is what tells the model it is not reading a whole page. + const twice = `${filler}\n\nThe tower cost 7.8 million francs.\n\n${filler}\n\nThe tower cost 40 million to repaint.\n\n${filler}` + const extracted = pageExtract(twice, 'What did the tower cost?') + + expect(extracted).toContain('7.8 million') + expect(extracted).toContain('40 million') + expect(extracted).toContain('[…]') + }) + + it('spends what is left over on the paragraphs either side of the answer', () => { + // A long page, one matching paragraph, and budget to spare: the sentence + // that continues the answer is worth more than the unused characters. + const short = `Paragraph before.\n\nThe tower cost 7.8 million francs.\n\nThat figure is in 1889 money.\n\n${filler}` + const extracted = pageExtract(short, 'What did the tower cost?', 400) + + expect(extracted).toContain('7.8 million') + expect(extracted).toContain('1889 money') + }) + + it('falls back to the head when there is no question', () => { + const extracted = pageExtract(buried) + + expect(extracted).toContain('Paragraph 0') + expect(extracted).toContain('[Truncated') + expect(extracted.length).toBeLessThanOrEqual(MAX_PAGE_CHARS + 60) + }) + + it('falls back to the head when the question matches nothing on the page', () => { + const extracted = pageExtract(buried, 'Wetter Hamburg Regenwahrscheinlichkeit') + + expect(extracted).toContain('Paragraph 0') + expect(extracted).toContain('[Truncated') + }) + }) +}) + +describe('queryTerms', () => { + it('keeps what a question is about and drops what every question has', () => { + expect(queryTerms('What did the tower cost?')).toEqual(['tower', 'cost']) + expect(queryTerms('Wie hoch ist der Eiffelturm?')).toEqual(['hoch', 'eiffelturm']) + }) + + it('keeps short acronyms and numbers, which the other tokenizers drop', () => { + expect(queryTerms('Who leads the UN in 2026?')).toEqual(['leads', 'un', '2026']) + }) +}) diff --git a/src/tools/extract.ts b/src/tools/extract.ts new file mode 100644 index 0000000..d8a2b16 --- /dev/null +++ b/src/tools/extract.ts @@ -0,0 +1,336 @@ +/** + * What of a web page the model actually gets to read. + * + * A page arrives from the reader as markdown of any length and left at 8,000 + * characters — roughly 2,000 tokens, and by far the largest thing in this + * model's context. The cap is not the problem; taking it from the top is. The + * sentence that answers the question sits wherever the page put it, and a + * head-first truncation is a bet that it was put first, while everything before + * it — the navigation, the cookie notice, the list of related links — is spent + * on a 0.8B model's attention regardless. + * + * So the page is stripped of its furniture and then, if it is still too long, + * reduced to the passages about the question. Long tool results are not a + * neutral cost: across several models, function-calling accuracy falls by + * between 7% and 91% as tool responses grow (arXiv:2505.10570), so a shorter + * and more relevant result is the same change twice over. + * + * Selection is lexical, for the same reasons `src/skills/retrieve.ts` gives: a + * dense retriever would mean shipping a second model into an app whose premise + * is one download. The seam is `rankPassages` if that ever changes. + */ + +/** + * Roughly 2,000 tokens, which leaves room for the prompt and the answer. + * + * This is a cap on what reaches the model, not on what was fetched. `readPage` + * returns the whole page and this decides how much of it is worth spending. + */ +export const MAX_PAGE_CHARS = 8000 + +/** Stands where passages were dropped, so the model can see that they were. */ +const GAP = '[…]' + +/** + * Words too common to tell one passage from another. + * + * A third stop list in this repository, and deliberately not one of the other + * two: `skills/retrieve.ts` scores curated keywords and `memory/text.ts` + * compares short sentences, and both drop tokens under three characters. A + * question about `UN`, `AI` or `GB` needs those, and the rarity of a term here + * is measured against the page itself rather than against a fixed corpus. + */ +const STOPWORDS = new Set([ + 'the', + 'and', + 'that', + 'this', + 'with', + 'for', + 'from', + 'are', + 'was', + 'were', + 'has', + 'have', + 'had', + 'its', + 'their', + 'what', + 'which', + 'who', + 'whom', + 'how', + 'why', + 'when', + 'where', + 'does', + 'did', + 'can', + 'could', + 'would', + 'should', + 'about', + 'into', + 'you', + 'your', + 'i', + 'me', + 'my', + 'we', + 'us', + 'is', + 'in', + 'on', + 'of', + 'at', + 'to', + 'it', + 'a', + 'an', + 'be', + 'as', + 'or', + 'tell', + 'give', + 'show', + 'please', + // German, because the app answers in the language it is asked in and the + // question is what these are matched against. + 'der', + 'die', + 'das', + 'den', + 'dem', + 'des', + 'ein', + 'eine', + 'einen', + 'und', + 'oder', + 'ist', + 'sind', + 'war', + 'waren', + 'hat', + 'haben', + 'wie', + 'was', + 'wer', + 'wo', + 'wann', + 'warum', + 'mir', + 'mich', + 'ich', + 'du', + 'sie', + 'von', + 'für', + 'auf', + 'mit', + 'im', + 'zu', + 'bitte', + 'sag', + 'nach', +]) + +const WORD = /[\p{L}\p{N}]+/gu + +/** The words of a question worth matching a page against. */ +export function queryTerms(question: string): string[] { + const words = [...question.toLowerCase().matchAll(WORD)].map((match) => match[0]) + return [...new Set(words.filter((word) => !STOPWORDS.has(word)))] +} + +/** + * Lines that are part of the site rather than part of the page. + * + * Every one of these was measured against reader output rather than imagined: + * `r.jina.ai` renders a navigation bar as a run of one-link lines, an image as + * its alt text in brackets, and a cookie banner as two short sentences. None of + * them can answer anything, and together they are routinely the first thousand + * characters of the page — which is precisely what a head-first cap keeps. + */ +const IMAGE_LINE = /^!\[[^\]]*\]\([^)]*\)$/ +const LINK_ONLY_LINE = /^[-*+]?\s*!?\[[^\]]*\]\([^)]*\)[.,;:]?$/ +const CHROME_WORDS = + /^(skip to (main )?content|menu|navigation|search|sign in|log in|register|subscribe|newsletter|share (this|on)?|follow us|advertisement|sponsored|cookies?|accept( all)?( cookies)?|privacy (policy|settings)|manage (cookies|preferences)|related( articles| stories)?|back to top|print this page|inhalt|hauptmenü|anmelden|abonnieren|teilen|werbung|datenschutz|cookie-einstellungen|zum inhalt springen)\b/i + +/** A consent notice, which is a sentence rather than a label. */ +const CONSENT = + /\b(we use cookies|uses cookies|accept (all )?cookies|cookie (settings|consent|banner)|wir verwenden cookies|cookies akzeptieren)\b/i + +function isFurniture(line: string): boolean { + const text = line.trim() + if (!text) return false + if (IMAGE_LINE.test(text) || LINK_ONLY_LINE.test(text)) return true + // Length is what keeps this honest in both cases: an article about cookie + // legislation has to keep its sentences about cookies, and it does not write + // them in twenty characters. + const bare = text.replace(/^[-*+#\s]+/, '') + if (text.length <= 60 && CHROME_WORDS.test(bare)) return true + return text.length <= 120 && CONSENT.test(bare) +} + +/** + * Splits a page into the blocks a passage can be. + * + * A heading is glued to the block beneath it: on its own it is four words that + * match a question and answer nothing, and attached it is the context for the + * paragraph that does. Consecutive duplicate lines are dropped, which is what a + * repeated navigation strip looks like once the links are gone. + */ +export function splitPassages(markdown: string): string[] { + const passages: string[] = [] + let block: string[] = [] + let heading: string | null = null + let previous = '' + + const flush = (): void => { + if (block.length === 0) { + return + } + const body = block.join('\n') + passages.push(heading ? `${heading}\n${body}` : body) + heading = null + block = [] + } + + for (const raw of markdown.split('\n')) { + const line = raw.trimEnd() + const text = line.trim() + + if (!text) { + flush() + continue + } + if (isFurniture(line)) continue + if (text === previous) continue + previous = text + + if (/^#{1,6}\s/.test(text)) { + flush() + heading = text + continue + } + block.push(line) + } + flush() + + return passages.filter((passage) => passage.trim().length > 0) +} + +/** + * How telling a term is, measured across this page's own passages. + * + * The BM25 idf, as in `retrieve.ts`, and computed here over the page rather than + * over a fixed corpus: the site's own name appears in every passage of it and + * should decide nothing, while the word the question turns on usually appears in + * two or three. + */ +function inverseFrequency(documentFrequency: number, total: number): number { + return Math.log((total - documentFrequency + 0.5) / (documentFrequency + 0.5) + 1) +} + +export interface RankedPassage { + index: number + text: string + score: number +} + +/** + * Scores every passage against the question, best first. + * + * Presence rather than frequency, which is what keeps a long passage from + * winning on repetition alone: a paragraph mentioning the term once and a + * paragraph mentioning it nine times are equally about it, and the second is + * usually a list. + */ +export function rankPassages(passages: string[], question: string): RankedPassage[] { + const terms = queryTerms(question) + if (terms.length === 0 || passages.length === 0) return [] + + const words = passages.map((passage) => new Set(queryTerms(passage))) + const weights = new Map( + terms.map((term) => [ + term, + inverseFrequency(words.filter((set) => set.has(term)).length, passages.length), + ]), + ) + + const scored = passages.map((text, index) => { + let score = 0 + for (const term of terms) { + if (words[index]?.has(term)) score += weights.get(term) ?? 0 + } + return { index, text, score } + }) + + return scored.filter((passage) => passage.score > 0).sort((a, b) => b.score - a.score || a.index - b.index) +} + +/** Joins passages in the order the page had them, marking what was skipped. */ +function assemble(chosen: RankedPassage[]): string { + const ordered = [...chosen].sort((a, b) => a.index - b.index) + const parts: string[] = [] + + ordered.forEach((passage, position) => { + const previous = ordered[position - 1] + if (previous && passage.index > previous.index + 1) parts.push(GAP) + parts.push(passage.text) + }) + + return parts.join('\n\n') +} + +/** + * The part of a page worth putting in front of the model. + * + * Three outcomes, in order of how often they happen. A page that fits after its + * furniture is stripped is returned whole. A page that does not, asked about + * something, is reduced to the passages about it. A page that does not, with + * nothing to go on — no question, or a question the page shares no word with — + * falls back to the head, which is where this started. + */ +export function pageExtract(markdown: string, question = '', limit = MAX_PAGE_CHARS): string { + const passages = splitPassages(markdown) + const whole = passages.join('\n\n') + if (whole.length <= limit) return whole + + const head = (): string => + `${whole.slice(0, limit).trimEnd()}\n\n[Truncated: the page continues beyond this point.]` + + const ranked = rankPassages(passages, question) + if (ranked.length === 0) return head() + + const chosen = new Map() + // The gap markers and the separators are part of what has to fit, so the + // budget is spent as the result is built rather than checked once at the end. + let used = 0 + const take = (index: number): void => { + const text = passages[index] + if (text === undefined || chosen.has(index)) return + const cost = text.length + GAP.length + 4 + if (used + cost > limit) return + chosen.set(index, { index, text, score: 0 }) + used += cost + } + + // No minimum length: a passage that matched has earned its place, and + // `Founded: 1889` is short precisely because it is the answer. + for (const passage of ranked) take(passage.index) + if (chosen.size === 0) return head() + + // Whatever is left over goes on context rather than on nothing: the paragraphs + // either side of the best match, where a sentence answering the question + // usually continues, and then the page's own opening, which is what says what + // the page is. Both are skipped silently when the budget is already spent. + const best = ranked[0]?.index ?? 0 + take(best + 1) + take(best - 1) + take(0) + + return `${assemble([...chosen.values()])}\n\n[Shortened: only the parts of this page about the question are shown.]` +} diff --git a/src/tools/types.ts b/src/tools/types.ts index 55ef83f..3b7bdcb 100644 --- a/src/tools/types.ts +++ b/src/tools/types.ts @@ -1,9 +1,23 @@ import type { ToolSchema } from '@/types' +/** + * What the turn is about, passed to every tool and asked of none. + * + * `read_page` needs the question to decide which part of a long page is worth + * the model's context, and the alternative was a fourth argument on the schema + * for the model to fill in. That would spend tool-calling accuracy — which falls + * as arguments multiply — on something the agent loop already knows. A tool that + * has no use for it simply ignores it. + */ +export interface ToolContext { + /** The user's message for this turn, verbatim. */ + question: string +} + export interface Tool { schema: ToolSchema /** Returns a compact string; it is fed straight back into the model's context. */ - execute: (args: Record) => Promise + execute: (args: Record, context?: ToolContext) => Promise } export function defineTool( From 0e5be8598149d861687b9e1a2b9fb2ca50cad7d6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 18:11:19 +0000 Subject: [PATCH 5/9] Check the answer is in the language the question was asked in Co-authored-by: Sebastian --- README.md | 5 +- src/agent/review.test.ts | 55 +++++++++++- src/agent/review.ts | 156 ++++++++++++++++++++++++++++++++- src/components/MessageItem.tsx | 1 + 4 files changed, 214 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2131077..5a05b7f 100644 --- a/README.md +++ b/README.md @@ -509,13 +509,14 @@ Every reply says which skill answered it and how it was found: `weather skill · A skill fires on some requests. This runs on all of them. -Between the model settling on an answer and that answer reaching the screen, `src/agent/review.ts` reads it back against what the turn actually produced — the results the tools returned, and the URLs already in the conversation. Three things are checked: +Between the model settling on an answer and that answer reaching the screen, `src/agent/review.ts` reads it back against what the turn actually produced — the results the tools returned, the URLs already in the conversation, and the question itself. Four things are checked: | Check | Fires when | | ----------------- | -------------------------------------------------------------------- | | `wrong-number` | `calculator` or `convert` returned a value the answer states nowhere | | `invented-source` | The answer cites a URL that no tool returned and nobody supplied | | `missing-source` | Tools returned sources and the answer cites none | +| `wrong-language` | The question is in one of the two languages and the answer the other | A failed check costs one further generation. The model is handed its own draft and told what to change — _The calculator returned 6748 \* 9 = 60732. Give that number, exactly as it came back._ — and the correction replaces the draft only if it leaves fewer problems behind. Otherwise the draft stands. That gate is the important half: the correction comes from the same 0.8B model, so a mechanism that could not tell an improvement from a regression would be a coin toss on every reply. @@ -523,6 +524,8 @@ A failed check costs one further generation. The model is handed its own draft a They are also deliberately shy. A clarifying question is asked for no citation; a long decimal quoted to fewer places counts as the calculator's number; citing the site when a page on it was read is close enough; a URL from an earlier reply is not an invention. Every check would rather miss a mistake than invent one, because a check that fires on a correct answer costs a generation and teaches you to ignore the whole mechanism. +**The language check is the one where that shyness does most of the work.** The system prompt asks for the language the user wrote in, and this model drifts back to English mid-conversation — the most visible way it is wrong, and the one thing that can be settled without asking it anything, since the evidence is the user's own message. Recognising a language needs enough of it to be sure, though, so both sides have to be recognisable before the check fires: it counts function words and German letters, needs three markers, and stands down otherwise. _330 Meter._ has no language. Neither does _Paris_. A German answer quoting an English sentence is still German. A question in a third language leaves both scores low and the check says nothing at all, which is the honest outcome for a check that knows two languages. The correction is written in the language it is asking for. + The interface says what happened rather than quietly rewriting the reply. While the corrected answer streams in it is labelled with what is being fixed, and afterwards it carries `corrected` — claimed only for an answer that now passes every check — or `flagged`, naming what is still wrong with the text on screen. An answer half fixed and advertised as corrected would be worse than no check at all. ## When a turn runs out of tool rounds diff --git a/src/agent/review.test.ts b/src/agent/review.test.ts index e683fac..9ceccb4 100644 --- a/src/agent/review.test.ts +++ b/src/agent/review.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest' import { collectEvidence, correctionPrompt, reviewAnswer, type ReviewEvidence } from './review' function evidence(overrides: Partial = {}): ReviewEvidence { - return { toolResults: [], knownUrls: [], ...overrides } + return { toolResults: [], knownUrls: [], question: '', ...overrides } } const searchResult = { @@ -83,6 +83,58 @@ describe('reviewAnswer', () => { }) }) + /** + * The system prompt asks for the language the user wrote in, and this model + * drifts back to English mid-conversation. The evidence is the question, so + * this is settled without asking the model anything. + */ + describe('language', () => { + const germanQuestion = evidence({ question: 'Wie hoch ist der Eiffelturm und wann wurde er gebaut?' }) + + it('catches an English answer to a German question', () => { + expect( + checks( + 'The Eiffel Tower is 330 metres tall, and it was completed in 1889 for the World Fair.', + germanQuestion, + ), + ).toEqual(['wrong-language']) + }) + + it('says which language to use, in that language', () => { + const [finding] = reviewAnswer('The tower is 330 metres tall and was built in 1889.', germanQuestion) + + expect(finding?.instruction).toBe('Die Frage war auf Deutsch. Antworte auf Deutsch.') + }) + + it('accepts a German answer to a German question', () => { + expect(checks('Der Eiffelturm ist 330 Meter hoch und wurde 1889 gebaut.', germanQuestion)).toEqual([]) + }) + + it('catches it the other way round too', () => { + const asked = evidence({ question: 'How tall is the Eiffel Tower and when was it built?' }) + + expect(checks('Der Eiffelturm ist 330 Meter hoch und wurde 1889 gebaut.', asked)).toEqual([ + 'wrong-language', + ]) + }) + + it.each([ + // Too short to have a language, which is most correct answers. + ['330 Meter.', 'Wie hoch ist der Eiffelturm?'], + ['Paris', 'Was ist die Hauptstadt von Frankreich?'], + ['Ja, 8,05 km.', 'Wie viel sind 5 Meilen in Kilometer?'], + // A German answer quoting an English source is still a German answer. + [ + 'Der Turm ist 330 Meter hoch. Die Quelle schreibt "the tower was completed in 1889".', + 'Wie hoch ist der Eiffelturm?', + ], + // Neither language is one this can read, so it says nothing at all. + ['La tour Eiffel mesure 330 mètres de haut.', 'Quelle est la hauteur de la tour Eiffel ?'], + ])('leaves %j alone', (answer, question) => { + expect(checks(answer, evidence({ question }))).toEqual([]) + }) + }) + describe('sources', () => { const searched = evidence({ toolResults: [searchResult] }) @@ -187,6 +239,7 @@ describe('collectEvidence', () => { // worked example is exactly the kind of thing the model should not cite. expect(collected).toEqual({ toolResults: [], + question: 'Summarise https://example.com/pricing', knownUrls: ['https://example.com/pricing', 'https://example.com/old'], }) }) diff --git a/src/agent/review.ts b/src/agent/review.ts index a9738fb..a0be316 100644 --- a/src/agent/review.ts +++ b/src/agent/review.ts @@ -15,7 +15,7 @@ import type { ChatTurn } from '@/llm/protocol' * costs a second generation and teaches the user to ignore the whole mechanism, * so every one of them prefers to miss a mistake over inventing one. */ -export type ReviewCheck = 'wrong-number' | 'invented-source' | 'missing-source' +export type ReviewCheck = 'wrong-number' | 'invented-source' | 'missing-source' | 'wrong-language' export interface ReviewFinding { check: ReviewCheck @@ -39,6 +39,8 @@ export interface ReviewEvidence { * answer above must not read as an invention. */ knownUrls: string[] + /** What the user asked, which is what the answer's language is checked against. */ + question: string } const URL_IN_TEXT = /https?:\/\/[^\s<>"'`)\]}]+/g @@ -55,6 +57,10 @@ export function collectEvidence(turns: ChatTurn[]): ReviewEvidence { knownUrls: turns .filter((turn) => turn.role === 'user' || turn.role === 'assistant') .flatMap((turn) => findUrls(turn.content)), + // The last real user turn, which is the one being answered. Exemplar turns a + // skill contributed sit ahead of it and are written in the skill's language, + // not the user's. + question: turns.findLast((turn) => turn.role === 'user')?.content ?? '', } } @@ -134,6 +140,146 @@ function statesNumber(answer: string, value: number): boolean { return renderings(value).some((rendering) => digits.includes(rendering)) } +/** + * Which language a turn is written in, or nothing. + * + * The system prompt asks for the language the user wrote in, and a 0.8B model + * drifts back to English mid-conversation — the most visible way this app is + * wrong, and the one thing a check can settle without a model: the evidence is + * the user's own message. + * + * Two languages, because those are the two the app is used in and the two whose + * function words are worth listing. A question in a third language leaves both + * scores low, `detectLanguage` answers `null`, and the check stands down rather + * than guessing — the same shyness every check here has. + */ +const GERMAN_LETTERS = /[äöüß]/i + +const GERMAN_WORDS = new Set([ + 'der', + 'die', + 'das', + 'den', + 'dem', + 'ein', + 'eine', + 'und', + 'oder', + 'aber', + 'nicht', + 'ist', + 'sind', + 'war', + 'waren', + 'hat', + 'haben', + 'wird', + 'werden', + 'kann', + 'ich', + 'du', + 'wir', + 'ihr', + 'mit', + 'auf', + 'aus', + 'bei', + 'nach', + 'von', + 'vor', + 'zum', + 'zur', + 'wie', + 'was', + 'wer', + 'wann', + 'warum', + 'auch', + 'noch', + 'schon', + 'sehr', + 'etwa', + 'ungefähr', + 'heute', + 'morgen', + 'liegt', + 'beträgt', + 'derzeit', + 'quelle', +]) + +const ENGLISH_WORDS = new Set([ + 'the', + 'a', + 'an', + 'and', + 'or', + 'but', + 'not', + 'is', + 'are', + 'was', + 'were', + 'has', + 'have', + 'will', + 'would', + 'can', + 'i', + 'you', + 'we', + 'they', + 'with', + 'from', + 'about', + 'into', + 'of', + 'to', + 'for', + 'how', + 'what', + 'who', + 'when', + 'why', + 'also', + 'still', + 'very', + 'roughly', + 'about', + 'today', + 'tomorrow', + 'currently', + 'source', +]) + +/** Enough markers to be a language rather than a coincidence. */ +const MIN_LANGUAGE_MARKERS = 3 + +export type Language = 'de' | 'en' + +export function detectLanguage(text: string): Language | null { + const words = text.toLowerCase().match(/[\p{L}]+/gu) ?? [] + let german = 0 + let english = 0 + for (const word of words) { + if (GERMAN_WORDS.has(word)) german += 1 + if (ENGLISH_WORDS.has(word)) english += 1 + } + // An umlaut or an eszett is worth a function word on its own, which is what + // lets a short German sentence be recognised at all. + if (GERMAN_LETTERS.test(text)) german += 1 + + if (german >= MIN_LANGUAGE_MARKERS && german > english) return 'de' + if (english >= MIN_LANGUAGE_MARKERS && english > german) return 'en' + return null +} + +/** Handed to the model in the language it should have used. */ +const LANGUAGE_INSTRUCTION: Record = { + de: 'Die Frage war auf Deutsch. Antworte auf Deutsch.', + en: 'The question was in English. Answer in English.', +} + /** A question back to the user, and a plain "I could not find it", cite nothing. */ const NOTHING_TO_CITE = /\?\s*$|\b(could ?n[o']t find|no results|don'?t know|do not know|unable to find)\b/i @@ -192,6 +338,14 @@ export function reviewAnswer(answer: string, evidence: ReviewEvidence): ReviewFi }) } + // Both have to be recognisable for this to fire, so a one-line answer, a + // number, and anything in a language this cannot read are all left alone. + const asked = detectLanguage(evidence.question) + const answered = detectLanguage(draft) + if (asked && answered && asked !== answered) { + findings.push({ check: 'wrong-language', instruction: LANGUAGE_INSTRUCTION[asked] }) + } + return findings } diff --git a/src/components/MessageItem.tsx b/src/components/MessageItem.tsx index 2c03079..71f565f 100644 --- a/src/components/MessageItem.tsx +++ b/src/components/MessageItem.tsx @@ -36,6 +36,7 @@ const REVIEW_REASON: Record = { 'wrong-number': 'a number the calculator disagreed with', 'invented-source': 'a source no tool returned', 'missing-source': 'a missing source', + 'wrong-language': 'the wrong language', } /** Reads as a phrase, so the same wording works before and after the fix. */ From 248e1504976fb053d6e37aa711f760c33154dbc8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 18:12:10 +0000 Subject: [PATCH 6/9] Give the budget fixture the question the evidence now carries Co-authored-by: Sebastian --- src/agent/budget.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/budget.test.ts b/src/agent/budget.test.ts index 6f791cf..624b901 100644 --- a/src/agent/budget.test.ts +++ b/src/agent/budget.test.ts @@ -5,7 +5,7 @@ import { budgetFallback, callFingerprint, repeatedCallNote, windDownNote } from import type { ReviewEvidence } from './review' function evidence(results: { tool: string; result: string }[] = []): ReviewEvidence { - return { toolResults: results, knownUrls: [] } + return { toolResults: results, knownUrls: [], question: 'Who leads Fictional Airways?' } } const searchResult = { From 6c42b016ae001e1478d5ddd51ac3f6e446da9e79 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 18:14:59 +0000 Subject: [PATCH 7/9] Cap the transcript the model is re-prefiled with on every round Co-authored-by: Sebastian --- README.md | 2 ++ src/store/chat.test.ts | 55 +++++++++++++++++++++++++++++++++++++++++- src/store/chat.ts | 36 ++++++++++++++++++++++++--- 3 files changed, 88 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5a05b7f..0ec5ea5 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,8 @@ Inference lives in a Web Worker. A 0.8B forward pass on the main thread would fr The model emits reasoning inside `` blocks and tool requests as JSON inside `` blocks. `src/agent/parse.ts` separates the three streams; `src/agent/loop.ts` executes the requested tools and feeds their output back until the model answers without asking for another tool. That is capped at four rounds, and [reaching the cap still produces an answer](#when-a-turn-runs-out-of-tool-rounds) rather than an apology. The answer it settles on is then [checked against what the tools returned](#checking-the-answer-before-it-is-shown) before anyone sees it. +**The prompt has a budget, and the window is not what sets it.** Qwen3.5 carries 262,144 positions, so no conversation here comes close to overflowing it — yet every part of the prompt is capped anyway: skill guidance at 600 characters, one skill's whole contribution at 3,000, recall at 400, a page read at 8,000, and the transcript at 8,000 in `toHistory`. What a long prompt costs is the two things this model is short of. It is re-prefilled on every round of every turn, twice over for the capped reasoning strategies. And length dilutes attention: a longer system prompt has been measured here dropping tool use to 1 in 6, and there is no reason to think the same tokens further down are free. Twenty-odd exchanges fit, and what falls off the front is gone — which is what [memory](#memory) is for. + ## The first screen The model has to be started by hand even once it is installed, so `ModelGate` stands in front of the chat on every visit. That makes it the page anyone arriving here reads first, and `src/components/Landing.tsx` treats it as one: what this is, what it can do, how a question becomes a checked answer, what actually leaves the browser, and what this browser needs before any of it works. The functional half — the GPU check, the storage figures, the download and its progress — is `src/components/InstallPanel.tsx`, and it sits in the hero as the call to action. It appears exactly once; a second copy would report two of every state. diff --git a/src/store/chat.test.ts b/src/store/chat.test.ts index 90a3203..9e3e36a 100644 --- a/src/store/chat.test.ts +++ b/src/store/chat.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it } from 'vitest' -import { residentSkill, rewindToLastPrompt, useChatStore } from './chat' +import { MAX_HISTORY_CHARS, residentSkill, rewindToLastPrompt, toHistory, useChatStore } from './chat' import type { AppliedSkill, Message } from '@/types' function message(role: Message['role'], content: string): Message { @@ -36,6 +36,59 @@ describe('rewindToLastPrompt', () => { }) }) +describe('toHistory', () => { + it('sends the conversation as the model should read it', () => { + const history = toHistory([ + message('user', 'first'), + message('assistant', 'first answer'), + message('user', 'second'), + ]) + + expect(history).toEqual([ + { role: 'user', content: 'first' }, + { role: 'assistant', content: 'first answer' }, + { role: 'user', content: 'second' }, + ]) + }) + + it('drops a reply that never produced anything', () => { + // An empty assistant turn takes the next turn's context with it, which is + // the second reason the loop promotes reasoning into an empty answer. + const history = toHistory([ + message('user', 'hi'), + message('assistant', ''), + message('user', 'still there?'), + ]) + + expect(history.map((turn) => turn.content)).toEqual(['hi', 'still there?']) + }) + + it('drops the oldest turns once the budget is spent, whole turns at a time', () => { + const history = toHistory( + [ + message('user', 'ancient question'), + message('assistant', 'ancient answer'), + message('user', 'recent question'), + message('assistant', 'recent answer'), + message('user', 'now'), + ], + // Room for the last three of these and not the first two. + 40, + ) + + expect(history.map((turn) => turn.content)).toEqual(['recent question', 'recent answer', 'now']) + }) + + it('keeps the turn being answered however long it is', () => { + // Truncating the question is worse than sending nothing else: what was asked + // is the one thing the turn cannot do without. + const asked = 'x'.repeat(MAX_HISTORY_CHARS * 2) + const history = toHistory([message('user', 'earlier'), message('user', asked)]) + + expect(history).toEqual([{ role: 'user', content: asked }]) + }) +}) + describe('residentSkill', () => { it('has nothing to carry into a fresh conversation', () => { expect(residentSkill([])).toBeNull() diff --git a/src/store/chat.ts b/src/store/chat.ts index c606e6b..ba8807b 100644 --- a/src/store/chat.ts +++ b/src/store/chat.ts @@ -140,14 +140,42 @@ export function getClient(): LlmClient { */ const catalog = loadCatalog() -/** Only user-visible turns go back to the model; reasoning is intentionally dropped. */ -function toHistory(messages: Message[]): ChatTurn[] { +/** + * How much of the conversation goes back to the model. + * + * Roughly 2,000 tokens, and not because the window is full: Qwen3.5 carries + * 262,144 positions, so nothing here is ever close to overflowing it. What a + * long transcript costs is the two things this model is actually short of. + * Every round of every turn re-prefills the whole thing, and the capped + * strategies prefill it twice; and length itself dilutes what the model attends + * to — this app has already measured a longer *system prompt* dropping tool use + * to 1 in 6, and there is no reason the same tokens further down are free. + * + * Generous on purpose. Twenty-odd exchanges fit, which is longer than the + * conversations this is used for, so the cap is a floor under the worst case + * rather than something a normal chat runs into. What is dropped is dropped for + * good, which is what [memory](#memory) is for. + */ +export const MAX_HISTORY_CHARS = 8000 + +/** + * Only user-visible turns go back to the model; reasoning is intentionally + * dropped. Oldest turns go first when the budget is spent, whole turns at a + * time — half a question is worse than no question — and the turn being + * answered is kept whatever it costs, because it is what was asked. + */ +export function toHistory(messages: Message[], budget = MAX_HISTORY_CHARS): ChatTurn[] { const turns: ChatTurn[] = [] - for (const message of messages) { + let used = 0 + + for (const message of messages.toReversed()) { if (message.role === 'assistant' && !message.content) continue + used += message.content.length + if (used > budget && turns.length > 0) break turns.push({ role: message.role, content: message.content }) } - return turns + + return turns.reverse() } function createMessage(role: Message['role'], content = ''): Message { From 40716f27bd90fd6b4e08d3145b085a2a56a8d90b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 18:18:57 +0000 Subject: [PATCH 8/9] Cap an MCP result the way a page read has always been, and say so where it was already claimed Co-authored-by: Sebastian --- .cursor/rules/tools.mdc | 8 +++-- .cursor/skills/add-agent-tool/SKILL.md | 16 +++++++--- README.md | 4 +-- src/tools/mcp.test.ts | 41 ++++++++++++++++++++++++-- src/tools/mcp.ts | 12 ++++++++ 5 files changed, 71 insertions(+), 10 deletions(-) diff --git a/.cursor/rules/tools.mdc b/.cursor/rules/tools.mdc index 7c7fd22..fc95c49 100644 --- a/.cursor/rules/tools.mdc +++ b/.cursor/rules/tools.mdc @@ -6,9 +6,13 @@ alwaysApply: false Read `.cursor/skills/add-agent-tool/SKILL.md` before changing a tool. - Tool arguments reach `execute` as untyped, trimmed strings, because the model emits XML. Coerce - and validate every one. + and validate every one, and read a phrase the model wrote into one argument rather than refusing + it: a refusal costs the whole round. +- `execute` gets a second argument, `ToolContext`, carrying the user's question for this turn. It is + optional by design — use it, as `read_page` does to pick the passages worth reading, but never + require it. - The return value is fed straight back into a 0.8B model's context. Keep it compact and cap - anything that could be unbounded. + anything that could be unbounded at `MAX_PAGE_CHARS` from `src/tools/extract.ts`. - There is no proxy. A tool that needs the network must call an endpoint that sends CORS headers, in `src/tools/web.ts`, or it cannot run in a browser at all. - A new tool needs a scenario in `src/eval/scenarios.ts`, or nothing measures whether the model diff --git a/.cursor/skills/add-agent-tool/SKILL.md b/.cursor/skills/add-agent-tool/SKILL.md index c0db65a..5f29126 100644 --- a/.cursor/skills/add-agent-tool/SKILL.md +++ b/.cursor/skills/add-agent-tool/SKILL.md @@ -14,12 +14,20 @@ handed to `runAgent`, which matches the model's request against `schema.function - **Arguments are untyped strings.** Qwen3.5 emits tool calls as XML, and `src/agent/parse.ts` passes every `` value through as a trimmed string. `execute` receives `Record`. Coerce and validate everything yourself — `Number(args.limit)` may be - `NaN`, `args.query` may be absent. + `NaN`, `args.query` may be absent. Be generous about what you accept: a rejected call spends the + whole round, and `normalizeExpression` in `calculator.ts`, `placeCandidates` in `weather.ts` and + `readConversionRequest` in `units.ts` all exist because the model wrote something readable that + was refused. - **The return value goes straight into the model's context, so cap it.** Return a compact string, not JSON, not megabytes. Long tool results are not a neutral cost: function-calling accuracy falls - by 7% to 91% as responses grow, which is why `read_page` truncates at `MAX_PAGE_CHARS` (8,000 - characters, roughly 2,000 tokens). Anything that can return an unbounded body needs the same - treatment. + by 7% to 91% as responses grow, which is why `read_page` and MCP results are both held to + `MAX_PAGE_CHARS` (8,000 characters, roughly 2,000 tokens, exported from `src/tools/extract.ts`). + Anything that can return an unbounded body needs the same treatment. +- **`execute` also receives a `ToolContext`**, holding the user's question for this turn. It comes + from the agent loop rather than from the model, so a tool can use what the turn is about without + spending an argument on it — `read_page` picks the passages of a long page that answer the + question that way. Ignore it if you have no use for it; never require it, since the eval harness + and a wind-down round can both call a tool with nothing useful in it. - **Throw on failure.** `runAgent` catches it and feeds `Tool "" failed: ` back to the model, which usually recovers. Never return an error string that reads like a result. - **The description is prompt text.** The chat template renders it into every prompt. Write one or diff --git a/README.md b/README.md index 0ec5ea5..d9a45c8 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A chat agent that runs its language model **inside your browser**. Qwen3.5-0.8B It does need a connection to answer, which is a deliberate limit rather than a missing feature — see [why it waits for a connection](#why-it-waits-for-a-connection). -The agent can search, read pages, calculate exactly, remember things you tell it, and call any MCP server you connect. Because a 0.8B model needs the help, common requests are routed through [skills](#skills) that show it a worked example rather than telling it what to do. +The agent can search, read pages, calculate exactly, convert units, remember things you tell it, and call any MCP server you connect. Because a 0.8B model needs the help, common requests are routed through [skills](#skills) that show it a worked example rather than telling it what to do. There is no backend. Not "a backend you can skip" — the project ships no server code at all, and `pnpm build` produces a directory of static files that needs nothing but a web server to host it. That is why the deployed site above has the full tool set rather than a reduced one. @@ -291,7 +291,7 @@ Open **Tools** in the header to connect a Model Context Protocol server over Str The server must send permissive CORS headers, because requests originate from the page with no proxy in between. A server that fails to connect is skipped rather than blocking startup, and the error is shown next to its entry. -Tool results are truncated at 8,000 characters before they reach the model. Long results are not a neutral cost: across several models, function-calling accuracy drops by between 7% and 91% as tool responses grow ([arXiv:2505.10570](https://arxiv.org/html/2505.10570)), and an unbounded web page would be by far the largest thing in this model's context. +An MCP result is truncated at 8,000 characters before it reaches the model, the same cap [a page read](#how-the-network-tools-work-without-a-server) is held to. Long results are not a neutral cost: across several models, function-calling accuracy drops by between 7% and 91% as tool responses grow ([arXiv:2505.10570](https://arxiv.org/html/2505.10570)), and a server on the other end of this can return anything of any size — so one verbose tool could otherwise spend the context the answer needed. ## Memory diff --git a/src/tools/mcp.test.ts b/src/tools/mcp.test.ts index 4c1110a..9a8a2b7 100644 --- a/src/tools/mcp.test.ts +++ b/src/tools/mcp.test.ts @@ -1,5 +1,6 @@ -import { describe, expect, it } from 'vitest' -import { isHttpUrl } from './mcp' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { MAX_PAGE_CHARS } from './extract' +import { isHttpUrl, McpClient } from './mcp' describe('isHttpUrl', () => { it('accepts the addresses fetch can reach', () => { @@ -22,3 +23,39 @@ describe('isHttpUrl', () => { expect(isHttpUrl('')).toBe(false) }) }) + +describe('callTool', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) + + function serverReturning(text: string): void { + vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ + ok: true, + headers: new Headers({ 'content-type': 'application/json' }), + text: async () => JSON.stringify({ jsonrpc: '2.0', id: 1, result: { content: [{ text }] } }), + })), + ) + } + + const client = new McpClient({ id: 'server', url: 'https://host/mcp' }) + + it('passes a compact result straight through', async () => { + serverReturning('42') + + await expect(client.callTool('answer', {})).resolves.toBe('42') + }) + + it('caps a result that would take the whole prompt', async () => { + // Nothing capped this before, so one verbose server could spend the context + // the answer needed — the same cost `read_page` has always been capped for. + serverReturning('x'.repeat(MAX_PAGE_CHARS * 3)) + + const result = await client.callTool('firehose', {}) + + expect(result.length).toBeLessThan(MAX_PAGE_CHARS + 100) + expect(result).toContain('[Truncated') + }) +}) diff --git a/src/tools/mcp.ts b/src/tools/mcp.ts index d6c43df..1030190 100644 --- a/src/tools/mcp.ts +++ b/src/tools/mcp.ts @@ -1,3 +1,4 @@ +import { MAX_PAGE_CHARS } from './extract' import { defineTool, type Tool } from './types' import type { ToolSchema } from '@/types' @@ -125,6 +126,14 @@ export class McpClient { ) } + /** + * The result is capped the same way a page read is. + * + * A server on the other end of this can return anything of any size, and it + * goes straight into a 0.8B model's context: function-calling accuracy falls + * by between 7% and 91% as tool responses grow (arXiv:2505.10570). Nothing + * capped it before, so one verbose server could take the whole prompt. + */ async callTool(name: string, args: Record): Promise { const result = await this.rpc<{ content?: McpContentBlock[]; isError?: boolean }>('tools/call', { name, @@ -135,6 +144,9 @@ export class McpClient { .join('\n') .trim() if (result?.isError) throw new Error(text || 'MCP tool reported an error') + if (text.length > MAX_PAGE_CHARS) { + return `${text.slice(0, MAX_PAGE_CHARS)}\n\n[Truncated: the result continues beyond this point.]` + } return text || '(empty result)' } } From 06b9c3476600906013fda27a26568e6268e989ff Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 18:24:21 +0000 Subject: [PATCH 9/9] Route a German question that carries a year, which every election question does Co-authored-by: Sebastian --- src/skills/library.test.ts | 2 ++ src/skills/research-question/SKILL.md | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/skills/library.test.ts b/src/skills/library.test.ts index 539b2b5..d03aa01 100644 --- a/src/skills/library.test.ts +++ b/src/skills/library.test.ts @@ -340,6 +340,8 @@ describe('research-question', () => { ['Wann wurde die Mauer gebaut?', 'trigger'], ['Wer hat das Telefon erfunden?', 'trigger'], ['Wer hat Dune geschrieben?', 'trigger'], + ['Wer hat die Bundestagswahl 2025 gewonnen?', 'trigger'], + ['Was war 2024 das meistverkaufte Auto?', 'trigger'], ['Wie viele Einwohner hat Wien im Vergleich zu Graz?', 'trigger'], ])('takes the unsupported fact %j by %s', (message, how) => { expect(routed(message)).toBe('research-question') diff --git a/src/skills/research-question/SKILL.md b/src/skills/research-question/SKILL.md index a865679..1d1a196 100644 --- a/src/skills/research-question/SKILL.md +++ b/src/skills/research-question/SKILL.md @@ -42,7 +42,15 @@ jarvis: - '\bwie viele?\b(?!\s+(?:uhr|erinnerung))(?!.{0,20}\bhabe ich\b)' - '\bwie (alt|hoch|gro(ß|ss)|lang|schwer|tief|weit|schnell) (ist|war|sind|waren)\b(?!\s+(?:du|sie|ich|wir|mein)\b)' - '\bwann (wurde|war|ist|sind|hat|kommt)\b(?!\s+(?:mein|ich|wir|du)\b)' - - '\bwer hat\s+(?:das\s+|die\s+|den\s+)?\S+\s*(geschrieben|erfunden|gegründet|gebaut|entdeckt|komponiert|gemalt)\b' + - '\bwer hat\s+(?:das\s+|die\s+|den\s+)?[^.?!]{0,40}(geschrieben|erfunden|gegründet|gebaut|entdeckt|komponiert|gemalt|gewonnen)\b' + # A year with a German question around it, which the English shape above + # covers and no German one did: *Wer hat die Bundestagswahl 2025 gewonnen?* + # reached nothing, and it is the shape every election question has. `was` is + # left out of the first pattern and given a verb in the second, because it is + # also English past tense — *I was born in 2024* is the exact sentence a bare + # year was removed from these triggers for. + - '\b(wer|wann|wo|warum|wieso|welche[rsn]?)\b[^.?!]{0,60}\b(20[2-9]\d)\b' + - '\bwas (war|waren|passierte|geschah|gab es)\b[^.?!]{0,40}\b(20[2-9]\d)\b' # A price is looked up, never worked out. `arithmetic` used to take these on # the strength of the words `how much is` alone. - '\bhow much (does|do|did) .{0,40} cost\b'