diff --git a/.cursor/skills/agent-memory/SKILL.md b/.cursor/skills/agent-memory/SKILL.md index d9d2d06..754829b 100644 --- a/.cursor/skills/agent-memory/SKILL.md +++ b/.cursor/skills/agent-memory/SKILL.md @@ -56,7 +56,8 @@ what counts as a repeat, where a false positive would silently drop what the use conversation; here that call runs on the user's own GPU and would double the cost of a turn. - **The conversation topic is derived, never stored.** `topic.ts` reads the last weather place or the last research subject off the transcript and injects one line, the same way recall is - injected. Recency wins, so an older Frankfurt does not leak onto _und der von Frankreich?_. It + injected. Recency wins, so an older Frankfurt does not leak onto _und der von Frankreich?_. A + correction that only names a place — _nein in Russland_ — keeps the last office the same way. It fires on a follow-up, a weather turn with no place, or an anaphor (_der Bürgermeister_, _there_). It stays off a fresh named subject — _Wer ist Elon Musk?_ after Frankfurt weather must not receive Frankfurt, and _und Elon Musk?_ after a chancellor turn must not receive Bundeskanzler. diff --git a/.cursor/skills/debug-model-output/SKILL.md b/.cursor/skills/debug-model-output/SKILL.md index 2884052..033b553 100644 --- a/.cursor/skills/debug-model-output/SKILL.md +++ b/.cursor/skills/debug-model-output/SKILL.md @@ -9,21 +9,23 @@ license: MIT Most of these symptoms have been hit before and have a known cause. Check this table before changing sampling parameters or the system prompt. -| Symptom | Cause | Where | -| -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | -| Reply renders empty | The model reasoned to a conclusion inside `` and stopped without restating it | `promoteReasoningIfEmpty` in `src/agent/loop.ts` | -| Model keeps going and writes the user's next turn | Published `generation_config.json` lists only `<\|endoftext\|>`, so generation runs past `<\|im_end\|>` | `endOfTurnTokens()` in `src/llm/worker.ts` | -| Model answers instantly, invents arithmetic, skips tools | Reasoning is off, so the template writes an empty `` pair into the prompt | `tokenizer_encode_kwargs: { enable_thinking: true }` in `src/llm/worker.ts` | -| `` or `<\|im_end\|>` visible in the chat | Parser is not stripping it | `parseModelOutput` / `SPECIAL_TOKEN` in `src/agent/parse.ts` | -| Tool call is emitted but never executed | Qwen3.5 emits XML, not JSON | `parseXmlToolCall` in `src/agent/parse.ts` | -| Reasoning shows up as the answer on a normal turn | Promotion is only correct when the turn is over | `parsed.toolCalls.length === 0` guard in `loop.ts` | -| Tool receives `"5"` where a number was expected | XML parameters carry no types | Coerce in the tool — see `add-agent-tool` | -| Model asks for a tool that does not exist | Usually long reasoning, not a broken tool list | `Unknown tool` branch in `loop.ts`; measured as `hallucination` in `src/eval` | -| One turn generates twice and the reply is retyped | The answer failed a check and a correction was requested | `reviewAnswer` in `src/agent/review.ts`, wired in `loop.ts` | -| A correction was generated and then thrown away | It left as many problems as the draft, so the draft stands | `settle` in `src/agent/loop.ts` | -| A turn searched four times and then answered anyway | The tool budget ran out, so the tools were withheld and the model asked to conclude | `FINAL_ANSWER_PROMPT` in `src/agent/budget.ts`, `windDown` in `loop.ts` | -| The model asked for a search and the card shows a note | It had already run that exact call this turn, so the earlier result came back instead | `callFingerprint` / `repeatedCallNote` in `src/agent/budget.ts` | -| A reply is a sentence and a list of links | Even the wind-down round came back empty, so the sources it found were handed over | `budgetFallback` in `src/agent/budget.ts` | +| Symptom | Cause | Where | +| -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| Reply renders empty | The model reasoned to a conclusion inside `` and stopped without restating it | `promoteReasoningIfEmpty` in `src/agent/loop.ts` | +| Model keeps going and writes the user's next turn | Published `generation_config.json` lists only `<\|endoftext\|>`, so generation runs past `<\|im_end\|>` | `endOfTurnTokens()` in `src/llm/worker.ts` | +| Model answers instantly, invents arithmetic, skips tools | Reasoning is off, so the template writes an empty `` pair into the prompt | `tokenizer_encode_kwargs: { enable_thinking: true }` in `src/llm/worker.ts` | +| `` or `<\|im_end\|>` visible in the chat | Parser is not stripping it | `parseModelOutput` / `SPECIAL_TOKEN` in `src/agent/parse.ts` | +| Tool call is emitted but never executed | Qwen3.5 emits XML, not JSON | `parseXmlToolCall` in `src/agent/parse.ts` | +| Reasoning shows up as the answer on a normal turn | Promotion is only correct when the turn is over | `parsed.toolCalls.length === 0` guard in `loop.ts` | +| Tool receives `"5"` where a number was expected | XML parameters carry no types | Coerce in the tool — see `add-agent-tool` | +| Model asks for a tool that does not exist | Usually long reasoning, not a broken tool list | `Unknown tool` branch in `loop.ts`; measured as `hallucination` in `src/eval` | +| One turn generates twice and the reply is retyped | The answer failed a check and a correction was requested | `reviewAnswer` in `src/agent/review.ts`, wired in `loop.ts` | +| A correction was generated and then thrown away | It left as many problems as the draft, so the draft stands | `settle` in `src/agent/loop.ts` | +| A turn searched four times and then answered anyway | The tool budget ran out, so the tools were withheld and the model asked to conclude | `FINAL_ANSWER_PROMPT` in `src/agent/budget.ts`, `windDown` in `loop.ts` | +| The model asked for a search and the card shows a note | It had already run that exact call this turn, so the earlier result came back instead | `callFingerprint` / `repeatedCallNote` in `src/agent/budget.ts` | +| A reply is a sentence and a list of links | Even the wind-down round came back empty, so the sources it found were handed over | `budgetFallback` in `src/agent/budget.ts` | +| German office holder for another country's president | `extractAnswer` treated any Amtsträger line as the answer; a nationality in the question now has to appear in that source | `fitsAskedPlace` in `src/tools/research.ts` | +| Follow-up says "I believe" and skips research | Correction fragments kept the skill and not the tool; the skill now has a _nein in Russland_ exemplar | `src/skills/research-question/SKILL.md`, `conversationTopic` | ## Facts that are easy to get wrong diff --git a/README.md b/README.md index ed1fb06..5027124 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,7 @@ Search and `read_page` share the reader's budget of 20 requests a minute per IP, **The search itself now carries the facts a 0.8B model would otherwise spend a round guessing at.** Every `web_search` result is stamped with today's local date, so "current" and "today's news" have a date without calling `current_time`. A German question searches German Wikipedia and, on DuckDuckGo, prefers German results (`kl=de-de`); English _who was Ada Lovelace_ is not mistaken for German because bare `was` is also English. German Wikipedia is smaller, so an empty result there falls through to English rather than telling the model the subject does not exist. -The `research-question` skill does not offer `web_search`. It offers `research`, which searches the live web and Wikipedia together, picks three **different sites** (Wikipedia first when it appeared, because MediaWiki is free and the lead paragraph usually answers; `investor.nvidia.com` and `nvidianews.nvidia.com` count as one), ranks the rest by whether their snippet already bears on the question, and reads them in parallel. A page that will not open, or that comes back as a login wall or a firewall interstitial, is replaced from the remaining hits rather than quoted; only when nothing else is left does the search snippet stand in. Inflected forms of a word still match — _Bundeskanzlers_ answers _Bundeskanzler_ — so a German page is not silent on a German question. The search query is narrowed the way the weather tool narrows a place: _What is the capital of France?_ is searched as `capital of France`, which ranks Paris rather than a page that uses the sentence as an example of a question; _why_ and _how_ keep the shell, because _sky blue_ ranks a colour swatch and _why is the sky blue_ ranks the physics. Wikipedia's own ranking still puts a list and _capital punishment_ ahead of Paris; the article whose short description _is_ the query takes the free first slot, and a Wikimedia list is pushed behind it. A one-line Wikipedia claim that names the incumbent (_Amtsträger ist seit dem 6. Mai 2025 Friedrich Merz_) is kept even when the lead is a definition of the office. When several passages name the same person or place, or one sentence states it as the predicate of the question (_Paris is the capital_, _Amtsträger ist … Friedrich Merz_), the digest opens with that as `Answer: Friedrich Merz.` so a 0.8B model can copy the line rather than extract it from a list. A population or a price gets the same treatment — `Answer: 13.96 million people.` — because those are the figures a 0.8B model otherwise invents to three significant digits. A name that is the subject of the question (_Who is Elon Musk?_) is not an answer and is left off; neither is the city on a population question, nor a tie between two different names or two figures that are not the same reading. If nothing readable comes back the tool throws rather than telling the model the subject does not exist. +The `research-question` skill does not offer `web_search`. It offers `research`, which searches the live web and Wikipedia together, picks three **different sites** (Wikipedia first when it appeared, because MediaWiki is free and the lead paragraph usually answers; `investor.nvidia.com` and `nvidianews.nvidia.com` count as one), ranks the rest by whether their snippet already bears on the question, and reads them in parallel. A page that will not open, or that comes back as a login wall or a firewall interstitial, is replaced from the remaining hits rather than quoted; only when nothing else is left does the search snippet stand in. Inflected forms of a word still match — _Bundeskanzlers_ answers _Bundeskanzler_ — so a German page is not silent on a German question. The search query is narrowed the way the weather tool narrows a place: _What is the capital of France?_ is searched as `capital of France`, which ranks Paris rather than a page that uses the sentence as an example of a question; _why_ and _how_ keep the shell, because _sky blue_ ranks a colour swatch and _why is the sky blue_ ranks the physics. Wikipedia's own ranking still puts a list and _capital punishment_ ahead of Paris; the article whose short description _is_ the query takes the free first slot, and a Wikimedia list is pushed behind it. A one-line Wikipedia claim that names the incumbent (_Amtsträger ist seit dem 6. Mai 2025 Friedrich Merz_) is kept even when the lead is a definition of the office. When several passages name the same person or place, or one sentence states it as the predicate of the question (_Paris is the capital_, _Amtsträger ist … Friedrich Merz_), the digest opens with that as `Answer: Friedrich Merz.` so a 0.8B model can copy the line rather than extract it from a list. A nationality or country in the question has to appear in the same source: _Amtsträger ist Friedrich Merz_ answers _Bundeskanzler_, not _russische Präsident_. A population or a price gets the same treatment — `Answer: 13.96 million people.` — because those are the figures a 0.8B model otherwise invents to three significant digits. A name that is the subject of the question (_Who is Elon Musk?_) is not an answer and is left off; neither is the city on a population question, nor a tie between two different names or two figures that are not the same reading. If nothing readable comes back the tool throws rather than telling the model the subject does not exist. A factual question that no other skill claimed — _What is the capital of France?_, _Warum ist der Himmel blau?_ — still reaches this skill, as a fourth routing stage rather than a broader trigger. Greetings do not: _wie geht's dir_ and _how are you_ stay with the model. `lookup-term` still searches and optionally reads one page — a name does not need three sources. @@ -342,7 +342,7 @@ Memories live in **IndexedDB**, in your browser, next to the model weights. Noth The [CoALA taxonomy](https://arxiv.org/abs/2309.02427) splits an agent's memory four ways: working memory is the live context window, and the three durable kinds are semantic, episodic and procedural. Only the durable three are stored — working memory is the transcript, which the tab already holds — and they are named in words the model can actually pick between. -The transcript is not enough on its own. A 0.8B model will answer tomorrow's weather without the city the last turn just resolved, so the last established place is pinned into the system prompt as one short line — _This conversation is about Frankfurt._ — the same way recall is. After a research turn the same line holds the office or subject — _This conversation is about Bundeskanzler._ — so _und der von Frankreich?_ is not searched as a fragment. Recency decides which one: an older Frankfurt does not leak onto a chancellor follow-up. It is derived from the chat, never written to IndexedDB, and it stays off a fresh question that names its own subject. Switching memory off does not drop it: that toggle is for stored facts, not for this conversation. +The transcript is not enough on its own. A 0.8B model will answer tomorrow's weather without the city the last turn just resolved, so the last established place is pinned into the system prompt as one short line — _This conversation is about Frankfurt._ — the same way recall is. After a research turn the same line holds the office or subject — _This conversation is about Bundeskanzler._ — so _und der von Frankreich?_ or _nein in Russland_ is not searched as a fragment. Recency decides which one: an older Frankfurt does not leak onto a chancellor follow-up. It is derived from the chat, never written to IndexedDB, and it stays off a fresh question that names its own subject. Switching memory off does not drop it: that toggle is for stored facts, not for this conversation. | Kind | Is | Recalled | | ------------ | ---------------------------------- | ----------------------------- | diff --git a/src/eval/runner.test.ts b/src/eval/runner.test.ts index 1945dea..49a6e38 100644 --- a/src/eval/runner.test.ts +++ b/src/eval/runner.test.ts @@ -255,6 +255,46 @@ Use the calculator.`, ) }) + it('pins the last research subject onto a correction that only names a place', async () => { + const catalog = loadCatalog() + const client = fakeClient([ + 'Präsident Russland', + 'Wladimir Putin.\n\nSource: https://de.wikipedia.org/wiki/Präsident_Russlands', + ]) + const [attempt] = await runEval(client, { + scenarios: [ + { + id: 'research-correction', + category: 'web', + history: [ + { role: 'user', content: 'Wer ist der russische Präsident?' }, + { role: 'assistant', content: 'Wladimir Putin.' }, + ], + prompt: 'nein in russland', + expectTool: 'research', + acceptCall: (calls) => + /russland|russia|präsident|president/i.test(String(calls[0]?.arguments.query ?? '')), + accept: () => true, + }, + ], + arms: [{ id: 'baseline+skills', strategy: STRATEGIES.baseline, skills: catalog }], + repeats: 1, + tools: builtinTools, + }) + + expect(attempt?.skill).toBe('research-question') + expect(attempt?.skillReason).toBe('carried-over') + expect(attempt?.calledWell).toBe(true) + + const [turns] = vi.mocked(client.generate).mock.calls[0] ?? [] + expect(turns?.[0]).toEqual( + expect.objectContaining({ + role: 'system', + content: expect.stringContaining('This conversation is about russische Präsident.'), + }), + ) + }) + it('records what the answer check found and whether it fixed it', async () => { const [attempt] = await runEval(fakeClient([CALL, 'It comes to 5.', '2 + 2 = 4']), { scenarios: [arithmetic], diff --git a/src/eval/scenarios.ts b/src/eval/scenarios.ts index 74ed4bf..86fd1af 100644 --- a/src/eval/scenarios.ts +++ b/src/eval/scenarios.ts @@ -557,6 +557,24 @@ export const SCENARIOS: Scenario[] = [ accept: matches(/macron/i), online: true, }, + { + id: 'web-follow-up-correction', + category: 'web', + // *nein in Russland* used to keep the skill and skip the tool, so the + // model guessed Putin from training data — hedged, and with the wrong date. + history: [ + { role: 'user', content: 'Wer ist der russische Präsident?' }, + { + role: 'assistant', + content: 'Wladimir Putin.\n\nSource: https://de.wikipedia.org/wiki/Präsident_Russlands', + }, + ], + prompt: 'nein in russland', + expectTool: 'research', + acceptCall: (calls) => /russland|russia|präsident|president|putin/i.test(searchQuery(calls) ?? ''), + accept: matches(/putin/i), + online: true, + }, { id: 'no-tool-summarize-pronoun', category: 'no-tool', diff --git a/src/memory/topic.test.ts b/src/memory/topic.test.ts index d2f6633..7f7d732 100644 --- a/src/memory/topic.test.ts +++ b/src/memory/topic.test.ts @@ -250,6 +250,25 @@ describe('conversationTopic', () => { ) }) + it('pins the last office onto a correction that only names a place', () => { + expect(conversationTopic('nein in russland', chancellorResearch)).toBe( + 'This conversation is about Bundeskanzler.', + ) + expect(conversationTopic('nein in Russland', chancellorResearch)).toBe( + 'This conversation is about Bundeskanzler.', + ) + }) + + it('pins a researched office onto nein in Russland after that office was asked', () => { + const russia: TopicTurn[] = [ + turn('user', 'Wer ist der russische Präsident?'), + turn('assistant', 'Wladimir Putin.'), + ] + expect(conversationTopic('nein in russland', russia)).toBe( + 'This conversation is about russische Präsident.', + ) + }) + it('stays silent when the follow-up names a different person', () => { expect(conversationTopic('und Elon Musk?', chancellorResearch)).toBe('') }) diff --git a/src/memory/topic.ts b/src/memory/topic.ts index adf3752..d4df9fe 100644 --- a/src/memory/topic.ts +++ b/src/memory/topic.ts @@ -12,8 +12,8 @@ import { tokenize } from './text' * 0.8B model does not reliably read earlier turns, so the one fact worth * keeping is pinned into the system prompt the same way recall is. For weather * and the clock that is the last place. For research it is the last office or - * question — *Bundeskanzler* — so *und der von Frankreich?* is not searched as - * a fragment. Nothing here is written down. + * question — *Bundeskanzler* — so *und der von Frankreich?* or *nein in + * Russland* is not searched as a fragment. Nothing here is written down. * * A fresh question that names its own subject — *Wer ist Elon Musk?* — must * not receive it. Mixing Frankfurt into that prompt is how the model answers @@ -37,7 +37,7 @@ export type Established = { kind: 'place'; text: string } | { kind: 'subject'; t const REFERS_BACK = /\b(dort|da|davon|darüber|dabei|hier|the (mayor|weather|forecast|city|airport)|der bürgermeister|die bürgermeisterin|die stadt|that (city|place|town)|there)\b/i -const FOLLOW_UP_PREFIX = /^\s*(and|und|auch|also|plus|what about|how about|was ist mit|oh and)\b/i +const FOLLOW_UP_PREFIX = /^\s*(and|und|auch|also|plus|what about|how about|was ist mit|oh and|nein|no)\b/i const AFTER_PREPOSITION = /\b(?:in|at|for|near|around|f(?:ü|ue)r|um)\s+(.+)$/i @@ -244,7 +244,7 @@ function namesNewResearchSubject(query: string, current: string): boolean { .replace(/^[?\s,]+/, '') .replace(/[?!.]+$/g, '') .trim() - if (!rest || mentionsTopic(rest, current)) return false + if (!rest || mentionsTopic(rest, current) || PLACE_ONLY_FOLLOW_UP.test(rest)) return false return tokenize(rest).length >= 2 } diff --git a/src/skills/library.test.ts b/src/skills/library.test.ts index 323a0d0..ed78204 100644 --- a/src/skills/library.test.ts +++ b/src/skills/library.test.ts @@ -295,6 +295,7 @@ describe('research-question', () => { ['How much are the tickets?', 'trigger'], ['Wer ist Elon Musk?', 'trigger'], ['Wer ist der Bundeskanzler?', 'trigger'], + ['Wer ist der russische Präsident?', 'trigger'], ['Wer war Ada Lovelace?', 'trigger'], ['Wer hat gewonnen?', 'trigger'], ['Was kostet ein iPhone?', 'trigger'], diff --git a/src/skills/research-question/SKILL.md b/src/skills/research-question/SKILL.md index 1554b7b..d0d11d3 100644 --- a/src/skills/research-question/SKILL.md +++ b/src/skills/research-question/SKILL.md @@ -60,52 +60,46 @@ jarvis: result: | Answer: António Guterres. - Researched 2026-08-26 for "current United Nations secretary-general" across 3 sources, all read in full. + Researched 2026-08-26 for "current United Nations secretary-general" across 2 sources, all read in full. 1. António Guterres - Wikipedia — https://en.wikipedia.org/wiki/António_Guterres "António Guterres has served as secretary-general of the United Nations since 2017." - 2. Secretary-General | United Nations — https://www.un.org/sg/en - "António Guterres is the ninth Secretary-General of the United Nations." answer: | António Guterres, since 2017. - Source: https://en.wikipedia.org/wiki/António_Guterres https://www.un.org/sg/en - - user: Wer ist der Bundeskanzler? + Source: https://en.wikipedia.org/wiki/António_Guterres + - user: Wer ist der russische Präsident? steps: - tool: research arguments: - query: Bundeskanzler + query: russische Präsident result: | - Answer: Friedrich Merz. + Answer: Wladimir Putin. - Researched 2026-08-26 for "Bundeskanzler" across 3 sources, all read in full. + Researched 2026-09-10 for "russische Präsident" across 1 source, all read in full. - 1. Bundeskanzler (Deutschland) – Wikipedia — https://de.wikipedia.org/wiki/Bundeskanzler_(Deutschland) - "Friedrich Merz ist seit dem 6. Mai 2025 Bundeskanzler der Bundesrepublik Deutschland." - 2. Bundeskanzler.de — https://www.bundeskanzler.de - "Friedrich Merz führt die Bundesregierung." + 1. Präsident Russlands — https://de.wikipedia.org/wiki/Präsident_Russlands + "Amtsträger ist seit 2012 Wladimir Putin." answer: | - Friedrich Merz, seit Mai 2025. + Wladimir Putin, seit 2012. - Source: https://de.wikipedia.org/wiki/Bundeskanzler_(Deutschland) https://www.bundeskanzler.de - - user: What is the capital of France? + Source: https://de.wikipedia.org/wiki/Präsident_Russlands + - user: Wer ist der Bundeskanzler? steps: - tool: research arguments: - query: capital of France + query: Bundeskanzler result: | - Answer: Paris. + Answer: Friedrich Merz. - Researched 2026-08-31 for "capital of France" across 2 sources, all read in full. + Researched 2026-08-26 for "Bundeskanzler" across 1 source, all read in full. - 1. Paris - Wikipedia — https://en.wikipedia.org/wiki/Paris - "Paris is the capital and most populous city of France." - 2. France - Wikipedia — https://en.wikipedia.org/wiki/France - "The capital is Paris." + 1. Bundeskanzler — https://de.wikipedia.org/wiki/Bundeskanzler_(Deutschland) + "Friedrich Merz ist seit dem 6. Mai 2025 Bundeskanzler." answer: | - Paris. + Friedrich Merz, seit Mai 2025. - Source: https://en.wikipedia.org/wiki/Paris + Source: https://de.wikipedia.org/wiki/Bundeskanzler_(Deutschland) - user: What's the population of Tokyo? steps: - tool: research @@ -114,12 +108,10 @@ jarvis: result: | Answer: 13.96 million people. - Researched 2026-09-09 for "population of Tokyo" across 2 sources, all read in full. + Researched 2026-09-09 for "population of Tokyo" across 1 source, all read in full. 1. Tokyo - Wikipedia — https://en.wikipedia.org/wiki/Tokyo "Tokyo has a population of 13.96 million people." - 2. Statistics Bureau — https://www.stat.go.jp/english/ - "The prefecture is home to 14 million residents." answer: | About 14 million. @@ -140,8 +132,24 @@ jarvis: Emmanuel Macron. Source: https://de.wikipedia.org/wiki/Emmanuel_Macron + - user: nein in Russland + steps: + - tool: research + arguments: + query: Präsident Russland + result: | + Answer: Wladimir Putin. + + Researched 2026-09-10 for "Präsident Russland" across 1 source, all read in full. + + 1. Präsident Russlands — https://de.wikipedia.org/wiki/Präsident_Russlands + "Amtsträger ist seit 2012 Wladimir Putin." + answer: | + Wladimir Putin. + + Source: https://de.wikipedia.org/wiki/Präsident_Russlands --- -Call `research` once. Copy the first line, in the language you were asked. Cite the source. +Call `research` once. Copy the first line, in the language you were asked. Cite the source. Do not guess. -A follow-up like _und der von Frankreich?_ keeps the last office and adds the new place. +A follow-up or a correction (_und der von Frankreich?_, _nein in Russland_) keeps the last office and adds the new place. diff --git a/src/skills/route.test.ts b/src/skills/route.test.ts index 7fb6ead..cd1bc5b 100644 --- a/src/skills/route.test.ts +++ b/src/skills/route.test.ts @@ -307,18 +307,28 @@ describe('keeping a skill across a follow-up', () => { expect(reason('und der von Frankreich?', research)).toBe('carried-over') }) + it('keeps research for a correction that only names a place', () => { + const research: SkillMemory = { name: 'research-question', carried: 0 } + expect(routed('nein in russland', research)).toBe('research-question') + expect(reason('nein in russland', research)).toBe('carried-over') + }) + it('forgets a skill that is no longer installed', () => { expect(routed('and in Lisbon?', { name: 'removed-skill', carried: 0 })).toBeNull() }) }) describe('isFollowUp', () => { - it.each(['and in Lisbon?', 'und morgen?', 'What about Rome?', 'Lisbon?', 'the day after?'])( - 'reads %j as a continuation', - (message) => { - expect(isFollowUp(message)).toBe(true) - }, - ) + it.each([ + 'and in Lisbon?', + 'und morgen?', + 'What about Rome?', + 'Lisbon?', + 'the day after?', + 'nein in russland', + ])('reads %j as a continuation', (message) => { + expect(isFollowUp(message)).toBe(true) + }) it.each([ // Six words, and answering it with another skill's exemplars resident would diff --git a/src/tools/research.test.ts b/src/tools/research.test.ts index 6db917e..790115b 100644 --- a/src/tools/research.test.ts +++ b/src/tools/research.test.ts @@ -107,6 +107,14 @@ describe('related', () => { expect(related('news', 'newspaper')).toBe(false) expect(related('the', 'there')).toBe(false) }) + + it('treats a nationality and its country as the same place', () => { + expect(related('russische', 'russland')).toBe(true) + expect(related('russische', 'russlands')).toBe(true) + expect(related('russische', 'russia')).toBe(true) + expect(related('französische', 'frankreich')).toBe(true) + expect(related('französische', 'france')).toBe(true) + }) }) describe('focusQuery', () => { @@ -214,6 +222,23 @@ describe('pickCandidates', () => { expect(chosen[0]?.url).toBe('https://en.wikipedia.org/wiki/Paris') expect(chosen[1]?.url).toBe('https://www.britannica.com/paris') }) + + it('gives the Wikipedia slot to the country the question named, not another president', () => { + const chosen = pickCandidates('Wer ist der russische Präsident?', [ + result( + 'https://de.wikipedia.org/wiki/Bundespräsident_(Deutschland)', + 'Bundespräsident (Deutschland)', + 'Amtsträger ist seit dem 18. März 2017 Frank-Walter Steinmeier.', + ), + result( + 'https://de.wikipedia.org/wiki/Präsident_Russlands', + 'Präsident Russlands', + 'Amtsträger ist seit dem 7. Mai 2012 Wladimir Putin.', + ), + ]) + + expect(chosen[0]?.url).toBe('https://de.wikipedia.org/wiki/Präsident_Russlands') + }) }) describe('paragraphsOf', () => { @@ -685,6 +710,54 @@ describe('extractAnswer', () => { ).toBe('Ursula von der Leyen') }) + it('does not take a German Amtsträger line for a different country', () => { + expect( + extractAnswer('Wer ist der russische Präsident?', [ + { + url: 'https://de.wikipedia.org/wiki/Bundeskanzler_(Deutschland)', + title: 'Bundeskanzler (Deutschland)', + passages: [ + 'Amtsträger ist seit dem 6. Mai 2025 Friedrich Merz (CDU).', + 'Der Bundeskanzler ist der Regierungschef der Bundesrepublik Deutschland.', + ], + read: true, + }, + ]), + ).toBeNull() + }) + + it('reads a Russian Amtsträger line as the incumbent', () => { + expect( + extractAnswer('Wer ist der russische Präsident?', [ + { + url: 'https://de.wikipedia.org/wiki/Präsident_Russlands', + title: 'Präsident Russlands', + passages: ['Amtsträger ist seit dem 7. Mai 2012 Wladimir Putin.'], + read: true, + }, + ]), + ).toBe('Wladimir Putin') + }) + + it('prefers the name that fits the country over a German office holder on another page', () => { + expect( + extractAnswer('Wer ist der russische Präsident?', [ + { + url: 'https://de.wikipedia.org/wiki/Bundespräsident_(Deutschland)', + title: 'Bundespräsident (Deutschland)', + passages: ['Amtsträger ist seit dem 18. März 2017 Frank-Walter Steinmeier.'], + read: true, + }, + { + url: 'https://de.wikipedia.org/wiki/Präsident_Russlands', + title: 'Präsident Russlands', + passages: ['Wladimir Putin ist seit 2012 der russische Präsident.'], + read: true, + }, + ]), + ).toBe('Wladimir Putin') + }) + it('does not answer a population question with the city name', () => { expect( extractAnswer("What's the population of Tokyo?", [ diff --git a/src/tools/research.ts b/src/tools/research.ts index 32a7b0b..bca63a9 100644 --- a/src/tools/research.ts +++ b/src/tools/research.ts @@ -85,6 +85,63 @@ function words(text: string): string[] { return [...text.toLowerCase().matchAll(WORD)].map((match) => match[0]) } +/** + * Nationality and country names that mean the same place. + * + * `related` only sees a shared prefix of five letters, so *russische* and + * *Russland* look unrelated — and a page titled *Präsident Russlands* then + * scores the same as any other president page on *präsident* alone. That is + * how *Wer ist der russische Präsident?* could open a German Amtsträger line. + */ +const PLACE_FAMILY = [ + /^(russisch\w*|russian|russland\w*|russia)$/, + /^(franz(?:ö|oe)sisch\w*|french|frankreich\w*|france)$/, + /^(amerikanisch\w*|american|usa)$/, + /^(britisch\w*|british|britain|england)$/, + /^(deutsch\w*|german|deutschland\w*|germany)$/, + /^(ukrainisch\w*|ukrainian|ukraine)$/, + /^(chinesisch\w*|chinese|china)$/, + /^(spanisch\w*|spanish|spanien\w*|spain)$/, + /^(italienisch\w*|italian|italien\w*|italy)$/, + /^(japanisch\w*|japanese|japan)$/, + /^(europ(?:ä|ae)isch\w*|european|europa\w*|europe)$/, + /^(indisch\w*|indian|indien\w*|india)$/, + /^(t(?:ü|ue)rkisch\w*|turkish|t(?:ü|ue)rkei\w*|turkey)$/, + /^(polnisch\w*|polish|polen\w*|poland)$/, + /^(niederl(?:ä|ae)ndisch\w*|dutch|niederlande\w*|netherlands)$/, + /^(brasilianisch\w*|brazilian|brasilien\w*|brazil)$/, + /^(kanadisch\w*|canadian|kanada\w*|canada)$/, + /^(australisch\w*|australian|australien\w*|australia)$/, + /^(mexikanisch\w*|mexican|mexiko\w*|mexico)$/, + /^(s(?:ü|ue)dkoreanisch\w*|korean|korea\w*)$/, + /^(israelisch\w*|israeli|israel)$/, + /^(schweizerisch\w*|swiss|schweiz\w*|switzerland)$/, + /^(oesterreich\w*|österreich\w*|austrian|austria)$/, +] + +function samePlace(a: string, b: string): boolean { + const left = a.toLowerCase() + const right = b.toLowerCase() + return PLACE_FAMILY.some((pattern) => pattern.test(left) && pattern.test(right)) +} + +/** + * Whether the question named a country or nationality that this evidence never + * mentions. + * + * An Amtsträger line is enough for *Wer ist der Bundeskanzler?* because that + * question names no place. The same line on a German page must not become + * `Answer: Friedrich Merz` for *russische Präsident* — that is the first + * failure in the screenshot, and `wrong-fact` then forces the model to repeat + * it. + */ +function fitsAskedPlace(question: string, evidence: string): boolean { + const asked = words(question).filter((token) => PLACE_FAMILY.some((pattern) => pattern.test(token))) + if (asked.length === 0) return true + const present = words(evidence) + return asked.every((token) => present.some((word) => samePlace(token, word))) +} + /** * Whether two tokens are the same word in different clothes. * @@ -92,10 +149,13 @@ function words(text: string): string[] { * answered by a sentence about the *Bundeskanzlers* Amt, and treating those as * unrelated is what made a German page silent on a German question. A shared * prefix of five letters, with only a short suffix on either side, catches the - * inflections without treating *news* as *newspaper*. + * inflections without treating *news* as *newspaper*. A nationality and its + * country are the same place even when the stems diverge (*russische* / + * *Russland*). */ export function related(a: string, b: string): boolean { if (a === b) return true + if (samePlace(a, b)) return true if (a.length < 5 || b.length < 5) return false const n = Math.min(a.length, b.length) let i = 0 @@ -1050,6 +1110,7 @@ export function extractFigure(question: string, sources: Source[]): string | nul for (const passage of source.passages) { for (const hit of figuresIn(passage)) { if (!figureFitsQuestion(hit, question)) continue + if (!fitsAskedPlace(question, `${source.title} ${source.url} ${passage}`)) continue const key = figureKey(hit) const current = found.get(key) const patterned = statesFigure(hit, passage, question) @@ -1108,6 +1169,10 @@ export function extractFigure(question: string, sources: Source[]): string | nul * * A question that asked for a figure — population, price — never falls * through to a name. *Tokyo* is not an answer to *how many people live there*. + * + * A nationality or country in the question has to appear in the same source. + * *Amtsträger ist Friedrich Merz* answers *Bundeskanzler*; it does not answer + * *russische Präsident*. */ export function extractAnswer(question: string, sources: Source[]): string | null { if (wantsFigure(question)) return extractFigure(question, sources) @@ -1119,6 +1184,7 @@ export function extractAnswer(question: string, sources: Source[]): string | nul const when = dated(passage) > 0 for (const name of namesIn(passage)) { if (namedInQuestion(name, question)) continue + if (!fitsAskedPlace(question, `${source.title} ${source.url} ${passage}`)) continue const key = nameKey(name) if (!key) continue const current = found.get(key)