From fcaa6b1a1eeb13b386464f81f12625ec26393ef2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 10 Sep 2026 09:33:13 +0000 Subject: [PATCH] Check that replies copy the researched Answer line. Research already extracts a name or figure onto the first line of the digest. The 0.8B model still invents a different one, or quotes the office instead of the incumbent. reviewAnswer now treats that line the way it already treats the calculator: if the draft drops it, one correction asks for it back. The check stays shy. A surname standing in for the full name, a German decimal, and a figure rounded the way extractFigure already treats as one reading all pass. A biography digest with no Answer line is left alone. When the wind-down round comes back empty, budgetFallback now hands over that same one-liner instead of an apology. Co-authored-by: Sebastian --- .cursor/skills/debug-model-output/SKILL.md | 3 +- README.md | 15 ++- src/agent/budget.test.ts | 22 ++++ src/agent/budget.ts | 11 +- src/agent/loop.test.ts | 23 ++++ src/agent/review.test.ts | 145 ++++++++++++++++++++- src/agent/review.ts | 106 ++++++++++++++- src/components/Landing.tsx | 2 +- src/components/MessageItem.test.tsx | 16 +++ src/components/MessageItem.tsx | 1 + 10 files changed, 331 insertions(+), 13 deletions(-) diff --git a/.cursor/skills/debug-model-output/SKILL.md b/.cursor/skills/debug-model-output/SKILL.md index 18381b4..2884052 100644 --- a/.cursor/skills/debug-model-output/SKILL.md +++ b/.cursor/skills/debug-model-output/SKILL.md @@ -60,7 +60,8 @@ Three things follow from that, and all three are easy to undo by accident: A check that cannot point at a tool result does not belong here. - **A check that fires on a correct answer is a bug**, not a strict setting. It costs a generation and trains the user to ignore the label. `review.test.ts` pins the shy cases — a clarifying - question, a rounded decimal, a source carried over from an earlier turn — and they are the point. + question, a rounded decimal, a researched surname standing in for the full name, a source + carried over from an earlier turn — and they are the point. - **Only successful tool results become evidence.** A failed fetch has nothing to check against, and demanding a citation for a page that never loaded is worse than saying nothing. diff --git a/README.md b/README.md index 26be74a..031717e 100644 --- a/README.md +++ b/README.md @@ -550,19 +550,20 @@ Every reply says which skill answered it and how it was found: `weather skill · A skill fires on some requests. This runs on all of them. -Between the model settling on an answer and that answer reaching the screen, `src/agent/review.ts` reads it back against what the turn actually produced — the results the tools returned, and the URLs already in the conversation. Three things are checked: +Between the model settling on an answer and that answer reaching the screen, `src/agent/review.ts` reads it back against what the turn actually produced — the results the tools returned, and the URLs already in the conversation. Four things are checked: -| Check | Fires when | -| ----------------- | --------------------------------------------------------------------- | -| `wrong-number` | The calculator's value, or the clock's local HH:MM, is stated nowhere | -| `invented-source` | The answer cites a URL that no tool returned and nobody supplied | -| `missing-source` | Tools returned sources and the answer cites none | +| Check | Fires when | +| ----------------- | ---------------------------------------------------------------------------- | +| `wrong-number` | The calculator's value, or the clock's local HH:MM, is stated nowhere | +| `wrong-fact` | `research` opened with `Answer: …` and that name or figure is stated nowhere | +| `invented-source` | The answer cites a URL that no tool returned and nobody supplied | +| `missing-source` | Tools returned sources and the answer cites none | A failed check costs one further generation. The model is handed its own draft and told what to change — _The calculator returned 6748 \* 9 = 60732. Give that number, exactly as it came back._ — and the correction replaces the draft only if it leaves fewer problems behind. Otherwise the draft stands. That gate is the important half: the correction comes from the same 0.8B model, so a mechanism that could not tell an improvement from a regression would be a coin toss on every reply. **The checks are deterministic, and that is the design.** Asking the model to grade its own answer spends exactly the capacity the answer needed, and intrinsic self-correction — re-reading with nothing new to go on — degrades reasoning rather than improving it ([arXiv:2310.01798](https://arxiv.org/html/2310.01798)). What works is external feedback, so every check compares the draft against something already in the context, and the correction states the fix rather than inviting the model to hunt for one. -They are also deliberately shy. A clarifying question is asked for no citation; a long decimal quoted to fewer places counts as the calculator's number; citing the site when a page on it was read is close enough; a URL from an earlier reply is not an invention; a year-only clock answer is left alone, and a German date like `27.08.2026` is not a time. Every check would rather miss a mistake than invent one, because a check that fires on a correct answer costs a generation and teaches you to ignore the whole mechanism. +They are also deliberately shy. A clarifying question is asked for no citation; a long decimal quoted to fewer places counts as the calculator's number; a researched surname (`Merz`) counts as the full extract (`Friedrich Merz`); 14 million counts as 13.96 million, which is the same reading the extractor already accepted; citing the site when a page on it was read is close enough; a URL from an earlier reply is not an invention; a year-only clock answer is left alone, and a German date like `27.08.2026` is not a time. Every check would rather miss a mistake than invent one, because a check that fires on a correct answer costs a generation and teaches you to ignore the whole mechanism. The interface says what happened rather than quietly rewriting the reply. While the corrected answer streams in it is labelled with what is being fixed, and afterwards it carries `corrected` — claimed only for an answer that now passes every check — or `flagged`, naming what is still wrong with the text on screen. An answer half fixed and advertised as corrected would be worse than no check at all. diff --git a/src/agent/budget.test.ts b/src/agent/budget.test.ts index 24797be..e098acb 100644 --- a/src/agent/budget.test.ts +++ b/src/agent/budget.test.ts @@ -93,4 +93,26 @@ describe('budgetFallback', () => { expect(text).toContain('Try narrowing the question') expect(splitSources(text).sources).toEqual([]) }) + + it('hands over the researched one-liner when the wind-down came back empty', () => { + const text = budgetFallback( + evidence([ + { + tool: 'research', + result: [ + 'Answer: Friedrich Merz.', + '', + 'Researched 2026-09-10 for "Bundeskanzler" across 1 source, all read in full.', + '', + '1. Bundeskanzler — https://de.wikipedia.org/wiki/Bundeskanzler', + ' "Amtsträger ist Friedrich Merz."', + ].join('\n'), + }, + ]), + ) + + expect(text).toMatch(/^Friedrich Merz\./) + expect(text).not.toContain('could not settle') + expect(splitSources(text).sources).toEqual(['https://de.wikipedia.org/wiki/Bundeskanzler']) + }) }) diff --git a/src/agent/budget.ts b/src/agent/budget.ts index 552ecb7..b1ce502 100644 --- a/src/agent/budget.ts +++ b/src/agent/budget.ts @@ -1,5 +1,5 @@ import { MAX_TOOL_ROUNDS } from '@/llm/config' -import { findUrls, type ReviewEvidence } from './review' +import { findUrls, researchedAnswer, type ReviewEvidence } from './review' /** * What happens when a turn runs out of tool rounds. @@ -117,12 +117,19 @@ const FALLBACK_SOURCES = 3 * into citation pills — the pages are the part worth clicking. */ export function budgetFallback(evidence: ReviewEvidence): string { - const opening = `I could not settle on an answer within ${MAX_TOOL_ROUNDS} rounds of tool calls.` + const extracted = researchedAnswer(evidence) const sources = [...new Set(evidence.toolResults.flatMap(({ result }) => findUrls(result)))].slice( 0, FALLBACK_SOURCES, ) + // `research` already committed to a one-liner. Handing that over beats an + // apology: the wind-down round failed, not the search. + if (extracted) { + return sources.length > 0 ? `${extracted}.\n\nSource: ${sources.join(' ')}` : `${extracted}.` + } + + const opening = `I could not settle on an answer within ${MAX_TOOL_ROUNDS} rounds of tool calls.` if (sources.length === 0) return `${opening} Try narrowing the question.` return `${opening} These pages came up on the way, in case one of them helps.\n\nSource: ${sources.join(' ')}` } diff --git a/src/agent/loop.test.ts b/src/agent/loop.test.ts index 957ce41..31da8f5 100644 --- a/src/agent/loop.test.ts +++ b/src/agent/loop.test.ts @@ -315,6 +315,29 @@ describe('checking the answer before returning it', () => { ) }) + it('corrects a researched name the answer dropped', async () => { + const research = defineTool('research', 'research', { type: 'object', properties: {} }, async () => + [ + 'Answer: Ama Osei.', + '', + 'Researched 2026-09-10 for "who runs Fictional Airways" across 1 source, all read in full.', + '', + '1. Leadership — https://fictionalairways.example/leadership', + ' "Ama Osei has led the airline since 2023."', + ].join('\n'), + ) + const client = fakeClient([ + toolCall('research', 'query', 'who runs Fictional Airways'), + 'guessingPiet Hendriks runs it.\n\nSource: https://fictionalairways.example/leadership', + 'reading the digestAma Osei.\n\nSource: https://fictionalairways.example/leadership', + ]) + + const result = await runAgent(client, turns, [research], callbacks()) + + expect(result.content).toContain('Ama Osei') + expect(result.review).toEqual({ found: ['wrong-fact'], corrected: true }) + }) + it('corrects a number the answer did not take from the calculator', async () => { const client = fakeClient([ toolCall('calculator', 'expression', '6748 * 9'), diff --git a/src/agent/review.test.ts b/src/agent/review.test.ts index 61a763d..f75574f 100644 --- a/src/agent/review.test.ts +++ b/src/agent/review.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it } from 'vitest' -import { collectEvidence, correctionPrompt, reviewAnswer, type ReviewEvidence } from './review' +import { + collectEvidence, + correctionPrompt, + researchedAnswer, + reviewAnswer, + type ReviewEvidence, +} from './review' function evidence(overrides: Partial = {}): ReviewEvidence { return { toolResults: [], knownUrls: [], ...overrides } @@ -223,6 +229,143 @@ describe('reviewAnswer', () => { ]) }) }) + + describe('researched facts', () => { + const researched = evidence({ + toolResults: [ + { + tool: 'research', + result: [ + 'Answer: Friedrich Merz.', + '', + 'Researched 2026-09-10 for "Bundeskanzler" across 2 sources, all read in full.', + '', + '1. Bundeskanzler — https://de.wikipedia.org/wiki/Bundeskanzler_(Deutschland)', + ' "Amtsträger ist seit dem 6. Mai 2025 Friedrich Merz (CDU)."', + ].join('\n'), + }, + ], + }) + + it('accepts the name the research digest opened with', () => { + expect( + checks( + 'Friedrich Merz, seit Mai 2025.\n\nSource: https://de.wikipedia.org/wiki/Bundeskanzler_(Deutschland)', + researched, + ), + ).toEqual([]) + }) + + it('accepts the surname when the full name was extracted', () => { + // A check that flagged "Merz" would fire on a correct German short answer. + expect( + checks('Merz.\n\nSource: https://de.wikipedia.org/wiki/Bundeskanzler_(Deutschland)', researched), + ).toEqual([]) + }) + + it('accepts a missing accent on a copied name', () => { + const un = evidence({ + toolResults: [ + { + tool: 'research', + result: + 'Answer: António Guterres.\n\nResearched 2026-09-10 for "UN" across 1 source, all read in full.\n\n1. UN — https://www.un.org/sg/en', + }, + ], + }) + + expect(checks('Antonio Guterres.\n\nSource: https://www.un.org/sg/en', un)).toEqual([]) + }) + + it('catches an invented name after research already answered', () => { + expect( + checks( + 'Olaf Scholz.\n\nSource: https://de.wikipedia.org/wiki/Bundeskanzler_(Deutschland)', + researched, + ), + ).toEqual(['wrong-fact']) + }) + + it('catches a reply that quotes the office and drops the incumbent', () => { + expect( + checks( + 'The chancellor is the head of government.\n\nSource: https://de.wikipedia.org/wiki/Bundeskanzler_(Deutschland)', + researched, + ), + ).toEqual(['wrong-fact']) + }) + + it('quotes the extract in the correction', () => { + const [finding] = reviewAnswer('Olaf Scholz.', researched) + + expect(finding?.instruction).toContain('Answer: Friedrich Merz') + }) + + it('accepts a figure the digest opened with, including German decimals', () => { + const population = evidence({ + toolResults: [ + { + tool: 'research', + result: + 'Answer: 13.96 million.\n\nResearched 2026-09-10 for "population of Tokyo" across 1 source, all read in full.\n\n1. Tokyo — https://en.wikipedia.org/wiki/Tokyo', + }, + ], + }) + + expect( + checks('About 13,96 Millionen.\n\nSource: https://en.wikipedia.org/wiki/Tokyo', population), + ).toEqual([]) + expect(checks('About 14 million.\n\nSource: https://en.wikipedia.org/wiki/Tokyo', population)).toEqual( + [], + ) + }) + + it('catches an invented figure after research already answered', () => { + const population = evidence({ + toolResults: [ + { + tool: 'research', + result: + 'Answer: 13.96 million.\n\nResearched 2026-09-10 for "population of Tokyo" across 1 source, all read in full.\n\n1. Tokyo — https://en.wikipedia.org/wiki/Tokyo', + }, + ], + }) + + expect( + checks('Tokyo has 11 million people.\n\nSource: https://en.wikipedia.org/wiki/Tokyo', population), + ).toEqual(['wrong-fact']) + }) + + it('leaves a biography digest alone when nothing was extracted', () => { + const bio = evidence({ + toolResults: [ + { + tool: 'research', + result: + 'Researched 2026-09-10 for "Who is Elon Musk" across 1 source, all read in full.\n\n1. Elon Musk — https://en.wikipedia.org/wiki/Elon_Musk\n "Elon Musk is a businessman."', + }, + ], + }) + + expect( + checks('Elon Musk is a businessman.\n\nSource: https://en.wikipedia.org/wiki/Elon_Musk', bio), + ).toEqual([]) + }) + + it('does not treat an Answer line inside a web_search snippet as an extract', () => { + const searched = evidence({ + toolResults: [ + { + tool: 'web_search', + result: '1. Quiz\n https://quiz.example\n Answer: Paris is a cheese.', + }, + ], + }) + + expect(researchedAnswer(searched)).toBeNull() + expect(checks('Lyon.\n\nSource: https://quiz.example', searched)).toEqual([]) + }) + }) }) describe('collectEvidence', () => { diff --git a/src/agent/review.ts b/src/agent/review.ts index 72d257d..ea0c628 100644 --- a/src/agent/review.ts +++ b/src/agent/review.ts @@ -16,7 +16,7 @@ import { localClockInResult } from '@/tools/clock' * costs a second generation and teaches the user to ignore the whole mechanism, * so every one of them prefers to miss a mistake over inventing one. */ -export type ReviewCheck = 'wrong-number' | 'invented-source' | 'missing-source' +export type ReviewCheck = 'wrong-number' | 'wrong-fact' | 'invented-source' | 'missing-source' export interface ReviewFinding { check: ReviewCheck @@ -196,9 +196,105 @@ function latestClock(evidence: ReviewEvidence): ClockTime | null { return null } +/** + * The one-liner `research` puts at the top of a digest when the sources agree. + * + * `digest` always writes `Answer: Friedrich Merz.` — a period after the extract, + * then a blank line. The period is the line ending, not part of the name. + */ +const RESEARCHED = /^Answer:\s+(.+)\.\s*$/m + /** A question back to the user, and a plain "I could not find it", cite nothing. */ const NOTHING_TO_CITE = /\?\s*$|\b(could ?n[o']t find|no results|don'?t know|do not know|unable to find)\b/i +/** + * The extract `research` already committed to, or `null` when the digest had + * nothing confident enough to put on the first line. A biography, a tie, or a + * failed call leaves this empty, and the check stays off — inventing a fact to + * demand would be the opposite of shy. + */ +export function researchedAnswer(evidence: ReviewEvidence): string | null { + for (const { tool, result } of [...evidence.toolResults].reverse()) { + if (tool !== 'research') continue + const match = RESEARCHED.exec(result) + const extracted = match?.[1]?.trim() + if (extracted) return extracted + } + return null +} + +function fold(value: string): string { + return value + .normalize('NFD') + .replace(/\p{M}+/gu, '') + .toLowerCase() +} + +/** + * The first number in an extract, accepting either decimal mark. + * + * `3,8 Millionen` and `13.96 million` are how the digest writes German and + * English figures. A thousands-grouped integer is left alone: `1,396` with + * three digits after the comma is thirteen hundred, not 1.396. + */ +function firstNumber(text: string): number | null { + const match = text.match(/-?\d+(?:[.,]\d+)?/) + if (!match?.[0]) return null + const raw = match[0] + const comma = raw.lastIndexOf(',') + const dot = raw.lastIndexOf('.') + let normalized = raw + if (comma >= 0 && dot < 0) { + const decimals = raw.length - comma - 1 + normalized = decimals > 0 && decimals <= 2 ? raw.replace(',', '.') : raw.replace(/,/g, '') + } else if (comma >= 0 && dot >= 0 && comma > dot) { + normalized = raw.replace(/\./g, '').replace(',', '.') + } else { + normalized = raw.replace(/,/g, '') + } + const value = Number(normalized) + return Number.isFinite(value) ? value : null +} + +/** + * Whether the draft already states the researched extract. + * + * A name passes when every token of it appears, so `Friedrich Merz, seit 2025` + * is enough and `Merz` alone is not a miss we invent — the last token of a + * multi-word name is the distinctive one, and asking for the rest costs a + * generation on an answer that is already right. A figure passes when the + * number is there with either decimal mark, or when the draft rounded it the + * way `extractFigure` already treats as the same reading (about ten per cent). + */ +function statesResearched(answer: string, extracted: string): boolean { + const foldedAnswer = fold(answer) + const foldedExtracted = fold(extracted) + if (foldedAnswer.includes(foldedExtracted)) return true + + const value = firstNumber(extracted) + if (value !== null && /\d/.test(extracted)) { + if (statesNumber(answer, value)) return true + const german = answer.replace(/(\d),(\d)/g, '$1.$2') + if (statesNumber(german, value)) return true + // The first number in the draft is often a year on the citation line, so + // every number is tried. Ten per cent is the same band `extractFigure` + // already treats as one reading — 13.96 million and 14 million pass, + // 11 million does not. + const scale = Math.max(Math.abs(value), 1) + for (const match of answer.matchAll(/-?\d+(?:[.,]\d+)?/g)) { + const stated = firstNumber(match[0] ?? '') + if (stated !== null && Math.abs(stated - value) / scale <= 0.1) return true + } + return false + } + + const tokens = foldedExtracted.match(/[\p{L}\p{N}]+/gu) ?? [] + if (tokens.length === 0) return false + if (tokens.every((token) => foldedAnswer.includes(token))) return true + const last = tokens[tokens.length - 1] + return tokens.length > 1 && last !== undefined && last.length >= 4 && foldedAnswer.includes(last) +} + /** The most recent URL a tool returned, which is the one worth citing. */ function preferredSource(evidence: ReviewEvidence): string | null { for (const { result } of [...evidence.toolResults].reverse()) { @@ -241,6 +337,14 @@ export function reviewAnswer(answer: string, evidence: ReviewEvidence): ReviewFi } } + const extracted = researchedAnswer(evidence) + if (extracted && !statesResearched(draft, extracted)) { + findings.push({ + check: 'wrong-fact', + instruction: `The research result opened with Answer: ${extracted}. Give that, in the language you were asked.`, + }) + } + const source = preferredSource(evidence) const known = [...evidence.knownUrls, ...evidence.toolResults.flatMap(({ result }) => findUrls(result))] .map(locate) diff --git a/src/components/Landing.tsx b/src/components/Landing.tsx index 3e60c4d..7ceaf48 100644 --- a/src/components/Landing.tsx +++ b/src/components/Landing.tsx @@ -68,7 +68,7 @@ const STEPS: { body: string; title: string }[] = [ }, { title: 'The answer is checked', - body: 'Before a reply is shown it is read back against what the tools returned. A number the tools disagree with, or a source nothing ever fetched, is corrected or flagged.', + body: 'Before a reply is shown it is read back against what the tools returned. A number the tools disagree with, a researched fact the reply dropped, or a source nothing ever fetched, is corrected or flagged.', }, ] diff --git a/src/components/MessageItem.test.tsx b/src/components/MessageItem.test.tsx index 51a0b03..15776de 100644 --- a/src/components/MessageItem.test.tsx +++ b/src/components/MessageItem.test.tsx @@ -205,6 +205,22 @@ describe('MessageItem', () => { expect(screen.getByText(/self-check found a source no tool returned/)).toBeInTheDocument() }) + it('names a researched fact the reply dropped', () => { + render( + , + ) + + expect(screen.getByText('corrected')).toBeInTheDocument() + expect( + screen.getByText(/self-check found a researched fact the reply dropped and fixed it/), + ).toBeInTheDocument() + }) + it('names the skill a reply was answered with', () => { render( = { 'wrong-number': 'a number the tools disagreed with', + 'wrong-fact': 'a researched fact the reply dropped', 'invented-source': 'a source no tool returned', 'missing-source': 'a missing source', }