Say when an answer came from memory; answer greetings, typos and German normally

src/agent/budget.test.ts

@@ -5,7 +5,7 @@ import { budgetFallback, callFingerprint, repeatedCallNote, windDownNote } from
 import type { ReviewEvidence } from './review'
 
 function evidence(results: { tool: string; result: string }[] = []): ReviewEvidence {
-  return { toolResults: results, knownUrls: [] }
+  return { toolResults: results, knownUrls: [], knownFigures: [] }
 }
 
 const searchResult = {

src/agent/loop.test.ts

@@ -34,8 +34,12 @@ function toolCall(name: string, parameter: string, value: string): string {
   return `thinking</think><tool_call><function=${name}><parameter=${parameter}>${value}</parameter></function></tool_call>`
 }
 
+/**
+ * The year is in here because the answers below state it. A figure the search
+ * did not return is its own finding now, and these cases are about citations.
+ */
 const searchResult =
-  '1. Leadership — Fictional Airways\n   https://fictionalairways.example/leadership\n   Ama Osei leads it.'
+  '1. Leadership — Fictional Airways\n   https://fictionalairways.example/leadership\n   Ama Osei has led it since 2023.'
 
 describe('runAgent', () => {
   it('keeps reasoning separate when the model states an answer', async () => {
@@ -380,6 +384,39 @@ describe('checking the answer before returning it', () => {
     expect(hooks.onCorrection).toHaveBeenCalledWith(['missing-source'])
   })
 
+  it('labels a lookup that was answered without looking anything up', async () => {
+    // The tool list is the signal: a skill narrowed this turn to research tools,
+    // so a reply that fetched nothing is the model's recollection. Nothing else
+    // in the review can see this case, because every other check needs evidence.
+    const client = fakeClient(['I know this</think>Hitler lived from 1889 to 1945.'])
+
+    const result = await runAgent(client, turns, [search], callbacks())
+
+    expect(client.generate).toHaveBeenCalledTimes(1)
+    expect(result.review).toEqual({ found: [], corrected: false, unsourced: true })
+  })
+
+  it('does not label a turn that was never a lookup', async () => {
+    // Six tools means nobody decided this was a lookup, and "write me a rhyme"
+    // must not come back marked as unsourced research.
+    const client = fakeClient(['done</think>Rain on the roof, and a tap on the proof.'])
+
+    const result = await runAgent(client, turns, [search, calculator], callbacks())
+
+    expect(result.review).toEqual({ found: [], corrected: false })
+  })
+
+  it('does not label a lookup that did search', async () => {
+    const client = fakeClient([
+      toolCall('web_search', 'query', 'Fictional Airways chief executive'),
+      'read it</think>Ama Osei, since 2023.\n\nSource: https://fictionalairways.example/leadership',
+    ])
+
+    const result = await runAgent(client, turns, [search], callbacks())
+
+    expect(result.review).toEqual({ found: [], corrected: false })
+  })
+
   it('can be switched off so the eval can measure what it is worth', async () => {
     const client = fakeClient([
       toolCall('web_search', 'query', 'Fictional Airways chief executive'),

src/agent/loop.ts

@@ -20,8 +20,10 @@ import { parseModelOutput, parsePartial, type ParsedToolCall } from './parse'
 import {
   collectEvidence,
   correctionPrompt,
+  isUnsourced,
   reviewAnswer,
   type ReviewCheck,
+  type ReviewEvidence,
   type ReviewOutcome,
 } from './review'
 
@@ -78,6 +80,33 @@ interface Draft {
   found: ReviewCheck[]
 }
 
+/** The tools whose whole purpose is to fetch something the model does not know. */
+const RESEARCH_TOOLS = new Set(['web_search', 'read_page'])
+
+/**
+ * Whether this turn was supposed to answer out of a source.
+ *
+ * Read off the tool list rather than plumbed in, because the tool list already
+ * carries the decision: a skill narrows it to what that kind of request needs,
+ * so a turn offered nothing but research tools is a turn some router decided was
+ * a lookup. The default list has six tools in it and never qualifies, which is
+ * what keeps the label off "write me a rhyme".
+ */
+function expectsSource(tools: Tool[]): boolean {
+  return tools.length > 0 && tools.every((tool) => RESEARCH_TOOLS.has(tool.schema.function.name))
+}
+
+/**
+ * Marks a finished answer that was meant to come from a source and did not.
+ *
+ * Applied last, to the text that is actually going out — including the wind-down
+ * fallback, which is nothing but sources and so is never unsourced.
+ */
+function labelSourcing(answer: AgentResult, evidence: ReviewEvidence): AgentResult {
+  if (!isUnsourced(answer.content, evidence)) return answer
+  return { ...answer, review: { ...(answer.review ?? { found: [], corrected: false }), unsourced: true } }
+}
+
 /**
  * Decides which of the two answers the user gets, and what to say about it.
  *
@@ -130,6 +159,7 @@ export async function runAgent(
   const strategy = options.strategy ?? DEFAULT_STRATEGY
   const checking = options.review ?? true
   const evidence = collectEvidence(turns)
+  const sourcing = checking && expectsSource(tools)
 
   let last: AgentResult = {
     content: '',
@@ -185,14 +215,16 @@ export async function runAgent(
       const found = findings.map((finding) => finding.check)
 
       if (findings.length === 0 || corrections >= MAX_CORRECTIONS) {
-        const answer = settle(last, found, draft)
-        if (!windDown) return answer
+        const settled = settle(last, found, draft)
         // The round that had to answer produced nothing, and `settle` found no
         // earlier draft to fall back on. What the tools did return is worth
         // more than the apology this used to end on. Substituted after the
         // check rather than before it, because deterministic text assembled
         // from tool results has nothing for a correction round to fix.
-        return { ...answer, content: answer.content || budgetFallback(evidence), windDown: true }
+        const answer = windDown
+          ? { ...settled, content: settled.content || budgetFallback(evidence), windDown: true }
+          : settled
+        return sourcing ? labelSourcing(answer, evidence) : answer
       }
 
       corrections += 1

src/agent/review.test.ts

@@ -1,14 +1,14 @@
 import { describe, expect, it } from 'vitest'
-import { collectEvidence, correctionPrompt, reviewAnswer, type ReviewEvidence } from './review'
+import { collectEvidence, correctionPrompt, isUnsourced, reviewAnswer, type ReviewEvidence } from './review'
 
 function evidence(overrides: Partial<ReviewEvidence> = {}): ReviewEvidence {
-  return { toolResults: [], knownUrls: [], ...overrides }
+  return { toolResults: [], knownUrls: [], knownFigures: [], ...overrides }
 }
 
 const searchResult = {
   tool: 'web_search',
   result:
-    '1. Leadership — Fictional Airways\n   https://fictionalairways.example/leadership\n   Ama Osei leads it.',
+    '1. Leadership — Fictional Airways\n   https://fictionalairways.example/leadership\n   Ama Osei has led it since 2023.',
 }
 
 function checks(answer: string, given: ReviewEvidence): string[] {
@@ -73,6 +73,66 @@ describe('reviewAnswer', () => {
     })
   })
 
+  describe('figures', () => {
+    const searched = evidence({ toolResults: [searchResult] })
+
+    it('catches a figure no source gave', () => {
+      // The reported failure: an answer about a life that stated an age nothing
+      // had returned. Its only finding used to be that it cited no source, so
+      // the correction attached a real URL to an invented number.
+      expect(
+        checks('He lived to 142.\n\nSource: https://fictionalairways.example/leadership', searched),
+      ).toEqual(['unsupported-figure'])
+    })
+
+    it('names the figure to drop', () => {
+      const [finding] = reviewAnswer('He was 200 years old.', searched)
+
+      expect(finding?.instruction).toContain('No source gives 200')
+    })
+
+    it('accepts a figure the search returned', () => {
+      expect(checks('Since 2023.\n\nSource: https://fictionalairways.example/leadership', searched)).toEqual(
+        [],
+      )
+    })
+
+    it('accepts a figure the user supplied', () => {
+      const asked = evidence({ toolResults: [searchResult], knownFigures: ['1889'] })
+
+      expect(
+        checks('Yes, 1889 is right.\n\nSource: https://fictionalairways.example/leadership', asked),
+      ).toEqual([])
+    })
+
+    it.each([
+      ['an age that is two digits', 'He was 56 when he died.'],
+      ['a percentage', 'Revenue grew 85%.'],
+      ['a figure with a decimal point', 'Revenue was 81.62 billion.'],
+      ['a figure with a thousands separator', 'It reached 46,700 units.'],
+      ['a thousands group written with a space', 'It reached 46 700 units.'],
+    ])('is too shy to challenge %s', (_case, answer) => {
+      // Every one of these is a way a correct number can be written that the
+      // evidence does not contain verbatim. A check that fires on a correct
+      // answer costs a generation and teaches the reader to ignore the label.
+      expect(checks(`${answer}\n\nSource: https://fictionalairways.example/leadership`, searched)).toEqual([])
+    })
+
+    it('ignores digits that are part of a URL', () => {
+      const linked = evidence({
+        toolResults: [{ tool: 'web_search', result: 'https://example.com/2026/08/report' }],
+      })
+
+      expect(checks('See https://example.com/2026/08/report', linked)).toEqual([])
+    })
+
+    it('says nothing about figures when no tool returned anything', () => {
+      // With no evidence every figure is unsupported, and reporting that would
+      // be telling the model off for answering. `isUnsourced` covers this case.
+      expect(checks('He lived to 142.', evidence())).toEqual([])
+    })
+  })
+
   describe('sources', () => {
     const searched = evidence({ toolResults: [searchResult] })
 
@@ -164,6 +224,48 @@ describe('reviewAnswer', () => {
   })
 })
 
+describe('isUnsourced', () => {
+  const searched = evidence({ toolResults: [searchResult] })
+
+  it('is true for a factual answer no tool contributed to', () => {
+    // The whole reason this exists: every other check needs evidence to fire, so
+    // the answer that consulted nothing was the one nothing was said about.
+    expect(isUnsourced('Hitler lived from 1889 to 1945.', evidence())).toBe(true)
+  })
+
+  it('is false once a tool returned a source, cited or not', () => {
+    // Not citing it is `missing-source`, which has a fix. Calling a turn that
+    // searched "answered from memory" would simply be untrue.
+    expect(isUnsourced('Ama Osei runs it.', searched)).toBe(false)
+  })
+
+  it('is false when the answer cites a source from earlier in the conversation', () => {
+    // A follow-up runs no tools of its own, and the source is still on screen.
+    const followUp = evidence({ knownUrls: ['https://fictionalairways.example/leadership'] })
+
+    expect(isUnsourced('Since 2023.\n\nSource: https://fictionalairways.example/leadership', followUp)).toBe(
+      false,
+    )
+  })
+
+  it('is false for a URL the model invented', () => {
+    // `invented-source` owns that one, and stacking both labels on one reply
+    // would say the same thing twice.
+    expect(isUnsourced('Ama Osei.\n\nSource: https://madeup.example/ceo', searched)).toBe(false)
+  })
+
+  it.each(['Which city do you mean?', 'I could not find out who runs it.'])(
+    'is false for %j, which claims nothing',
+    (answer) => {
+      expect(isUnsourced(answer, evidence())).toBe(false)
+    },
+  )
+
+  it('is false for an empty draft, which the reasoning promotion handles', () => {
+    expect(isUnsourced('   ', evidence())).toBe(false)
+  })
+})
+
 describe('collectEvidence', () => {
   it('takes the URLs already in the conversation and no tool results yet', () => {
     const collected = collectEvidence([
@@ -178,8 +280,21 @@ describe('collectEvidence', () => {
     expect(collected).toEqual({
       toolResults: [],
       knownUrls: ['https://example.com/pricing', 'https://example.com/old'],
+      knownFigures: [],
     })
   })
+
+  it('takes figures from the user and nowhere else', () => {
+    // A skill's worked example is an assistant turn. Reading figures out of it
+    // would whitelist the example's own numbers on every turn that skill wins.
+    const collected = collectEvidence([
+      { role: 'user', content: 'Is 1889 right?' },
+      { role: 'assistant', content: 'Earlier I said 1723.' },
+      { role: 'tool', content: 'A tool once returned 1456.' },
+    ])
+
+    expect(collected.knownFigures).toEqual(['1889'])
+  })
 })
 
 describe('correctionPrompt', () => {