From fa6bf0a100fda7cffe244d7d76a6d9de7fa68425 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 31 Aug 2026 10:33:37 +0000 Subject: [PATCH 1/2] Make research pull Wikipedia, retry failed pages, and match inflections A 0.8B model cannot recover from a firewall page or a German noun that only appears inflected. Research now searches Wikipedia alongside the web, ranks candidates by snippet, replaces a blocked page from the remaining hits, and treats Bundeskanzler/Bundeskanzlers as one term. Co-authored-by: Sebastian --- README.md | 2 +- src/eval/scenarios.ts | 6 +- src/tools/research.test.ts | 238 +++++++++++++++++++++++++++-- src/tools/research.ts | 296 ++++++++++++++++++++++++++++++------- 4 files changed, 477 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 9d010de..b82be7c 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,7 @@ Search and `read_page` share the reader's budget of 20 requests a minute per IP, **The search itself now carries the facts a 0.8B model would otherwise spend a round guessing at.** Every `web_search` result is stamped with today's local date, so "current" and "today's news" have a date without calling `current_time`. A German question searches German Wikipedia and, on DuckDuckGo, prefers German results (`kl=de-de`); English _who was Ada Lovelace_ is not mistaken for German because bare `was` is also English. German Wikipedia is smaller, so an empty result there falls through to English rather than telling the model the subject does not exist. -The `research-question` skill does not offer `web_search`. It offers `research`, which searches, picks three **different sites** (`investor.nvidia.com` and `nvidianews.nvidia.com` count as one), reads them in parallel, and returns the passages that bear on the question. Wikipedia pages go through MediaWiki, so a typical call spends one reader request on the search and two on the other sites rather than six. A page that will not open becomes its search snippet; if nothing readable comes back the tool throws rather than telling the model the subject does not exist. `lookup-term` still searches and optionally reads one page — a name does not need three sources. +The `research-question` skill does not offer `web_search`. It offers `research`, which searches the live web and Wikipedia together, picks three **different sites** (Wikipedia first when it appeared, because MediaWiki is free and the lead paragraph usually answers; `investor.nvidia.com` and `nvidianews.nvidia.com` count as one), ranks the rest by whether their snippet already bears on the question, and reads them in parallel. A page that will not open, or that comes back as a login wall or a firewall interstitial, is replaced from the remaining hits rather than quoted; only when nothing else is left does the search snippet stand in. Inflected forms of a word still match — _Bundeskanzlers_ answers _Bundeskanzler_ — so a German page is not silent on a German question. If nothing readable comes back the tool throws rather than telling the model the subject does not exist. `lookup-term` still searches and optionally reads one page — a name does not need three sources. **LangSearch is the way off that shared budget without paying for one.** `api.langsearch.com` is a search API rather than a results page, its free tier allows 1,000 searches a day and one a second, and a key needs no card — so a search stops competing with `read_page` for the same 20 requests a minute. Two things about it are worth knowing before choosing it. Its snippets are index text rather than prose, lower-cased and with spaces around the punctuation, which a 0.8B model reads less confidently than a sentence. And it answers in an envelope: a refusal it decides to report with a 200 arrives as a `msg` and no result set, so `searchLangSearch` raises that rather than passing an empty list to a model that would relay it as "this does not exist". Long summaries are available per result and are switched off — each is the whole page behind the result, which would leave a 0.8B context with no room for the answer. diff --git a/src/eval/scenarios.ts b/src/eval/scenarios.ts index 3e6c11d..3b39d68 100644 --- a/src/eval/scenarios.ts +++ b/src/eval/scenarios.ts @@ -380,7 +380,8 @@ export const SCENARIOS: Scenario[] = [ // `who is` and `who won` were triggers and authorship was not, so the one // question a search engine answers best reached nothing. prompt: 'Who wrote Dune?', - expectTool: 'web_search', + expectTool: 'research', + acceptCall: (calls) => /dune/i.test(searchQuery(calls) ?? ''), accept: matches(/herbert/i), online: true, }, @@ -390,7 +391,8 @@ export const SCENARIOS: Scenario[] = [ // A figure a 0.8B model will otherwise invent, confidently and to three // significant figures. prompt: "What's the population of Tokyo?", - expectTool: 'web_search', + expectTool: 'research', + acceptCall: (calls) => /tokyo|tokio/i.test(searchQuery(calls) ?? ''), accept: (answer) => /\d/.test(answer), online: true, }, diff --git a/src/tools/research.test.ts b/src/tools/research.test.ts index 4b37bad..4bfc2d4 100644 --- a/src/tools/research.test.ts +++ b/src/tools/research.test.ts @@ -1,5 +1,15 @@ import { afterEach, describe, expect, it, vi } from 'vitest' -import { digest, diverseFirst, looksBlocked, paragraphsOf, passagesFor, researchQuestion } from './research' +import { + digest, + diverseFirst, + isUnreadableUrl, + looksBlocked, + paragraphsOf, + passagesFor, + pickCandidates, + related, + researchQuestion, +} from './research' import type { SearchResult, WebAccessConfig } from './web' function result(url: string, title = 'Title', snippet = 'A snippet.'): SearchResult { @@ -82,6 +92,85 @@ describe('diverseFirst', () => { }) }) +describe('related', () => { + it('treats a German inflection as the same word', () => { + expect(related('bundeskanzler', 'bundeskanzlers')).toBe(true) + expect(related('kanzler', 'kanzlerin')).toBe(true) + }) + + it('does not treat a short stem as every word that starts with it', () => { + expect(related('news', 'newspaper')).toBe(false) + expect(related('the', 'there')).toBe(false) + }) +}) + +describe('isUnreadableUrl', () => { + it('flags a login wall and a PDF gallery', () => { + expect(isUnreadableUrl('https://example.com/login')).toBe(true) + expect(isUnreadableUrl('https://nvidianews.nvidia.com/_gallery/download_pdf/68af/')).toBe(true) + }) + + it('leaves an ordinary article alone', () => { + expect(isUnreadableUrl('https://investor.nvidia.com/news/q2/')).toBe(false) + }) +}) + +describe('pickCandidates', () => { + const question = 'Who is the chief executive of Nvidia?' + + it('puts a Wikipedia article ahead of other sites, because MediaWiki is free', () => { + const chosen = pickCandidates(question, [ + result('https://www.reuters.com/nvidia', 'Reuters', 'Nvidia named a new chief executive.'), + result('https://en.wikipedia.org/wiki/Nvidia', 'Nvidia', 'Nvidia is a computing company.'), + result('https://www.bbc.co.uk/nvidia', 'BBC', 'Nvidia chief executive spoke today.'), + ]) + + expect(chosen[0]?.url).toBe('https://en.wikipedia.org/wiki/Nvidia') + }) + + it('ranks other sites by whether the snippet already answers the question', () => { + const chosen = pickCandidates(question, [ + result('https://flights.example/routes', 'Routes', 'The airline flies to thirty cities.'), + result( + 'https://fictionalairways.example/leadership', + 'Leadership', + 'Ama Osei was appointed chief executive in March 2023.', + ), + ]) + + expect(chosen[0]?.url).toBe('https://fictionalairways.example/leadership') + }) + + it('demotes a login wall behind pages that can actually be read', () => { + const chosen = pickCandidates(question, [ + result('https://example.com/login', 'Sign in', 'Sign in to read about the chief executive.'), + result('https://news.example/nvidia', 'News', 'Nvidia chief executive Jensen Huang.'), + ]) + + expect(chosen.map((entry) => entry.url)).toEqual([ + 'https://news.example/nvidia', + 'https://example.com/login', + ]) + }) + + it('does not let a second Wikipedia article crowd out another site', () => { + const chosen = pickCandidates(question, [ + result('https://en.wikipedia.org/wiki/Nvidia', 'Nvidia', 'Nvidia is a computing company.'), + result( + 'https://en.wikipedia.org/wiki/Jensen_Huang', + 'Huang', + 'Jensen Huang is Nvidia chief executive.', + ), + result('https://www.reuters.com/nvidia', 'Reuters', 'Nvidia chief executive spoke today.'), + ]) + + expect(chosen.slice(0, 2).map((entry) => entry.url)).toEqual([ + 'https://en.wikipedia.org/wiki/Nvidia', + 'https://www.reuters.com/nvidia', + ]) + }) +}) + describe('paragraphsOf', () => { it('keeps link text and drops the address', () => { const [paragraph] = paragraphsOf( @@ -286,6 +375,17 @@ describe('passagesFor', () => { expect(first).toContain('Ama Osei') }) + it('matches a German inflection, so the answering sentence is not silent', () => { + const page = [ + 'Die Bundesrepublik hat eine parlamentarische Demokratie mit einem Bundestag.', + 'Das Amt des Bundeskanzlers übt Friedrich Merz seit dem 6. Mai 2025 aus.', + ].join('\n\n') + + const [first] = forOnePage('Wer ist der Bundeskanzler?', page) + + expect(first).toContain('Friedrich Merz') + }) + it('returns at most two passages, however long the page is', () => { expect(forOnePage('airline chief executive founded Accra 1974', PAGE)).toHaveLength(2) }) @@ -439,6 +539,7 @@ describe('digest', () => { /** * The whole call, over a stubbed network: LangSearch answers the search and the * reader answers each page, which is the shape every provider reduces to. + * Wikipedia is searched alongside and answers empty unless a test fills it. */ describe('researchQuestion', () => { const config: WebAccessConfig = { provider: 'langsearch', langsearchApiKey: 'key' } @@ -446,16 +547,28 @@ describe('researchQuestion', () => { const LEADERSHIP = 'https://fictionalairways.example/leadership' const AIRTIMES = 'https://airtimes.example/osei' + const GAZETTE = 'https://gazette.example/osei' + const WIKI = 'https://en.wikipedia.org/wiki/Fictional_Airways' const HITS = [ { name: 'Leadership', url: LEADERSHIP, snippet: 'Ama Osei leads the airline.' }, { name: 'Airtimes', url: AIRTIMES, snippet: 'The board appointed Ama Osei in 2023.' }, ] + const LEADERSHIP_PAGE = 'Ama Osei has led Fictional Airways as chief executive since March 2023 in Accra.' + const AIRTIMES_PAGE = + 'The board appointed Ama Osei as chief executive in March 2023, succeeding Piet Hendriks.' + const WIKI_EXTRACT = + 'Fictional Airways is an airline based in Accra. Ama Osei has been its chief executive since 2023.' + /** A page body, or the status the reader should refuse it with. */ type Reply = string | number - function stubNetwork(pages: Record, hits = HITS) { + function stubNetwork( + pages: Record, + hits = HITS, + wiki: { title: string; url: string; extract: string }[] = [], + ) { const calls: string[] = [] vi.stubGlobal( @@ -468,6 +581,53 @@ describe('researchQuestion', () => { return { ok: true, status: 200, json: async () => ({ data: { webPages: { value: hits } } }) } } + if (url.includes('wikipedia.org')) { + const parsed = new URL(url) + if (parsed.searchParams.has('gsrsearch')) { + return { + ok: true, + status: 200, + json: async () => ({ + query: { + pages: Object.fromEntries( + wiki.map((entry, at) => [ + String(at + 1), + { + pageid: at + 1, + title: entry.title, + index: at + 1, + extract: entry.extract, + fullurl: entry.url, + }, + ]), + ), + }, + }), + } + } + const requested = (parsed.searchParams.get('titles') ?? '').replace(/_/g, ' ') + const match = wiki.find((entry) => entry.title.replace(/_/g, ' ') === requested) + if (match) { + return { + ok: true, + status: 200, + json: async () => ({ + query: { + pages: { + '1': { + pageid: 1, + title: match.title, + extract: match.extract, + fullurl: match.url, + }, + }, + }, + }), + } + } + return { ok: true, status: 200, json: async () => ({ batchcomplete: '' }) } + } + const target = url.replace('https://r.jina.ai/', '') const reply = pages[target] if (reply === undefined || typeof reply === 'number') { @@ -491,8 +651,8 @@ describe('researchQuestion', () => { it('quotes every source it could read, each against its own URL', async () => { stubNetwork({ - [LEADERSHIP]: 'Ama Osei has led Fictional Airways as chief executive since March 2023 in Accra.', - [AIRTIMES]: 'The board appointed Ama Osei as chief executive in March 2023, succeeding Piet Hendriks.', + [LEADERSHIP]: LEADERSHIP_PAGE, + [AIRTIMES]: AIRTIMES_PAGE, }) const result = await researchQuestion(question, config) @@ -508,15 +668,18 @@ describe('researchQuestion', () => { it('reads the pages at the same time rather than one after another', async () => { const calls = stubNetwork({ - [LEADERSHIP]: 'Ama Osei has led Fictional Airways as chief executive since March 2023 in Accra.', - [AIRTIMES]: 'The board appointed Ama Osei as chief executive in March 2023, succeeding Piet Hendriks.', + [LEADERSHIP]: LEADERSHIP_PAGE, + [AIRTIMES]: AIRTIMES_PAGE, }) await researchQuestion(question, config) - // One search and one read per source: the cost of the call is what the - // reader's per-minute budget is spent on, so it is worth pinning. - expect(calls).toHaveLength(3) + const searches = calls.filter( + (url) => url.startsWith('https://api.langsearch.com') || url.includes('gsrsearch'), + ) + const reads = calls.filter((url) => url.startsWith('https://r.jina.ai/')) + expect(searches).toHaveLength(2) + expect(reads).toHaveLength(2) }) /** @@ -526,7 +689,7 @@ describe('researchQuestion', () => { */ it('stands a page that would not open in as its search snippet', async () => { stubNetwork({ - [LEADERSHIP]: 'Ama Osei has led Fictional Airways as chief executive since March 2023 in Accra.', + [LEADERSHIP]: LEADERSHIP_PAGE, [AIRTIMES]: 429, }) @@ -536,6 +699,61 @@ describe('researchQuestion', () => { expect(result).toContain('"The board appointed Ama Osei in 2023."') }) + it('replaces a blocked page from the remaining hits rather than quoting the firewall', async () => { + const fourth = 'https://profile.example/osei' + stubNetwork( + { + [LEADERSHIP]: 'Sucuri WebSite Firewall - Access Denied. Time: 2026-08-31.', + [AIRTIMES]: AIRTIMES_PAGE, + [GAZETTE]: 'Ama Osei took office as chief executive of Fictional Airways in Accra in 2023.', + [fourth]: 'Ama Osei joined Fictional Airways from the civil aviation authority in 2014.', + }, + [ + { + name: 'Leadership', + url: LEADERSHIP, + snippet: 'Ama Osei is the chief executive of Fictional Airways.', + }, + ...HITS.slice(1), + { name: 'Gazette', url: GAZETTE, snippet: 'Ama Osei took office in Accra.' }, + { name: 'Profile', url: fourth, snippet: 'A profile of the airline.' }, + ], + ) + + const result = await researchQuestion(question, config) + + expect(result).not.toContain('Access Denied') + expect(result).toContain(fourth) + expect(result).toContain('all read in full') + }) + + it('searches Wikipedia alongside the web and reads the article through MediaWiki', async () => { + stubNetwork( + { + [LEADERSHIP]: LEADERSHIP_PAGE, + [AIRTIMES]: AIRTIMES_PAGE, + }, + HITS, + [{ title: 'Fictional Airways', url: WIKI, extract: WIKI_EXTRACT }], + ) + + const result = await researchQuestion(question, config) + + expect(result).toContain(WIKI) + expect(result).toContain('Ama Osei has been its chief executive since 2023') + expect(result).toContain('across 3 sources, all read in full') + }) + + it('does not search Wikipedia twice when Wikipedia is already the provider', async () => { + const calls = stubNetwork({}, [], [{ title: 'Fictional Airways', url: WIKI, extract: WIKI_EXTRACT }]) + + await researchQuestion(question, { provider: 'wikipedia' }) + + const searches = calls.filter((url) => url.includes('gsrsearch')) + expect(searches).toHaveLength(1) + expect(calls.some((url) => url.startsWith('https://api.langsearch.com'))).toBe(false) + }) + it('says so when no page opened and only snippets are left', async () => { stubNetwork({ [LEADERSHIP]: 500, [AIRTIMES]: 429 }) diff --git a/src/tools/research.ts b/src/tools/research.ts index 9d7b6ef..2aa8f3b 100644 --- a/src/tools/research.ts +++ b/src/tools/research.ts @@ -14,12 +14,16 @@ * so a typical call is one search and two reads. Five sources was six requests * and three questions in a minute before the rate limit. * - * The pages are read in full and quoted in part. Selection is lexical: paragraphs - * are scored by the question's terms, each weighted by how rare it is across - * every paragraph the turn fetched. + * Wikipedia is fetched alongside the web, because MediaWiki is free and the + * lead paragraph is usually the sentence that names the person. A page that + * comes back as a firewall, a login wall or empty prose is replaced from the + * remaining hits rather than quoted; the search snippet only stands in when + * nothing else could be opened. Passages are scored lexically against the + * question, with inflected forms of a word counting as the same term, so a + * German page is not silent on a German question. */ -import { readPage, searchWeb, type SearchResult, type WebAccessConfig } from './web' +import { readPage, searchWeb, wikipediaPage, type SearchResult, type WebAccessConfig } from './web' /** * How many results to ask for before narrowing them. Larger than `MAX_SOURCES` @@ -31,6 +35,19 @@ const SEARCH_LIMIT = 8 /** Three independent sites. A fourth is usually the same claim from a mirror. */ const MAX_SOURCES = 3 +/** + * How many page-reads a turn may spend filling those three slots. + * + * The first wave is `MAX_SOURCES` in parallel. A blocked or empty page spends + * one of the remainder on a replacement rather than quoting its snippet while + * unread hits sit unused. Five is one search-plus-three plus two retries, still + * inside the reader's 20-a-minute budget for a single question. + */ +const MAX_READ_ATTEMPTS = 5 + +/** Enough Wikipedia hits to have a lead article after disambiguations are demoted. */ +const WIKI_SEARCH_LIMIT = 3 + /** Two passages carry a claim and its context. A third is usually the same claim again. */ const MAX_PASSAGES_PER_SOURCE = 2 @@ -55,6 +72,32 @@ function words(text: string): string[] { return [...text.toLowerCase().matchAll(WORD)].map((match) => match[0]) } +/** + * Whether two tokens are the same word in different clothes. + * + * German office titles inflect: a question about the *Bundeskanzler* is + * answered by a sentence about the *Bundeskanzlers* Amt, and treating those as + * unrelated is what made a German page silent on a German question. A shared + * prefix of five letters, with only a short suffix on either side, catches the + * inflections without treating *news* as *newspaper*. + */ +export function related(a: string, b: string): boolean { + if (a === b) return true + if (a.length < 5 || b.length < 5) return false + const n = Math.min(a.length, b.length) + let i = 0 + while (i < n && a[i] === b[i]) i += 1 + return i >= 5 && a.length - i <= 4 && b.length - i <= 4 +} + +function holds(haystack: Set, term: string): boolean { + if (haystack.has(term)) return true + for (const word of haystack) { + if (related(term, word)) return true + } + return false +} + function collapse(value: string): string { return value.replace(/\s+/g, ' ').trim() } @@ -76,6 +119,35 @@ function hostOf(url: string): string { } } +function isWikipediaUrl(url: string): boolean { + try { + return wikipediaPage(new URL(url)) !== null + } catch { + return false + } +} + +/** + * A URL that will not yield a page worth quoting. + * + * Login walls survive the reader as a 200 with prose on them, and then score + * against the question because they mention the site. `download_pdf` is the + * NVIDIA gallery that used to occupy a source slot with a binary rather than + * an article. Demoted rather than dropped, so a search that returned nothing + * else still has something to try. + */ +export function isUnreadableUrl(url: string): boolean { + try { + const path = new URL(url).pathname.toLowerCase() + return ( + /\/(?:login|log-in|signin|sign-in|sign-up|signup|register|consent)(?:\/|$)/.test(path) || + path.includes('download_pdf') + ) + } catch { + return true + } +} + /** * Compound public suffixes where the last two labels are not the site. * @@ -304,18 +376,83 @@ function weigh(question: string, corpus: string[]): Map { const weights = new Map() for (const term of terms) { - const seen = tokenized.filter((paragraph) => paragraph.has(term)).length + const seen = tokenized.filter((paragraph) => holds(paragraph, term)).length if (seen > 0) weights.set(term, inverseFrequency(seen, tokenized.length)) } return weights } function score(text: string, weights: Map): number { + const present = new Set(words(text)) let total = 0 - for (const term of new Set(words(text))) total += weights.get(term) ?? 0 + for (const [term, weight] of weights) { + if (holds(present, term)) total += weight + } return total } +function rankBySnippet(question: string, results: SearchResult[]): SearchResult[] { + if (results.length <= 1) return results + const weights = weigh( + question, + results.map((result) => `${result.title} ${result.snippet}`), + ) + return results + .map((result, at) => ({ + result, + at, + score: score(`${result.title} ${result.snippet}`, weights), + })) + .sort((a, b) => b.score - a.score || a.at - b.at) + .map((entry) => entry.result) +} + +/** + * The results worth reading, in the order they should be tried. + * + * Wikipedia first, because MediaWiki is free and the lead paragraph usually + * names the person. Then one hit per remaining site, ranked by whether the + * snippet already bears on the question, so a PDF gallery sitting at rank two + * does not spend a reader request ahead of the article that answers it. Extra + * pages from a site already chosen, and URLs that will not yield a page, come + * last — they fill a slot only when nothing else is left. + */ +export function pickCandidates(question: string, results: SearchResult[]): SearchResult[] { + const ordered = diverseFirst(results, results.length) + const seen = new Set() + const primary: SearchResult[] = [] + const extra: SearchResult[] = [] + + for (const entry of ordered) { + const site = siteOf(entry.url) + if (seen.has(site)) extra.push(entry) + else { + seen.add(site) + primary.push(entry) + } + } + + const wiki: SearchResult[] = [] + const other: SearchResult[] = [] + const junk: SearchResult[] = [] + for (const entry of primary) { + if (isUnreadableUrl(entry.url)) junk.push(entry) + else if (isWikipediaUrl(entry.url)) wiki.push(entry) + else other.push(entry) + } + + const extraReadable = extra.filter((entry) => !isUnreadableUrl(entry.url)) + const extraJunk = extra.filter((entry) => isUnreadableUrl(entry.url)) + + return [ + ...wiki, + ...rankBySnippet(question, other), + ...rankBySnippet(question, extraReadable), + ...junk, + ...extraJunk, + ] +} + const SENTENCE_END = /(?<=[.!?…])\s+/ /** @@ -451,71 +588,126 @@ export function digest(question: string, sources: Source[]): string { return `${body.slice(0, MAX_DIGEST_CHARS)}\n\n[Truncated: further sources were dropped.]` } -function reasonsFrom(settled: PromiseSettledResult[]): string { - return settled - .flatMap((outcome) => - outcome.status === 'rejected' - ? [outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason)] - : [], - ) - .join('; ') +/** + * Wikipedia alongside the web, never instead of it. + * + * MediaWiki is free of the reader's budget and the lead paragraph usually + * names the person, so a DuckDuckGo or LangSearch turn that never returned + * Wikipedia used to spend three reader requests on news pages and still miss + * the sentence that answered the question. A failure here is swallowed: the + * web results are still an answer, and a thrown encyclopedia search would + * take them down with it. + */ +async function encyclopediaHits(question: string, config: WebAccessConfig): Promise { + if (config.provider === 'wikipedia') return [] + try { + return await searchWeb(question, WIKI_SEARCH_LIMIT, { provider: 'wikipedia' }) + } catch { + return [] + } +} + +interface Opened { + url: string + title: string + paragraphs: string[] } /** - * Searches, reads the most promising results in parallel and returns the - * passages that bear on the question, each with the URL it came from. + * Reads until three pages have prose on them, or the attempt budget is gone. * - * A page that fails is replaced by its search snippet rather than allowed to - * take the answer down with it — the reader's per-minute budget is shared, so a - * 429 on the third page is an ordinary event and not a reason to abandon the - * two that arrived. + * The first wave is parallel. A blocked, empty or refused page does not keep + * its slot: the next unread candidate is tried, up to `MAX_READ_ATTEMPTS`. + * Snippets from the failed hits only stand in once nothing else can be opened, + * and a firewall body is never quoted — the search snippet for that URL at + * least came from the index. */ -export async function researchQuestion(question: string, config: WebAccessConfig): Promise { - const results = await searchWeb(question, SEARCH_LIMIT, config) - if (results.length === 0) return `Researched ${todayStamp()} for "${question}". No results.` - - const selected = diverseFirst(results, MAX_SOURCES) - const settled = await Promise.allSettled(selected.map((result) => readPage(result.url, config))) +async function readBest( + question: string, + candidates: SearchResult[], + config: WebAccessConfig, +): Promise<{ sources: Source[]; reasons: string[] }> { + const opened: Opened[] = [] + const fallbacks: SearchResult[] = [] + const reasons: string[] = [] + let next = 0 + let attempts = 0 + + while (opened.length < MAX_SOURCES && next < candidates.length && attempts < MAX_READ_ATTEMPTS) { + const take = Math.min(MAX_SOURCES - opened.length, MAX_READ_ATTEMPTS - attempts, candidates.length - next) + const batch = candidates.slice(next, next + take) + next += batch.length + attempts += batch.length + + const settled = await Promise.allSettled(batch.map((entry) => readPage(entry.url, config))) + + for (const [at, result] of batch.entries()) { + const outcome = settled[at] + if (outcome?.status === 'fulfilled') { + const { title, text, url } = outcome.value + if (!looksBlocked(title, text)) { + const paragraphs = paragraphsOf(text) + if (paragraphs.length > 0) { + opened.push({ url, title: title || result.title, paragraphs }) + continue + } + } + } else if (outcome?.status === 'rejected') { + reasons.push(outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason)) + } + fallbacks.push(result) + } + } const chosen = passagesFor( question, - settled.map((outcome) => { - if (outcome.status !== 'fulfilled') return [] - const { title, text } = outcome.value - // A page the site refused to serve arrives as a 200 with prose on it. Left - // in, it is a source that says nothing and cannot be told from one that - // does; the search snippet for the same URL at least came from the index. - return looksBlocked(title, text) ? [] : paragraphsOf(text) - }), + opened.map((entry) => entry.paragraphs), ) - const sources = selected.flatMap((result, at): Source[] => { - const outcome = settled[at] + const sources: Source[] = [] + for (const [at, entry] of opened.entries()) { const passages = chosen[at] ?? [] + if (passages.length === 0) continue + sources.push({ url: entry.url, title: entry.title, passages, read: true }) + } - if (outcome?.status === 'fulfilled' && passages.length > 0) { - return [ - { - url: outcome.value.url, - title: outcome.value.title || result.title, - passages, - read: true, - }, - ] - } - + for (const result of fallbacks) { + if (sources.length >= MAX_SOURCES) break const snippet = collapse(result.snippet) - if (!snippet) return [] - return [{ url: result.url, title: result.title, passages: [snippet], read: false }] - }) + if (!snippet) continue + sources.push({ url: result.url, title: result.title, passages: [snippet], read: false }) + } + + return { sources, reasons } +} + +/** + * Searches the web and Wikipedia, reads the most promising results in parallel + * and returns the passages that bear on the question, each with the URL it + * came from. + * + * A page that fails is replaced from the remaining hits rather than allowed to + * take the answer down with it — the reader's per-minute budget is shared, so a + * 429 on the third page is an ordinary event and not a reason to abandon the + * two that arrived. + */ +export async function researchQuestion(question: string, config: WebAccessConfig): Promise { + const [webResults, wikiResults] = await Promise.all([ + searchWeb(question, SEARCH_LIMIT, config), + encyclopediaHits(question, config), + ]) + + const combined = [...wikiResults, ...webResults] + if (combined.length === 0) return `Researched ${todayStamp()} for "${question}". No results.` + + const { sources, reasons } = await readBest(question, pickCandidates(question, combined), config) // Every source silent means the search found pages and nothing could be read // off any of them. Reporting that as a result would have the model relay it as // "there is nothing on this", which is the one thing it must not say. if (sources.length === 0) { - const reasons = reasonsFrom(settled) throw new Error( - `Found ${results.length} results for "${question}" but could not read any of them${reasons ? `: ${reasons}` : '.'}`, + `Found ${combined.length} results for "${question}" but could not read any of them${reasons.length > 0 ? `: ${reasons.join('; ')}` : '.'}`, ) } From 60f1375ec9ef698e9d9381a675a8d8c16c959c13 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 31 Aug 2026 10:42:10 +0000 Subject: [PATCH 2/2] Keep Wikipedia's incumbent line when the lead is a definition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live run of Wer ist der Bundeskanzler? quoted the etymology of Kanzler and never Friedrich Merz. The name sits in a one-line Amtsträger paragraph at the end of a 1,500-character lead; the length floor dropped it and condense preferred the definition at the start. Dated claims now join the paragraph they follow, and a dated lead is trimmed to a window that keeps the year. Co-authored-by: Sebastian --- README.md | 2 +- src/tools/research.test.ts | 47 ++++++++++++ src/tools/research.ts | 148 +++++++++++++++++++++++-------------- 3 files changed, 142 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index b82be7c..f22dee2 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,7 @@ Search and `read_page` share the reader's budget of 20 requests a minute per IP, **The search itself now carries the facts a 0.8B model would otherwise spend a round guessing at.** Every `web_search` result is stamped with today's local date, so "current" and "today's news" have a date without calling `current_time`. A German question searches German Wikipedia and, on DuckDuckGo, prefers German results (`kl=de-de`); English _who was Ada Lovelace_ is not mistaken for German because bare `was` is also English. German Wikipedia is smaller, so an empty result there falls through to English rather than telling the model the subject does not exist. -The `research-question` skill does not offer `web_search`. It offers `research`, which searches the live web and Wikipedia together, picks three **different sites** (Wikipedia first when it appeared, because MediaWiki is free and the lead paragraph usually answers; `investor.nvidia.com` and `nvidianews.nvidia.com` count as one), ranks the rest by whether their snippet already bears on the question, and reads them in parallel. A page that will not open, or that comes back as a login wall or a firewall interstitial, is replaced from the remaining hits rather than quoted; only when nothing else is left does the search snippet stand in. Inflected forms of a word still match — _Bundeskanzlers_ answers _Bundeskanzler_ — so a German page is not silent on a German question. If nothing readable comes back the tool throws rather than telling the model the subject does not exist. `lookup-term` still searches and optionally reads one page — a name does not need three sources. +The `research-question` skill does not offer `web_search`. It offers `research`, which searches the live web and Wikipedia together, picks three **different sites** (Wikipedia first when it appeared, because MediaWiki is free and the lead paragraph usually answers; `investor.nvidia.com` and `nvidianews.nvidia.com` count as one), ranks the rest by whether their snippet already bears on the question, and reads them in parallel. A page that will not open, or that comes back as a login wall or a firewall interstitial, is replaced from the remaining hits rather than quoted; only when nothing else is left does the search snippet stand in. Inflected forms of a word still match — _Bundeskanzlers_ answers _Bundeskanzler_ — and a one-line Wikipedia claim that names the incumbent (_Amtsträger ist seit dem 6. Mai 2025 Friedrich Merz_) is kept even when the lead is a definition of the office. If nothing readable comes back the tool throws rather than telling the model the subject does not exist. `lookup-term` still searches and optionally reads one page — a name does not need three sources. **LangSearch is the way off that shared budget without paying for one.** `api.langsearch.com` is a search API rather than a results page, its free tier allows 1,000 searches a day and one a second, and a key needs no card — so a search stops competing with `read_page` for the same 20 requests a minute. Two things about it are worth knowing before choosing it. Its snippets are index text rather than prose, lower-cased and with spaces around the punctuation, which a 0.8B model reads less confidently than a sentence. And it answers in an envelope: a refusal it decides to report with a 200 arrives as a `msg` and no result set, so `searchLangSearch` raises that rather than passing an empty list to a model that would relay it as "this does not exist". Long summaries are available per result and are switched off — each is the whole page behind the result, which would leave a 0.8B context with no room for the answer. diff --git a/src/tools/research.test.ts b/src/tools/research.test.ts index 4bfc2d4..84b92f3 100644 --- a/src/tools/research.test.ts +++ b/src/tools/research.test.ts @@ -273,6 +273,23 @@ describe('paragraphsOf', () => { ).toEqual([]) }) + /** + * Wikipedia names the incumbent in a one-line paragraph that fails every + * length floor. Joining it to the definition it follows is what keeps + * *Friedrich Merz* in the source for *wer ist der Bundeskanzler*. + */ + it('joins a short dated claim to the paragraph it follows', () => { + const [paragraph] = paragraphsOf( + [ + 'Vor Ablauf der Legislaturperiode kann der Bundeskanzler nur durch ein konstruktives Misstrauensvotum abgelöst werden, indem der Bundestag mit absoluter Mehrheit einen Nachfolger wählt.', + 'Amtsträger ist seit dem 6. Mai 2025 Friedrich Merz (CDU).', + ].join('\n\n'), + ) + + expect(paragraph).toContain('Friedrich Merz') + expect(paragraph).toContain('Misstrauensvotum') + }) + /** * Both of these outranked the sentence that answered the question when this ran * against the live web, and they are why the filters exist rather than being a @@ -386,6 +403,36 @@ describe('passagesFor', () => { expect(first).toContain('Friedrich Merz') }) + it('prefers the dated claim that names the person over a definition of the office', () => { + const page = [ + 'Der Bundeskanzler der Bundesrepublik Deutschland ist der Regierungschef der Bundesrepublik Deutschland.', + 'Vor Ablauf der Legislaturperiode kann der Bundeskanzler nur durch ein konstruktives Misstrauensvotum abgelöst werden, indem der Bundestag mit absoluter Mehrheit einen Nachfolger wählt.', + 'Amtsträger ist seit dem 6. Mai 2025 Friedrich Merz (CDU).', + ].join('\n\n') + + const [first] = forOnePage('Wer ist der Bundeskanzler?', page) + + expect(first).toContain('Friedrich Merz') + }) + + it('keeps the incumbent line when condensing a long Wikipedia lead', () => { + const lead = [ + 'Der Bundeskanzler der Bundesrepublik Deutschland (kurz: Bundeskanzler; Abkürzung BK) ist der Regierungschef der Bundesrepublik Deutschland.', + 'Er bildet zusammen mit den Bundesministern die Bundesregierung und bestimmt laut Verfassung die Richtlinien deren Politik.', + 'In der Praxis muss er allerdings die Vorstellungen seiner eigenen Partei und der Koalitionspartner berücksichtigen.', + 'Zuweilen wird, aufgrund der Machtfülle des Bundeskanzlers, auch von einer Kanzlerdemokratie gesprochen.', + 'Vor Ablauf der Legislaturperiode kann der Bundeskanzler nur durch ein konstruktives Misstrauensvotum abgelöst werden, indem der Bundestag mit absoluter Mehrheit einen Nachfolger wählt.', + 'Für den Fall, dass ein Bundeskanzler stirbt oder zurücktritt, endet die Kanzlerschaft und damit auch die Bundesregierung.', + 'In diesem Fall bittet der Bundespräsident gemäß Verfassung einen Bundesminister, bis zur Ernennung eines Nachfolgers weiterhin die Geschäfte zu führen.', + 'Amtsträger ist seit dem 6. Mai 2025 Friedrich Merz (CDU).', + ].join(' ') + + expect(lead.length).toBeGreaterThan(280) + const [first] = forOnePage('Wer ist der Bundeskanzler?', lead) + + expect(first).toContain('Friedrich Merz') + }) + it('returns at most two passages, however long the page is', () => { expect(forOnePage('airline chief executive founded Accra 1974', PAGE)).toHaveLength(2) }) diff --git a/src/tools/research.ts b/src/tools/research.ts index 2aa8f3b..a37ec97 100644 --- a/src/tools/research.ts +++ b/src/tools/research.ts @@ -259,6 +259,21 @@ const LINE_FURNITURE = /^\s*(?:[-*+]\s+|>\s+|\d+\.\s+)/ */ const ENDS_A_SENTENCE = /[.!?…][)"'”’]*$/ +/** + * A year in this century. + * + * Wikipedia names a current office holder in a one-line paragraph of its own — + * *Amtsträger ist seit dem 6. Mai 2025 Friedrich Merz (CDU).* — which is shorter + * than the prose floor and never repeats the office title. The year is what + * distinguishes that claim from a heading, and what prefers it over the + * etymology that otherwise wins on the title's own words. + */ +const THIS_CENTURY = /\b20\d{2}\b/ + +function dated(text: string): number { + return THIS_CENTURY.test(text) ? 1 : 0 +} + /** * Boilerplate that reads exactly like prose and answers nothing. * @@ -296,41 +311,59 @@ const REFERENCE_LIST = * `#cite_note-fitch20240226-50`, which is exactly that. */ export function paragraphsOf(markdown: string): string[] { - return ( - markdown - .replace(IMAGE, ' ') - .replace(LINK_TARGET, ']') - .replace(FOOTNOTE, '') - // Whatever brackets are left were a link's text. A nested `[[47]](url)` is why - // this strips them rather than matching a whole link in one pattern. - // - // A space, not nothing, and the same for the emphasis marks below: a page - // writes two links with nothing between them, so deleting the brackets fuses - // what they held. `our@NVIDIATwitter account,NVIDIA Facebookpage` was three - // adjacent links, and it is the trap `unbold` in `web.ts` already documents. - .replace(/[[\]]/g, ' ') - .replace(BARE_URL, ' ') - .split(/\n\s*\n/) - .map((block) => - tidy( - collapse( - block - .split('\n') - .map((line) => line.replace(LINE_FURNITURE, '')) - .join(' ') - .replace(HEADING_MARK, ' ') - .replace(EMPHASIS, ' '), - ), + const blocks = markdown + .replace(IMAGE, ' ') + .replace(LINK_TARGET, ']') + .replace(FOOTNOTE, '') + // Whatever brackets are left were a link's text. A nested `[[47]](url)` is why + // this strips them rather than matching a whole link in one pattern. + // + // A space, not nothing, and the same for the emphasis marks below: a page + // writes two links with nothing between them, so deleting the brackets fuses + // what they held. `our@NVIDIATwitter account,NVIDIA Facebookpage` was three + // adjacent links, and it is the trap `unbold` in `web.ts` already documents. + .replace(/[[\]]/g, ' ') + .replace(BARE_URL, ' ') + .split(/\n\s*\n/) + .map((block) => + tidy( + collapse( + block + .split('\n') + .map((line) => line.replace(LINE_FURNITURE, '')) + .join(' ') + .replace(HEADING_MARK, ' ') + .replace(EMPHASIS, ' '), ), - ) - .filter( - (block) => - block.length >= MIN_PASSAGE_CHARS && - words(block).length >= MIN_PASSAGE_WORDS && - ENDS_A_SENTENCE.test(block) && - !BOILERPLATE.test(block) && - !REFERENCE_LIST.test(block), - ) + ), + ) + .filter(Boolean) + + // A dated one-liner is usually the sentence that names the incumbent, sitting + // on its own after a long definition. Joining it to the paragraph it follows + // is what keeps it above the length floor without treating every short + // sentence as prose. + const merged: string[] = [] + for (const block of blocks) { + if ( + merged.length > 0 && + block.length < MIN_PASSAGE_CHARS && + ENDS_A_SENTENCE.test(block) && + THIS_CENTURY.test(block) + ) { + merged[merged.length - 1] += ` ${block}` + continue + } + merged.push(block) + } + + return merged.filter( + (block) => + block.length >= MIN_PASSAGE_CHARS && + words(block).length >= MIN_PASSAGE_WORDS && + ENDS_A_SENTENCE.test(block) && + !BOILERPLATE.test(block) && + !REFERENCE_LIST.test(block), ) } @@ -470,10 +503,7 @@ function condense(paragraph: string, weights: Map): string { if (paragraph.length <= MAX_PASSAGE_CHARS) return paragraph const sentences = paragraph.split(SENTENCE_END) - let best = '' - let bestScore = -1 - let substantial = '' - let substantialScore = -1 + const windows: { text: string; score: number }[] = [] for (let start = 0; start < sentences.length; start += 1) { let window = '' @@ -481,24 +511,34 @@ function condense(paragraph: string, weights: Map): string { const extended = window ? `${window} ${sentences[end]}` : (sentences[end] ?? '') if (extended.length > MAX_PASSAGE_CHARS) break window = extended + windows.push({ text: window, score: score(window, weights) }) + } + } - const windowScore = score(window, weights) - if (windowScore > bestScore || (windowScore === bestScore && window.length > best.length)) { - best = window - bestScore = windowScore - } - if ( - window.length >= MIN_PASSAGE_CHARS && - (windowScore > substantialScore || - (windowScore === substantialScore && window.length > substantial.length)) - ) { - substantial = window - substantialScore = windowScore - } + const substantial = windows.filter((entry) => entry.text.length >= MIN_PASSAGE_CHARS) + // A long lead names the incumbent in its last sentence. Scoring by the + // question's words prefers the definition at the start, which never has the + // year; if the paragraph is dated, only windows that kept the year compete. + const datedWindows = substantial.filter((entry) => dated(entry.text) > 0) + const pool = + dated(paragraph) > 0 && datedWindows.length > 0 + ? datedWindows + : substantial.length > 0 + ? substantial + : windows + + let best = pool[0] + for (const entry of pool) { + if ( + best === undefined || + entry.score > best.score || + (entry.score === best.score && entry.text.length > best.text.length) + ) { + best = entry } } - return substantial || best || `${paragraph.slice(0, MAX_PASSAGE_CHARS).trimEnd()}…` + return best?.text || `${paragraph.slice(0, MAX_PASSAGE_CHARS).trimEnd()}…` } /** Enough of a passage to recognise the same claim written out twice. */ @@ -517,8 +557,8 @@ function choose(candidates: string[], weights: Map): string[] { if (candidates.length === 0) return [] const ranked = candidates - .map((text, at) => ({ text, at, score: score(text, weights) })) - .sort((a, b) => b.score - a.score || a.at - b.at) + .map((text, at) => ({ text, at, score: score(text, weights), dated: dated(text) })) + .sort((a, b) => b.score - a.score || b.dated - a.dated || a.at - b.at) const relevant = ranked.filter((candidate) => candidate.score > 0) const passages: string[] = []