diff --git a/README.md b/README.md index a384a6d..c720a47 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Browser tab ├── Answer check ────► every reply, read back against the tool results └── Tool loop ───────► search provider (DuckDuckGo, Wikipedia, or LangSearch/Jina with your key) ├► r.jina.ai (page reader) + ├► MediaWiki (Wikipedia pages, no reader budget) └► MCP servers over HTTP ``` @@ -205,6 +206,7 @@ Three details make the app work from a repository sub-path rather than a domain | -------------- | ------------------------------------------------------------------- | | `web_search` | Full web search with no key; Wikipedia, LangSearch or Jina instead. | | `read_page` | Fetches a URL and returns its readable text. | +| `research` | Search, read three independent sites, return quoted passages. | | `calculator` | Exact arithmetic via a hand-written parser. | | `current_time` | Local date, time, and timezone. | | `weather` | Current conditions and a three-day outlook, from several forecasts. | @@ -233,7 +235,7 @@ Search and `read_page` share the reader's budget of 20 requests a minute per IP, **The search itself now carries the facts a 0.8B model would otherwise spend a round guessing at.** Every `web_search` result is stamped with today's local date, so "current" and "today's news" have a date without calling `current_time`. A German question searches German Wikipedia and, on DuckDuckGo, prefers German results (`kl=de-de`); English _who was Ada Lovelace_ is not mistaken for German because bare `was` is also English. German Wikipedia is smaller, so an empty result there falls through to English rather than telling the model the subject does not exist. -The `research-question` skill used to teach only "search, then answer from the snippet". That is how Wikipedia's lead about an office never naming the incumbent became a wrong answer. It now also shows opening the page when the snippet is not enough, and answering a German office-holder question in German from a German source. +The `research-question` skill does not offer `web_search`. It offers `research`, which searches, picks three **different sites** (`investor.nvidia.com` and `nvidianews.nvidia.com` count as one), reads them in parallel, and returns the passages that bear on the question. Wikipedia pages go through MediaWiki, so a typical call spends one reader request on the search and two on the other sites rather than six. A page that will not open becomes its search snippet; if nothing readable comes back the tool throws rather than telling the model the subject does not exist. `lookup-term` still searches and optionally reads one page — a name does not need three sources. **LangSearch is the way off that shared budget without paying for one.** `api.langsearch.com` is a search API rather than a results page, its free tier allows 1,000 searches a day and one a second, and a key needs no card — so a search stops competing with `read_page` for the same 20 requests a minute. Two things about it are worth knowing before choosing it. Its snippets are index text rather than prose, lower-cased and with spaces around the punctuation, which a 0.8B model reads less confidently than a sentence. And it answers in an envelope: a refusal it decides to report with a 200 arrives as a `msg` and no result set, so `searchLangSearch` raises that rather than passing an empty list to a model that would relay it as "this does not exist". Long summaries are available per result and are switched off — each is the whole page behind the result, which would leave a 0.8B context with no room for the answer. diff --git a/src/eval/scenarios.ts b/src/eval/scenarios.ts index 3292b86..05ddb15 100644 --- a/src/eval/scenarios.ts +++ b/src/eval/scenarios.ts @@ -59,7 +59,7 @@ export interface Scenario { } function searchQuery(calls: Invocation[]): string | null { - const search = calls.find((call) => call.name === 'web_search') + const search = calls.find((call) => call.name === 'web_search' || call.name === 'research') return search ? String(search.arguments.query ?? '') : null } @@ -261,11 +261,10 @@ export const SCENARIOS: Scenario[] = [ id: 'web-current-event', category: 'web', prompt: 'Who is the current secretary-general of the United Nations?', - expectTool: 'web_search', + expectTool: 'research', accept: matches(/guterres/i), - // The hardest of these under Wikipedia, whose lead extract describes the - // office and never names the incumbent: passing there needs a follow-up - // `read_page`. A web provider names him in the snippets. + // Wikipedia's lead about the office often never names the incumbent; this + // tool reads three pages, so the person page or a UN page can supply it. online: true, }, { @@ -344,7 +343,7 @@ export const SCENARIOS: Scenario[] = [ // A price is looked up, never worked out. `how much is` was an arithmetic // keyword, so this reached the calculator with nothing to calculate. prompt: 'How much is a Big Mac in Japan?', - expectTool: 'web_search', + expectTool: 'research', accept: (answer) => /\d/.test(answer), online: true, }, @@ -353,7 +352,7 @@ export const SCENARIOS: Scenario[] = [ category: 'web', // `today` was a current-date trigger, which answered this with the date. prompt: "What's today's news?", - expectTool: 'web_search', + expectTool: 'research', accept: (answer) => answer.trim().length > 20, online: true, }, @@ -364,7 +363,7 @@ export const SCENARIOS: Scenario[] = [ // answer in English. The query has to keep the German word; translating it // to "chancellor of germany" is the 1inch failure in another language. prompt: 'Wer ist der Bundeskanzler?', - expectTool: 'web_search', + expectTool: 'research', acceptCall: (calls) => /bundeskanzler/i.test(searchQuery(calls) ?? ''), accept: matches(/merz|scholz|kanzler/i), online: true, diff --git a/src/lib/tool-labels.test.ts b/src/lib/tool-labels.test.ts index 806ee7b..9a0604c 100644 --- a/src/lib/tool-labels.test.ts +++ b/src/lib/tool-labels.test.ts @@ -8,8 +8,8 @@ describe('describeTool', () => { }) it('speaks in the past tense once it is over, however it ended', () => { - expect(describeTool('read_page', 'done')).toBe('Read a page') - expect(describeTool('read_page', 'error')).toBe('Read a page') + expect(describeTool('research', 'running')).toBe('Researching') + expect(describeTool('research', 'done')).toBe('Researched') }) it('leaves a tool it does not ship under its own name', () => { diff --git a/src/lib/tool-labels.ts b/src/lib/tool-labels.ts index 9970c32..2d44ff3 100644 --- a/src/lib/tool-labels.ts +++ b/src/lib/tool-labels.ts @@ -14,6 +14,7 @@ import type { ToolCall } from '@/types' const PHRASES: Record = { web_search: { running: 'Searching the web', done: 'Searched the web' }, read_page: { running: 'Reading a page', done: 'Read a page' }, + research: { running: 'Researching', done: 'Researched' }, calculator: { running: 'Calculating', done: 'Calculated' }, current_time: { running: 'Checking the date', done: 'Checked the date' }, weather: { running: 'Checking the weather', done: 'Checked the weather' }, diff --git a/src/skills/library.test.ts b/src/skills/library.test.ts index 9ec119f..2d7e06a 100644 --- a/src/skills/library.test.ts +++ b/src/skills/library.test.ts @@ -33,7 +33,7 @@ describe('the shipped library', () => { ['current-date', 25, ['current_time']], ['summarize-url', 20, ['read_page']], ['lookup-term', 15, ['web_search', 'read_page']], - ['research-question', 10, ['web_search', 'read_page']], + ['research-question', 10, ['research']], ]) }) @@ -427,7 +427,7 @@ describe('activating each shipped skill', () => { ['What year is it?', 'current-date', ['current_time']], ['What does https://example.com/pricing say?', 'summarize-url', ['read_page']], ['What is Stripe?', 'lookup-term', ['web_search', 'read_page']], - ['Who is the current secretary-general of the UN?', 'research-question', ['web_search', 'read_page']], + ['Who is the current secretary-general of the UN?', 'research-question', ['research']], ] it.each(cases)('materialises %s for %j with only its tools', (message, name, tools) => { diff --git a/src/skills/research-question/SKILL.md b/src/skills/research-question/SKILL.md index 5547bfb..2e58bfd 100644 --- a/src/skills/research-question/SKILL.md +++ b/src/skills/research-question/SKILL.md @@ -1,11 +1,10 @@ --- name: research-question -description: Answers a question about current or verifiable facts by searching the web, opening the most promising result and citing it. Use for anything recent, anything about a named person or organisation, and anything you would otherwise be guessing at. +description: Answers a question about current or verifiable facts by researching it across several independent sources and citing them. Use for anything recent, anything about a named person or organisation, and anything you would otherwise be guessing at. jarvis: priority: 10 tools: - - web_search - - read_page + - research keywords: - look it up - find out @@ -53,61 +52,38 @@ jarvis: # thermometer must not be answered with a search. - '^\s*aktuelle[rs]?\s+(?!(uhrzeit|zeit|datum|temperatur|wetter|jahr|tag|woche|monat)\b)[a-zäöüß]{3,}\s*\??\s*$' exemplars: - # The failure this exists to catch: Wikipedia's lead names the office and - # not the person, so answering from the snippet invents or omits the name. - user: Who is the current secretary-general of the UN? steps: - - tool: web_search + - tool: research arguments: query: current United Nations secretary-general result: | - Searched 2026-08-26 for "current United Nations secretary-general". + Researched 2026-08-26 for "current United Nations secretary-general" across 3 sources, all read in full. - 1. Secretary-General of the United Nations - Wikipedia - https://en.wikipedia.org/wiki/Secretary-General_of_the_United_Nations - The secretary-general is the chief administrative officer of the United Nations. - - tool: read_page - arguments: - url: https://en.wikipedia.org/wiki/Secretary-General_of_the_United_Nations - result: | - # Secretary-General of the United Nations - Source: https://en.wikipedia.org/wiki/Secretary-General_of_the_United_Nations - - António Guterres of Portugal has served as secretary-general since 2017. + 1. António Guterres - Wikipedia — https://en.wikipedia.org/wiki/António_Guterres + "António Guterres has served as secretary-general of the United Nations since 2017." + 2. Secretary-General | United Nations — https://www.un.org/sg/en + "António Guterres is the ninth Secretary-General of the United Nations." answer: | António Guterres, since 2017. - Source: https://en.wikipedia.org/wiki/Secretary-General_of_the_United_Nations - - user: Who is the chief executive of Fictional Airways? - steps: - - tool: web_search - arguments: - query: Fictional Airways chief executive - result: | - Searched 2026-08-26 for "Fictional Airways chief executive". - - 1. Leadership — Fictional Airways - https://fictionalairways.example/leadership - Ama Osei has led Fictional Airways as chief executive since 2023. - answer: | - Ama Osei, chief executive since 2023. - - Source: https://fictionalairways.example/leadership + Source: https://en.wikipedia.org/wiki/António_Guterres https://www.un.org/sg/en - user: Wer ist der Bundeskanzler? steps: - - tool: web_search + - tool: research arguments: query: Bundeskanzler result: | - Searched 2026-08-26 for "Bundeskanzler". + Researched 2026-08-26 for "Bundeskanzler" across 3 sources, all read in full. - 1. Bundeskanzler (Deutschland) – Wikipedia - https://de.wikipedia.org/wiki/Bundeskanzler_(Deutschland) - Friedrich Merz ist seit Mai 2025 Bundeskanzler der Bundesrepublik Deutschland. + 1. Bundeskanzler (Deutschland) – Wikipedia — https://de.wikipedia.org/wiki/Bundeskanzler_(Deutschland) + "Friedrich Merz ist seit dem 6. Mai 2025 Bundeskanzler der Bundesrepublik Deutschland." + 2. Bundeskanzler.de — https://www.bundeskanzler.de + "Friedrich Merz führt die Bundesregierung." answer: | Friedrich Merz, seit Mai 2025. - Source: https://de.wikipedia.org/wiki/Bundeskanzler_(Deutschland) + Source: https://de.wikipedia.org/wiki/Bundeskanzler_(Deutschland) https://www.bundeskanzler.de --- -Search first, then answer from the results. Open a result with `read_page` when the snippet does not name the answer. Answer in the language you were asked. Always end with the source URL. +Call `research` once. Answer from the quoted passages, in the language you were asked. Cite more than one source URL when several came back. diff --git a/src/tools/builtins.test.ts b/src/tools/builtins.test.ts index 6cafbb9..4647d4f 100644 --- a/src/tools/builtins.test.ts +++ b/src/tools/builtins.test.ts @@ -71,6 +71,16 @@ describe('web_search', () => { expect(description('langsearch')).toMatch(/Search the web/) }) + it('refuses an empty research query without asking the network', async () => { + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + const tool = createBuiltinTools(DEFAULT_WEB_ACCESS).find( + (candidate) => candidate.schema.function.name === 'research', + )! + await expect(tool.execute({ query: ' ' })).rejects.toThrow('query must not be empty') + expect(fetchMock).not.toHaveBeenCalled() + }) + it('stamps today on the results so a current-events answer has a date', async () => { vi.useFakeTimers() vi.setSystemTime(new Date('2026-08-26T15:00:00')) diff --git a/src/tools/builtins.ts b/src/tools/builtins.ts index 130989d..b6de6ab 100644 --- a/src/tools/builtins.ts +++ b/src/tools/builtins.ts @@ -1,5 +1,6 @@ import { evaluateExpression } from './calculator' import { memory } from './memory' +import { researchQuestion } from './research' import { defineTool, type Tool } from './types' import { DEFAULT_WEB_ACCESS, readPage, searchWeb, type SearchProvider, type WebAccessConfig } from './web' import { weatherReport } from './weather' @@ -136,6 +137,23 @@ export const currentTime = defineTool( }, ) +function createResearch(config: WebAccessConfig): Tool { + return defineTool( + 'research', + 'Search the web, read the three most independent results and return quoted passages from each. Use for current events, people, organisations, or anything you would otherwise be guessing at.', + { + type: 'object', + properties: { query: { type: 'string', description: 'The question to research' } }, + required: ['query'], + }, + async (args) => { + const query = String(args.query ?? '').trim() + if (!query) throw new Error('query must not be empty') + return researchQuestion(query, config) + }, + ) +} + /** * The network tools close over the current provider settings, so they are * rebuilt when those change. Every tool ships in every deployment: none of them @@ -147,7 +165,14 @@ export const currentTime = defineTool( * who asked not to be remembered. */ export function createBuiltinTools(config: WebAccessConfig, options: { memory?: boolean } = {}): Tool[] { - const tools = [createWebSearch(config), createReadPage(config), calculator, currentTime, weather] + const tools = [ + createWebSearch(config), + createReadPage(config), + createResearch(config), + calculator, + currentTime, + weather, + ] return options.memory === false ? tools : [...tools, memory] } diff --git a/src/tools/research.test.ts b/src/tools/research.test.ts new file mode 100644 index 0000000..4b37bad --- /dev/null +++ b/src/tools/research.test.ts @@ -0,0 +1,569 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { digest, diverseFirst, looksBlocked, paragraphsOf, passagesFor, researchQuestion } from './research' +import type { SearchResult, WebAccessConfig } from './web' + +function result(url: string, title = 'Title', snippet = 'A snippet.'): SearchResult { + return { url, title, snippet } +} + +describe('diverseFirst', () => { + it('puts the first hit on each host ahead of any second hit on one', () => { + const chosen = diverseFirst( + [ + result('https://news.example/one'), + result('https://news.example/two'), + result('https://other.example/a'), + result('https://third.example/b'), + ], + 3, + ) + + expect(chosen.map((entry) => entry.url)).toEqual([ + 'https://news.example/one', + 'https://other.example/a', + 'https://third.example/b', + ]) + }) + + it('treats www as the same host', () => { + const chosen = diverseFirst( + [result('https://www.news.example/one'), result('https://news.example/two')], + 2, + ) + + expect(chosen.map((entry) => entry.url)).toEqual([ + 'https://www.news.example/one', + 'https://news.example/two', + ]) + }) + + // Wikipedia returns every result on one host. Collapsing to a single source + // there would make the tool useless under that provider, so the list refills + // with further pages from the same host rather than being cut short. + it('fills the quota from one host when that is all there is', () => { + const chosen = diverseFirst( + [ + result('https://en.wikipedia.org/wiki/A'), + result('https://en.wikipedia.org/wiki/B'), + result('https://en.wikipedia.org/wiki/C'), + ], + 3, + ) + + expect(chosen).toHaveLength(3) + }) + + it('drops a URL the provider returned twice', () => { + const chosen = diverseFirst([result('https://news.example/one'), result('https://news.example/one')], 5) + + expect(chosen).toHaveLength(1) + }) + + it('treats company subdomains as one site', () => { + const chosen = diverseFirst( + [ + result('https://investor.nvidia.com/a'), + result('https://nvidianews.nvidia.com/b'), + result('https://www.reuters.com/c'), + ], + 2, + ) + + expect(chosen.map((entry) => entry.url)).toEqual([ + 'https://investor.nvidia.com/a', + 'https://www.reuters.com/c', + ]) + }) + + it('keeps bbc.co.uk and theguardian.co.uk as two sites', () => { + expect( + diverseFirst([result('https://www.bbc.co.uk/news/1'), result('https://www.theguardian.co.uk/2')], 2), + ).toHaveLength(2) + }) +}) + +describe('paragraphsOf', () => { + it('keeps link text and drops the address', () => { + const [paragraph] = paragraphsOf( + 'The airline was founded by [Ama Osei](https://who.example/osei) in 1974 and still flies daily.', + ) + + expect(paragraph).toContain('Ama Osei') + expect(paragraph).not.toContain('who.example') + }) + + it('drops headings, bullets and images but keeps what they introduced', () => { + const paragraphs = paragraphsOf( + '# Leadership\n\n![Image 1: portrait](https://img.example/1.png)\n\n- Ama Osei has led the airline as chief executive since March 2023.', + ) + + expect(paragraphs).toEqual(['Ama Osei has led the airline as chief executive since March 2023.']) + }) + + it('leaves out nav items and bylines, which are too short to be prose', () => { + expect(paragraphsOf('Home\n\nBy our reporter\n\nShare')).toEqual([]) + }) + + it('strips a footnote link without leaving the anchor behind as a source', () => { + // Observed on Wikipedia. `reviewAnswer` reads every URL in a tool result as a + // source the answer may cite, so a citation anchor left in a passage becomes + // a citable source that states nothing. + const [paragraph] = paragraphsOf( + 'Huang has been Nvidia\'s chief executive for three decades, a tenure described as "almost unheard of".[[47]](https://en.wikipedia.org/wiki/Jensen_Huang#cite_note-fitch20240226-50) He owns 3.6% of Nvidia.', + ) + + expect(paragraph).not.toContain('http') + expect(paragraph).not.toContain('cite_note') + // The number goes with it: a bare `47` mid-sentence reads as part of the claim. + expect(paragraph).toBe( + 'Huang has been Nvidia\'s chief executive for three decades, a tenure described as "almost unheard of". He owns 3.6% of Nvidia.', + ) + }) + + it('strips emphasis and a heading mark that survived being joined into a line', () => { + const [paragraph] = paragraphsOf( + '### Sommer-Pressekonferenz des Bundeskanzlers\nRede von Bundeskanzler Merz in _Paderborn_ über **Europa** und Deutschland.', + ) + + expect(paragraph).toBe( + 'Sommer-Pressekonferenz des Bundeskanzlers Rede von Bundeskanzler Merz in Paderborn über Europa und Deutschland.', + ) + }) + + /** + * The reader drops the spaces around its own emphasis marks, so deleting them + * fuses the words either side. Observed as `,NVIDIA Facebookpage` in a passage + * that had read `,**NVIDIA Facebook**page`. + */ + it('separates words the reader fused with emphasis rather than joining them', () => { + const [paragraph] = paragraphsOf( + 'We intend to use our **NVIDIA** Twitter account,**NVIDIA Facebook**page and company **blog** as a means of disclosing information about the company.', + ) + + expect(paragraph).toContain('account, NVIDIA Facebook page') + expect(paragraph).not.toContain('Facebookpage') + }) + + /** + * The same fusion from the other direction: a page writes two links with + * nothing between them, so removing the brackets joins what they held. + * `our@NVIDIATwitter account,NVIDIA Facebookpage` was three adjacent links. + */ + it('separates adjacent links instead of running their text together', () => { + const [paragraph] = paragraphsOf( + 'We intend to use our [@NVIDIA](https://x.example/nvidia)[Twitter](https://x.example) account,[NVIDIA Facebook](https://fb.example)page as a means of disclosing information.', + ) + + expect(paragraph).toBe( + 'We intend to use our @NVIDIA Twitter account, NVIDIA Facebook page as a means of disclosing information.', + ) + }) + + it('strips a Wikipedia link whose target carries brackets of its own', () => { + // `[^)]*` stops at the bracket inside `Betreuung_(Recht)` and leaves the + // reader's quoted title stranded in the passage. + const [paragraph] = paragraphsOf( + 'Auch [Betreuung](https://de.wikipedia.org/wiki/Betreuung_(Recht) "Betreuung (Recht)") oder Unterbringung in einem Krankenhaus würden ihn disqualifizieren.', + ) + + expect(paragraph).toBe( + 'Auch Betreuung oder Unterbringung in einem Krankenhaus würden ihn disqualifizieren.', + ) + }) + + /** + * Nav rows, breadcrumbs and share links clear every length floor and answer + * nothing. Both of these were quoted as sources for *wer ist der Bundeskanzler*. + */ + it('drops a breadcrumb and a share row, which no full stop ends', () => { + expect( + paragraphsOf('Sie befinden sich hier Bundesregierung | Startseite Bundesregierung Bundeskabinett'), + ).toEqual([]) + expect( + paragraphsOf('Governance Management Team Board of Directors Governance Documents Contact the Board'), + ).toEqual([]) + }) + + /** + * Both of these outranked the sentence that answered the question when this ran + * against the live web, and they are why the filters exist rather than being a + * precaution. + */ + it('drops a consent notice, which is prose and answers nothing', () => { + expect( + paragraphsOf( + 'These cookies may store a unique ID so that our system will remember you when you return, and are used to improve website performance.', + ), + ).toEqual([]) + expect( + paragraphsOf( + 'Das Tool verwendet Cookies. Mit diesen Cookies können wir Besuche zählen und die Nutzung der Seite auswerten.', + ), + ).toEqual([]) + }) + + it('drops a citation list, which repeats the subject in every entry', () => { + expect( + paragraphsOf( + '"Here\'s how Nvidia CEO Jensen Huang won over his wife". Business Insider. Retrieved December 24, 2024. ↑ "#61 Jen-Hsun Huang". Forbes. Archived from the original on May 9, 2008.', + ), + ).toEqual([]) + }) + + it('keeps prose that merely mentions a year and a source', () => { + const kept = paragraphsOf( + 'Jensen Huang founded NVIDIA in 1993 and has served since its inception as president and chief executive officer.', + ) + + expect(kept).toHaveLength(1) + }) + + it('joins the lines of one paragraph and separates two', () => { + expect( + paragraphsOf( + 'Ama Osei has led the airline as its chief\nexecutive since March 2023.\n\nThe airline was founded in 1974 and is based in Accra, Ghana.', + ), + ).toEqual([ + 'Ama Osei has led the airline as its chief executive since March 2023.', + 'The airline was founded in 1974 and is based in Accra, Ghana.', + ]) + }) +}) + +describe('looksBlocked', () => { + // The reader answers 200 with these, so nothing upstream can tell them from a + // page. One was quoted as a source for *wer ist der Bundeskanzler*. + it('recognises a firewall page the reader returned as a result', () => { + expect( + looksBlocked( + 'Sucuri WebSite Firewall - Access Denied', + 'Access Denied - Sucuri Website Firewall\n\nTime:2026-08-25 17:56:05 Server ID:20017', + ), + ).toBe(true) + }) + + it('recognises a JavaScript gate', () => { + expect(looksBlocked('Just a moment...', 'Enable JavaScript and cookies to continue')).toBe(true) + }) + + /** Length is half the test, or an article about Cloudflare would be discarded. */ + it('leaves a long article that happens to name a firewall vendor', () => { + const article = `Cloudflare reported revenue growth this quarter. ${'The company operates a global network. '.repeat(40)}` + + expect(looksBlocked('Cloudflare earnings', article)).toBe(false) + }) + + it('leaves an ordinary short page alone', () => { + expect(looksBlocked('Leadership', 'Ama Osei has led the airline since 2023.')).toBe(false) + }) +}) + +describe('passagesFor', () => { + /** + * As many paragraphs as a real page has, and that is the point rather than the + * prose: the term weights are measured over them. A four-paragraph fixture + * makes `the` exactly as rare as `executive` and ranks the wrong paragraph + * first, which is what pooling the weights across everything fetched fixes. + */ + const PAGE = [ + 'Skip to content and sign up for the newsletter so you never miss an update from us.', + 'The airline was founded in 1974 in Accra and now flies to thirty destinations.', + 'Ama Osei was appointed chief executive in March 2023, succeeding Piet Hendriks.', + 'The cabin refurbishment programme was completed across the whole fleet last summer.', + 'The company reported a modest profit for the financial year ending in December 2025.', + 'Cookies help the site deliver its services, and you may choose to accept all of them.', + 'The head office moved to a new building near the airport in the early part of 2024.', + 'Passengers may check two bags on the international routes without paying a fee.', + ].join('\n\n') + + function forOnePage(question: string, markdown: string): string[] { + return passagesFor(question, [paragraphsOf(markdown)])[0] ?? [] + } + + it('picks the paragraph that answers the question over one that merely shares its words', () => { + const [first] = forOnePage('Who is the chief executive of the airline?', PAGE) + + expect(first).toContain('Ama Osei') + }) + + it('returns at most two passages, however long the page is', () => { + expect(forOnePage('airline chief executive founded Accra 1974', PAGE)).toHaveLength(2) + }) + + it('gives every page its own list, in the order the sources were selected', () => { + const chosen = passagesFor('chief executive', [paragraphsOf(PAGE), [], paragraphsOf(PAGE)]) + + // The middle page is one that could not be read. Dropping it here would + // shift every source after it onto the wrong URL. + expect(chosen.map((passages) => passages.length)).toEqual([1, 0, 1]) + }) + + it('has nothing to say about a page with no prose on it', () => { + expect(forOnePage('who runs it', 'Home\n\nContact\n\nShare')).toEqual([]) + }) + + /** + * A definition is answered by a lead paragraph that repeats none of the words + * in the question, so a page whose score is zero everywhere still contributes + * its opening rather than going silent. + */ + it('falls back to the opening paragraph when nothing matches', () => { + const [first] = forOnePage( + 'zzz qqq', + 'Stripe is a payments company that builds financial infrastructure for online businesses.', + ) + + expect(first).toBe( + 'Stripe is a payments company that builds financial infrastructure for online businesses.', + ) + }) + + /** + * The densest window is often a heading. Asked who runs Nvidia, this returned + * the FAQ question "Who leads NVIDIA?" — every word of it earning, and the + * answer underneath it left out. + */ + it('does not shrink a passage to a heading just because it scores densely', () => { + const faq = [ + 'Who leads NVIDIA?', + 'Jensen Huang founded NVIDIA in 1993 and has served since its inception as president and chief executive officer of the company.', + 'Who is part of the NVIDIA executive team?', + 'The executive staff includes Colette Kress as chief financial officer and Debora Shoquist in operations, alongside several others.', + ].join(' ') + + const [first] = forOnePage('Who leads NVIDIA?', faq) + + expect(first!.length).toBeGreaterThanOrEqual(60) + expect(first).toContain('Jensen Huang') + }) + + it('cuts a long paragraph down to the sentences that carry the question', () => { + const filler = 'The catering was reviewed in a report nobody read. '.repeat(6) + const [first] = forOnePage( + 'Who was appointed chief executive?', + `${filler}Ama Osei was appointed chief executive in March 2023.${filler}`, + ) + + expect(first).toContain('Ama Osei was appointed chief executive in March 2023.') + expect(first!.length).toBeLessThanOrEqual(280) + }) + + it('cuts a single sentence that is longer than the cap on its own', () => { + const [first] = forOnePage('chief executive', `Ama Osei ${'and others '.repeat(60)}is chief executive.`) + + expect(first!.length).toBeLessThanOrEqual(281) + expect(first).toContain('…') + }) + + it('does not quote the same claim twice', () => { + const twice = [ + 'Ama Osei was appointed chief executive of the airline in March 2023 after nine years.', + 'Ama Osei was appointed chief executive of the airline in March 2023 after nine years.', + ].join('\n\n') + + expect(forOnePage('who is the chief executive', twice)).toHaveLength(1) + }) +}) + +describe('digest', () => { + const sources = [ + { + url: 'https://fictionalairways.example/leadership', + title: 'Leadership', + passages: ['Ama Osei has led Fictional Airways as chief executive since 2023.'], + read: true, + }, + { + url: 'https://airtimes.example/osei', + title: 'Airtimes', + passages: ['The board appointed Ama Osei in March 2023.'], + read: true, + }, + ] + + it('numbers each source and puts its URL on the same line as its name', () => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-08-26T15:00:00')) + + expect(digest('Who runs Fictional Airways?', sources)).toBe( + [ + 'Researched 2026-08-26 for "Who runs Fictional Airways?" across 2 sources, all read in full.', + '', + '1. Leadership — https://fictionalairways.example/leadership', + ' "Ama Osei has led Fictional Airways as chief executive since 2023."', + '2. Airtimes — https://airtimes.example/osei', + ' "The board appointed Ama Osei in March 2023."', + ].join('\n'), + ) + + vi.useRealTimers() + }) + + // A snippet is weaker evidence than a page, and once both are quoted lines in + // a list nothing else says which is which. + it('says how many sources are snippets rather than pages', () => { + const mixed = [sources[0]!, { ...sources[1]!, read: false }] + + expect(digest('Who runs it?', mixed)).toContain( + 'across 2 sources; 1 read in full, 1 from the search snippet only', + ) + }) + + it('warns when no page could be opened at all', () => { + const snippets = sources.map((source) => ({ ...source, read: false })) + + expect(digest('Who runs it?', snippets)).toContain('none could be opened, so these are search snippets') + }) + + it('caps the whole result, because a page-length digest is what this avoids', () => { + const long = Array.from({ length: 5 }, (_, at) => ({ + url: `https://example.com/${at}`, + title: 'T'.repeat(400), + passages: ['x'.repeat(600), 'y'.repeat(600)], + read: true, + })) + + const built = digest('anything', long) + + expect(built).toContain('[Truncated: further sources were dropped.]') + expect(built.length).toBeLessThan(4_100) + }) + + it('does not double the quotes around a passage that arrived with them', () => { + const quoted = [{ ...sources[0]!, passages: ['"Already quoted."'] }] + + expect(digest('q', quoted)).toContain(' "Already quoted."') + }) +}) + +/** + * The whole call, over a stubbed network: LangSearch answers the search and the + * reader answers each page, which is the shape every provider reduces to. + */ +describe('researchQuestion', () => { + const config: WebAccessConfig = { provider: 'langsearch', langsearchApiKey: 'key' } + const question = 'Who is the chief executive of Fictional Airways?' + + const LEADERSHIP = 'https://fictionalairways.example/leadership' + const AIRTIMES = 'https://airtimes.example/osei' + + const HITS = [ + { name: 'Leadership', url: LEADERSHIP, snippet: 'Ama Osei leads the airline.' }, + { name: 'Airtimes', url: AIRTIMES, snippet: 'The board appointed Ama Osei in 2023.' }, + ] + + /** A page body, or the status the reader should refuse it with. */ + type Reply = string | number + + function stubNetwork(pages: Record, hits = HITS) { + const calls: string[] = [] + + vi.stubGlobal( + 'fetch', + vi.fn(async (input: string) => { + const url = String(input) + calls.push(url) + + if (url.startsWith('https://api.langsearch.com')) { + return { ok: true, status: 200, json: async () => ({ data: { webPages: { value: hits } } }) } + } + + const target = url.replace('https://r.jina.ai/', '') + const reply = pages[target] + if (reply === undefined || typeof reply === 'number') { + return { ok: false, status: typeof reply === 'number' ? reply : 404 } + } + return { + ok: true, + status: 200, + json: async () => ({ data: { title: 'T', url: target, content: reply } }), + } + }), + ) + + return calls + } + + afterEach(() => { + vi.unstubAllGlobals() + vi.useRealTimers() + }) + + it('quotes every source it could read, each against its own URL', async () => { + stubNetwork({ + [LEADERSHIP]: 'Ama Osei has led Fictional Airways as chief executive since March 2023 in Accra.', + [AIRTIMES]: 'The board appointed Ama Osei as chief executive in March 2023, succeeding Piet Hendriks.', + }) + + const result = await researchQuestion(question, config) + + expect(result).toContain('across 2 sources, all read in full') + expect(result).toContain(`1. T — ${LEADERSHIP}`) + expect(result).toContain( + '"Ama Osei has led Fictional Airways as chief executive since March 2023 in Accra."', + ) + expect(result).toContain(`2. T — ${AIRTIMES}`) + expect(result).toContain('succeeding Piet Hendriks') + }) + + it('reads the pages at the same time rather than one after another', async () => { + const calls = stubNetwork({ + [LEADERSHIP]: 'Ama Osei has led Fictional Airways as chief executive since March 2023 in Accra.', + [AIRTIMES]: 'The board appointed Ama Osei as chief executive in March 2023, succeeding Piet Hendriks.', + }) + + await researchQuestion(question, config) + + // One search and one read per source: the cost of the call is what the + // reader's per-minute budget is spent on, so it is worth pinning. + expect(calls).toHaveLength(3) + }) + + /** + * The reader's budget is shared between search and every page, so a 429 on one + * source is an ordinary event. Losing the two that arrived over it would turn a + * partial answer into no answer. + */ + it('stands a page that would not open in as its search snippet', async () => { + stubNetwork({ + [LEADERSHIP]: 'Ama Osei has led Fictional Airways as chief executive since March 2023 in Accra.', + [AIRTIMES]: 429, + }) + + const result = await researchQuestion(question, config) + + expect(result).toContain('1 read in full, 1 from the search snippet only') + expect(result).toContain('"The board appointed Ama Osei in 2023."') + }) + + it('says so when no page opened and only snippets are left', async () => { + stubNetwork({ [LEADERSHIP]: 500, [AIRTIMES]: 429 }) + + const result = await researchQuestion(question, config) + + expect(result).toContain('none could be opened, so these are search snippets only') + }) + + /** + * Nothing readable and nothing to quote is a failure, not an empty result. A + * 0.8B model relays "no sources" as "there is nothing on this subject", which + * is the one answer that must never come out of a page that simply would not + * load. + */ + it('fails rather than reporting that it found nothing', async () => { + stubNetwork({ [LEADERSHIP]: 500 }, [{ name: 'Leadership', url: LEADERSHIP, snippet: '' }]) + + await expect(researchQuestion(question, config)).rejects.toThrow('could not read any of them') + }) + + it('reports a search that genuinely matched nothing as such', async () => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-08-26T15:00:00')) + stubNetwork({}, []) + + await expect(researchQuestion(question, config)).resolves.toBe( + `Researched 2026-08-26 for "${question}". No results.`, + ) + vi.useRealTimers() + }) +}) diff --git a/src/tools/research.ts b/src/tools/research.ts new file mode 100644 index 0000000..9d7b6ef --- /dev/null +++ b/src/tools/research.ts @@ -0,0 +1,523 @@ +/** + * Researching a question across several sources in one tool call. + * + * `web_search` and `read_page` can do this between them, and the model has to + * chain them to get there: search, pick a result, read it, and — if it wants a + * second opinion — read another. A turn is capped at `MAX_TOOL_ROUNDS` rounds, + * so that chain runs out of budget at roughly two sources, and every link in it + * is a decision a 0.8B model can get wrong. This is the `weather` shape applied + * to the web: the fan-out happens here, and what reaches the model is one + * compact result it did not have to assemble. + * + * Three sources, not five: search plus three page-reads would spend four of the + * reader's 20 requests a minute, and Wikipedia pages skip the reader entirely, + * so a typical call is one search and two reads. Five sources was six requests + * and three questions in a minute before the rate limit. + * + * The pages are read in full and quoted in part. Selection is lexical: paragraphs + * are scored by the question's terms, each weighted by how rare it is across + * every paragraph the turn fetched. + */ + +import { readPage, searchWeb, type SearchResult, type WebAccessConfig } from './web' + +/** + * How many results to ask for before narrowing them. Larger than `MAX_SOURCES` + * because the narrowing drops duplicate sites, and a page of results from one + * newspaper should still leave three sources to read. + */ +const SEARCH_LIMIT = 8 + +/** Three independent sites. A fourth is usually the same claim from a mirror. */ +const MAX_SOURCES = 3 + +/** Two passages carry a claim and its context. A third is usually the same claim again. */ +const MAX_PASSAGES_PER_SOURCE = 2 + +const MAX_PASSAGE_CHARS = 280 + +/** Shorter than this is a heading, a byline or a nav item rather than prose. */ +const MIN_PASSAGE_CHARS = 60 + +/** Words, not characters: a long run of link text can clear the character floor. */ +const MIN_PASSAGE_WORDS = 8 + +/** + * The whole result, well under `read_page`'s 8,000. Three short sources cost + * less context than one whole page, which is the failure this tool exists to + * avoid. + */ +const MAX_DIGEST_CHARS = 4000 + +const WORD = /[\p{L}\p{N}]+/gu + +function words(text: string): string[] { + return [...text.toLowerCase().matchAll(WORD)].map((match) => match[0]) +} + +function collapse(value: string): string { + return value.replace(/\s+/g, ' ').trim() +} + +/** + * Closes the gap the separators above open in front of punctuation. A quote is + * only known to be a closing one when punctuation follows it, so that is the + * only case where a space in front of one is removed. + */ +function tidy(value: string): string { + return value.replace(/\s+(["'”’][.,;:!?])/g, '$1').replace(/\s+([.,;:!?])/g, '$1') +} + +function hostOf(url: string): string { + try { + return new URL(url).hostname.replace(/^www\./, '').toLowerCase() + } catch { + return url + } +} + +/** + * Compound public suffixes where the last two labels are not the site. + * + * `bbc.co.uk` and `theguardian.co.uk` are two newsrooms. Without this list they + * collapse to `co.uk`. The list is the suffixes that actually show up in search + * results, not the public suffix list. + */ +const COMPOUND_SUFFIX = new Set([ + 'co.uk', + 'org.uk', + 'ac.uk', + 'gov.uk', + 'com.au', + 'net.au', + 'org.au', + 'co.nz', + 'co.jp', + 'co.kr', + 'com.br', + 'co.in', + 'com.mx', + 'co.za', + 'com.tr', + 'com.ar', +]) + +/** + * The registrable site, so `investor.nvidia.com` and `nvidianews.nvidia.com` + * count as one source rather than two readings of the same company. + */ +export function siteOf(url: string): string { + const host = hostOf(url) + const parts = host.split('.').filter(Boolean) + if (parts.length <= 2) return host + const lastTwo = parts.slice(-2).join('.') + if (COMPOUND_SUFFIX.has(lastTwo)) return parts.slice(-3).join('.') + return lastTwo +} + +function todayStamp(): string { + const now = new Date() + const month = String(now.getMonth() + 1).padStart(2, '0') + const day = String(now.getDate()).padStart(2, '0') + return `${now.getFullYear()}-${month}-${day}` +} + +/** + * Orders the results so that the first hit on each *site* comes before any + * second hit on one, then takes the first `max`. + * + * "Many sources" has to mean many *different* sources: a search for a news story + * returns four pages of the same newspaper, and reading all four is four reader + * requests spent to hear one newsroom repeat itself. Grouping by site rather + * than by host is what stops `investor.nvidia.com` and `nvidianews.nvidia.com` + * counting as two opinions. Reordering rather than discarding is what keeps this + * from being a special case for Wikipedia, whose results are all one site — + * there the list refills with further articles instead of collapsing to one. + */ +export function diverseFirst(results: SearchResult[], max: number): SearchResult[] { + const sites = new Set() + const urls = new Set() + const first: SearchResult[] = [] + const rest: SearchResult[] = [] + + for (const result of results) { + if (!result.url || urls.has(result.url)) continue + urls.add(result.url) + + const site = siteOf(result.url) + if (sites.has(site)) { + rest.push(result) + continue + } + sites.add(site) + first.push(result) + } + + return [...first, ...rest].slice(0, max) +} + +const IMAGE = /!\[[^\]]*\]\([^)]*\)/g + +/** + * A markdown link target, tolerating one level of nesting inside it. + * + * `[^)]*` is the obvious pattern and stops at the first bracket of + * `Betreuung_(Recht)`, which left `"Betreuung (Recht)")` sitting in a German + * Wikipedia passage. Wikipedia URLs carry parenthesised disambiguators and the + * reader adds a quoted title beside them, so both have to survive being matched. + */ +const LINK_TARGET = /\]\((?:[^()]|\([^()]*\))*\)/g + +/** What is left of `[47]` once its target is gone: a footnote number mid-sentence. */ +const FOOTNOTE = /\[\d+\]/g + +const BARE_URL = /https?:\/\/\S+/g +const HEADING_MARK = /#{1,6}\s+/g +const EMPHASIS = /[*_`]+/g +const LINE_FURNITURE = /^\s*(?:[-*+]\s+|>\s+|\d+\.\s+)/ + +/** + * Prose ends in a full stop. Menus, breadcrumbs and share rows do not, and they + * clear every length floor: *Sie befinden sich hier Bundesregierung | Startseite + * Bundeskabinett Bundeskanzler* was quoted as a source on what a chancellor is. + * + * Cheaper and less parochial than naming the furniture — it needs no word list + * and works in either language. Trailing quotes and brackets are allowed through + * because a paragraph often ends inside them. + */ +const ENDS_A_SENTENCE = /[.!?…][)"'”’]*$/ + +/** + * Boilerplate that reads exactly like prose and answers nothing. + * + * Not a nicety. Run against the live web, two of five sources for *who is the + * chief executive of Nvidia* came back quoting consent notices — "these cookies + * may store a unique ID", "das Tool verwendet Cookies" — because a cookie banner + * is several sentences long and mentions the site it is on, which is all the + * scoring has to go on. + * + * The cost is that a paragraph genuinely about cookies or a privacy policy is + * dropped with them. That is the right way round: this tool is asked who runs a + * company far more often than it is asked what an HTTP cookie is, and the + * failure it prevents was happening on most commercial sites. + */ +const BOILERPLATE = + /\bcookies?\b|\bconsent\b|\bnewsletter\b|\bsubscribe\b|\bprivacy policy\b|\bterms of (use|service)\b|\ball rights reserved\b|\bdatenschutz\b|\beinwilligung\b|\bnutzungsbedingungen\b/i + +/** + * A citation list, which scores well and states nothing. + * + * Wikipedia's references were the top-ranked paragraph for *who is the chief + * executive of Nvidia*: they repeat the subject's name in every entry, so they + * out-score the sentence that answers the question. + */ +const REFERENCE_LIST = + /↑|\bretrieved\b\s+\w+\s+\d{1,2},?\s+\d{4}|\barchived from the original\b|\babgerufen am\b/i + +/** + * Reads the reader's markdown back into candidate paragraphs. + * + * Link text is kept and the target dropped, and any bare URL goes with it. That + * second part is not tidiness: `reviewAnswer` treats every URL in a tool result + * as a source the answer may cite, so a footnote anchor left in a passage would + * become a citable source that says nothing. A Wikipedia passage arrived carrying + * `#cite_note-fitch20240226-50`, which is exactly that. + */ +export function paragraphsOf(markdown: string): string[] { + return ( + markdown + .replace(IMAGE, ' ') + .replace(LINK_TARGET, ']') + .replace(FOOTNOTE, '') + // Whatever brackets are left were a link's text. A nested `[[47]](url)` is why + // this strips them rather than matching a whole link in one pattern. + // + // A space, not nothing, and the same for the emphasis marks below: a page + // writes two links with nothing between them, so deleting the brackets fuses + // what they held. `our@NVIDIATwitter account,NVIDIA Facebookpage` was three + // adjacent links, and it is the trap `unbold` in `web.ts` already documents. + .replace(/[[\]]/g, ' ') + .replace(BARE_URL, ' ') + .split(/\n\s*\n/) + .map((block) => + tidy( + collapse( + block + .split('\n') + .map((line) => line.replace(LINE_FURNITURE, '')) + .join(' ') + .replace(HEADING_MARK, ' ') + .replace(EMPHASIS, ' '), + ), + ), + ) + .filter( + (block) => + block.length >= MIN_PASSAGE_CHARS && + words(block).length >= MIN_PASSAGE_WORDS && + ENDS_A_SENTENCE.test(block) && + !BOILERPLATE.test(block) && + !REFERENCE_LIST.test(block), + ) + ) +} + +/** + * What a firewall or a JavaScript gate serves instead of the page. + * + * The reader answers 200 with it, so nothing upstream can tell it from a result: + * *wer ist der Bundeskanzler* came back with "Sucuri WebSite Firewall — Access + * Denied" quoted as one of five sources. Length is half the test, because an + * article about Cloudflare is long and a page refusing to serve one is not. + */ +const BLOCKED = + /access denied|attention required|just a moment|enable javascript|are you a robot|verify you are human|cloudflare|sucuri|forbidden|zugriff verweigert/i + +const BLOCK_PAGE_CHARS = 1200 + +export function looksBlocked(title: string, text: string): boolean { + return text.length < BLOCK_PAGE_CHARS && BLOCKED.test(`${title} ${text.slice(0, 300)}`) +} + +/** BM25's idf without the length normalisation, as `skills/retrieve.ts` uses it. */ +function inverseFrequency(documentFrequency: number, total: number): number { + return Math.log((total - documentFrequency + 0.5) / (documentFrequency + 0.5) + 1) +} + +/** + * What each word of the question is worth, measured over every paragraph the + * turn fetched rather than over one page at a time. + * + * Pooling is what makes a stop list unnecessary, and it has to be pooled to + * work: across a hundred paragraphs *the* appears in nearly all of them and ends + * up worth about two per cent of *executive*, but within a single four-paragraph + * page it can be exactly as rare and score just as high. Measured on one page, + * "who is the chief executive of the airline" ranked the paragraph containing + * *the airline* level with the one naming the chief executive. + * + * A length floor would be the cheap way to drop *of* and *is*, and it is the + * wrong one: *UN*, *EU* and *AI* are two letters and are the whole question. + */ +function weigh(question: string, corpus: string[]): Map { + const terms = new Set(words(question).filter((term) => term.length > 1)) + const tokenized = corpus.map((paragraph) => new Set(words(paragraph))) + + const weights = new Map() + for (const term of terms) { + const seen = tokenized.filter((paragraph) => paragraph.has(term)).length + if (seen > 0) weights.set(term, inverseFrequency(seen, tokenized.length)) + } + return weights +} + +function score(text: string, weights: Map): number { + let total = 0 + for (const term of new Set(words(text))) total += weights.get(term) ?? 0 + return total +} + +const SENTENCE_END = /(?<=[.!?…])\s+/ + +/** + * Cuts a paragraph down to the run of sentences that carries the most of the + * question, so a long page contributes its relevant lines rather than its first + * ones. A single sentence over the cap is cut mid-way, which is the one case + * where this cannot avoid it. + * + * A window has to clear `MIN_PASSAGE_CHARS` to win on score alone, because + * scoring rewards density and the densest window is often a heading. Asked who + * runs Nvidia, this returned the FAQ question "Who leads NVIDIA?" — every word of + * it earning, and the answer beneath it left out. + */ +function condense(paragraph: string, weights: Map): string { + if (paragraph.length <= MAX_PASSAGE_CHARS) return paragraph + + const sentences = paragraph.split(SENTENCE_END) + let best = '' + let bestScore = -1 + let substantial = '' + let substantialScore = -1 + + for (let start = 0; start < sentences.length; start += 1) { + let window = '' + for (let end = start; end < sentences.length; end += 1) { + const extended = window ? `${window} ${sentences[end]}` : (sentences[end] ?? '') + if (extended.length > MAX_PASSAGE_CHARS) break + window = extended + + const windowScore = score(window, weights) + if (windowScore > bestScore || (windowScore === bestScore && window.length > best.length)) { + best = window + bestScore = windowScore + } + if ( + window.length >= MIN_PASSAGE_CHARS && + (windowScore > substantialScore || + (windowScore === substantialScore && window.length > substantial.length)) + ) { + substantial = window + substantialScore = windowScore + } + } + } + + return substantial || best || `${paragraph.slice(0, MAX_PASSAGE_CHARS).trimEnd()}…` +} + +/** Enough of a passage to recognise the same claim written out twice. */ +function fingerprint(passage: string): string { + return words(passage).slice(0, 8).join(' ') +} + +/** + * The passages from one page worth putting in front of the model. + * + * A page whose prose never repeats the question's words still falls back to its + * opening paragraph: *what is Stripe* is answered by a lead paragraph that says + * "Stripe is a payments company" and may never say "what" or "is" again. + */ +function choose(candidates: string[], weights: Map): string[] { + if (candidates.length === 0) return [] + + const ranked = candidates + .map((text, at) => ({ text, at, score: score(text, weights) })) + .sort((a, b) => b.score - a.score || a.at - b.at) + + const relevant = ranked.filter((candidate) => candidate.score > 0) + const passages: string[] = [] + const seen = new Set() + + for (const candidate of relevant.length > 0 ? relevant : ranked.slice(0, 1)) { + if (passages.length >= MAX_PASSAGES_PER_SOURCE) break + const passage = condense(candidate.text, weights) + const mark = fingerprint(passage) + if (mark === '' || seen.has(mark)) continue + seen.add(mark) + passages.push(passage) + } + + return passages +} + +/** + * Picks the passages for every page at once, so each is ranked against the same + * weights. Pages arrive as paragraph lists and leave as passage lists, index for + * index; a page nothing could be read off stays empty rather than being dropped, + * because its position still names a source. + */ +export function passagesFor(question: string, pages: string[][]): string[][] { + const weights = weigh(question, pages.flat()) + return pages.map((candidates) => choose(candidates, weights)) +} + +export interface Source { + url: string + title: string + /** Verbatim, best first. */ + passages: string[] + /** False when the page could not be opened and its search snippet stood in. */ + read: boolean +} + +/** Straight quotes wrap each passage, so the model is not handed its own edges to trip on. */ +function unquote(passage: string): string { + return passage.replace(/^["'“”]+|["'“”]+$/g, '').trim() +} + +function entry(source: Source, at: number): string { + const heading = `${at + 1}. ${source.title || hostOf(source.url)} — ${source.url}` + return [heading, ...source.passages.map((passage) => ` "${unquote(passage)}"`)].join('\n') +} + +/** + * Says what was consulted before quoting any of it. + * + * A source that could only be reached as a search snippet is named as one: a + * snippet is weaker evidence than a page, and the difference is invisible once + * both are quoted lines in a list. + */ +function header(question: string, sources: Source[]): string { + const read = sources.filter((source) => source.read).length + const subject = `Researched ${todayStamp()} for "${question}" across ${sources.length} source${sources.length === 1 ? '' : 's'}` + + if (read === 0) return `${subject}; none could be opened, so these are search snippets only.` + if (read === sources.length) return `${subject}, all read in full.` + return `${subject}; ${read} read in full, ${sources.length - read} from the search snippet only.` +} + +export function digest(question: string, sources: Source[]): string { + const body = [header(question, sources), '', ...sources.map(entry)].join('\n') + if (body.length <= MAX_DIGEST_CHARS) return body + return `${body.slice(0, MAX_DIGEST_CHARS)}\n\n[Truncated: further sources were dropped.]` +} + +function reasonsFrom(settled: PromiseSettledResult[]): string { + return settled + .flatMap((outcome) => + outcome.status === 'rejected' + ? [outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason)] + : [], + ) + .join('; ') +} + +/** + * Searches, reads the most promising results in parallel and returns the + * passages that bear on the question, each with the URL it came from. + * + * A page that fails is replaced by its search snippet rather than allowed to + * take the answer down with it — the reader's per-minute budget is shared, so a + * 429 on the third page is an ordinary event and not a reason to abandon the + * two that arrived. + */ +export async function researchQuestion(question: string, config: WebAccessConfig): Promise { + const results = await searchWeb(question, SEARCH_LIMIT, config) + if (results.length === 0) return `Researched ${todayStamp()} for "${question}". No results.` + + const selected = diverseFirst(results, MAX_SOURCES) + const settled = await Promise.allSettled(selected.map((result) => readPage(result.url, config))) + + const chosen = passagesFor( + question, + settled.map((outcome) => { + if (outcome.status !== 'fulfilled') return [] + const { title, text } = outcome.value + // A page the site refused to serve arrives as a 200 with prose on it. Left + // in, it is a source that says nothing and cannot be told from one that + // does; the search snippet for the same URL at least came from the index. + return looksBlocked(title, text) ? [] : paragraphsOf(text) + }), + ) + + const sources = selected.flatMap((result, at): Source[] => { + const outcome = settled[at] + const passages = chosen[at] ?? [] + + if (outcome?.status === 'fulfilled' && passages.length > 0) { + return [ + { + url: outcome.value.url, + title: outcome.value.title || result.title, + passages, + read: true, + }, + ] + } + + const snippet = collapse(result.snippet) + if (!snippet) return [] + return [{ url: result.url, title: result.title, passages: [snippet], read: false }] + }) + + // Every source silent means the search found pages and nothing could be read + // off any of them. Reporting that as a result would have the model relay it as + // "there is nothing on this", which is the one thing it must not say. + if (sources.length === 0) { + const reasons = reasonsFrom(settled) + throw new Error( + `Found ${results.length} results for "${question}" but could not read any of them${reasons ? `: ${reasons}` : '.'}`, + ) + } + + return digest(question, sources) +}