From 21d6f8f4262bda4b5a254986fd4c276e970f2c5b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 25 Aug 2026 21:54:39 +0000 Subject: [PATCH 1/2] Add a research tool that answers from several sources at once Answering a question from the web took the model three of its four tool rounds: search, choose a result, read it, and read another for a second opinion. In practice it stopped at one source, and each link in the chain was a decision a 0.8B model could get wrong. `research` is that chain folded into one call, the way `weather` folds three forecast services. It searches, reorders the results so the first hit on each host comes first, fetches up to five in parallel, and returns two quoted passages from each with the URL it came from. Most of what it reads is deliberately thrown away. Function-calling accuracy falls as tool responses grow, and five pages at `read_page`'s cap would be 40,000 characters, so passages are selected by scoring paragraphs against the question with each term weighted by how rare it is across everything the call fetched. Those weights have to be pooled: measured per page, "who is the chief executive of the airline" ranked the paragraph saying "the airline" level with the one naming the executive, because within four paragraphs `the` is as rare as `executive`. A page that will not open falls back to its search snippet and the header says how many did, since the reader's per-minute budget is shared and a 429 on the fourth source is ordinary. A call that can read none of them throws rather than reporting an empty result, which the model would relay as "there is nothing on this subject". The answer check gains `single-source`, which fires when three or more sites were returned and the reply rests on one. Without it the extra sources are five reader requests spent on nothing. It stays shy: three sites are required, because two is what search-then-read produces, and it counts hosts so two pages of one newspaper are one source. Co-authored-by: Sebastian --- README.md | 41 ++- src/agent/review.test.ts | 108 ++++++++ src/agent/review.ts | 53 +++- src/components/MessageItem.tsx | 1 + src/eval/scenarios.ts | 63 ++++- src/lib/tool-labels.test.ts | 7 + src/lib/tool-labels.ts | 1 + src/skills/library.test.ts | 7 +- src/skills/research-question/SKILL.md | 26 +- src/tools/builtins.test.ts | 23 ++ src/tools/builtins.ts | 53 +++- src/tools/research.test.ts | 379 ++++++++++++++++++++++++++ src/tools/research.ts | 361 ++++++++++++++++++++++++ 13 files changed, 1094 insertions(+), 29 deletions(-) create mode 100644 src/tools/research.test.ts create mode 100644 src/tools/research.ts diff --git a/README.md b/README.md index fd9d169..cc988cb 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A chat agent that runs its language model **inside your browser**. Qwen3.5-0.8B is executed on your own GPU through WebGPU, so there is no API key, no per-token cost, and no conversation sent to a model provider. Install it once and it keeps working offline. -The agent can search, read pages, calculate exactly, remember things you tell it, and call any MCP server you connect. Because a 0.8B model needs the help, common requests are routed through [skills](#skills) that show it a worked example rather than telling it what to do. +The agent can research a question across several sources at once, search, read pages, calculate exactly, remember things you tell it, and call any MCP server you connect. Because a 0.8B model needs the help, common requests are routed through [skills](#skills) that show it a worked example rather than telling it what to do. There is no backend. Not "a backend you can skip" — the project ships no server code at all, and `pnpm build` produces a directory of static files that needs nothing but a web server to host it. That is why the deployed site above has the full tool set rather than a reduced one. @@ -181,6 +181,7 @@ Three details make the app work from a repository sub-path rather than a domain | Tool | What it does | | -------------- | ------------------------------------------------------------------- | +| `research` | Answers a question from several sources at once, quoting each. | | `web_search` | Full web search with no key; Wikipedia, LangSearch or Jina instead. | | `read_page` | Fetches a URL and returns its readable text. | | `calculator` | Exact arithmetic via a hand-written parser. | @@ -207,7 +208,7 @@ The default takes no key and no signup: `r.jina.ai` is pointed at a DuckDuckGo r That fragility has already been paid once, and not in the way it looked. `lite.duckduckgo.com` was the only page asked, and one afternoon the reader could not load it: it waited on the page and returned a 422, so the default provider could not search at all. The html page answered the same query in the same second, and an hour later both were fine. So the fault was never that one page died — it is that one page is enough for the search to work and not enough for it to keep working. Both are asked now, `duckduckgo.com/html/` first because it is what answered during the outage. They write a hit differently, `1.[Title](link)` against `## [Title](link)`, and the parser reads both. -Search and `read_page` share the reader's budget of 20 requests a minute per IP, so one search plus one page read spends two. A Jina key raises the ceiling for both and is what the Jina provider needs outright. +Search and `read_page` share the reader's budget of 20 requests a minute per IP, so one search plus one page read spends two — and a [`research`](#researching-a-question-across-several-sources) call spends six. A Jina key raises the ceiling for both and is what the Jina provider needs outright. **LangSearch is the way off that shared budget without paying for one.** `api.langsearch.com` is a search API rather than a results page, its free tier allows 1,000 searches a day and one a second, and a key needs no card — so a search stops competing with `read_page` for the same 20 requests a minute. Two things about it are worth knowing before choosing it. Its snippets are index text rather than prose, lower-cased and with spaces around the punctuation, which a 0.8B model reads less confidently than a sentence. And it answers in an envelope: a refusal it decides to report with a 200 arrives as a `msg` and no result set, so `searchLangSearch` raises that rather than passing an empty list to a model that would relay it as "this does not exist". Long summaries are available per result and are switched off — each is the whole page behind the result, which would leave a 0.8B context with no room for the answer. @@ -215,6 +216,37 @@ The tool description changes with the provider, so the model is told whether it Keys are entered at runtime and kept in `localStorage`. None of this reads a build-time environment variable, deliberately: a key compiled into the bundle is a key published to every visitor. +### Researching a question across several sources + +`web_search` and `read_page` can answer a question between them, and the model has to chain them to get there: search, choose a result, open it, and — if it wants a second opinion — open another. That chain is four decisions and at least three of the [four tool rounds](#how-it-works) a turn has, which in practice means one source and no corroboration. `research` is the same work with the chaining taken out of the conversation: one call, one question, and what comes back is several sources already read. + +It is the [`weather` tool's](#tools) shape applied to the web, and it exists for the same reason. `src/tools/research.ts` searches, picks up to **five sources**, fetches them in parallel, and returns two quoted passages from each with the URL it came from: + +```text +Researched "Who is the chief executive of Fictional Airways?" across 5 sources, all read in full. + +1. Leadership — https://fictionalairways.example/leadership + "Ama Osei has led Fictional Airways as chief executive since 2023." +2. Fictional Airways names new chief — https://airtimes.example/osei-appointed + "The board appointed Ama Osei in March 2023, succeeding Piet Hendriks." +``` + +**More of the web reaching the model would make the answers worse, so most of it does not.** Function-calling accuracy falls by between 7% and 91% as tool responses grow ([arXiv:2505.10570](https://arxiv.org/html/2505.10570)), and five pages at `read_page`'s cap would be 40,000 characters — an order of magnitude more than this model can answer out of. So the pages are read in full and quoted in part, and the whole result is capped at 4,000 characters. Five sources cost half of what one whole page does. + +Which parts are quoted is decided lexically, for the same reason [skill retrieval](#finding-the-right-skill) is: a dense retriever would mean shipping a second model into an app whose premise is one download. Paragraphs are scored by the question's words, each weighted by how rare it is across everything the call fetched — so a stop list is unnecessary, because a word that appears in every paragraph earns almost nothing without anyone having to write it down. + +**That weighting has to be pooled across the sources, and finding out why is what the tests pin.** Measured per page, _who is the chief executive of the airline_ scored the paragraph containing "the airline" exactly level with the one naming the chief executive: within four paragraphs `the` is as rare as `executive`. Across the hundred-odd paragraphs five pages actually produce, `the` appears in nearly all of them and ends up worth about two per cent of `executive`. The mechanism only works at the scale it runs at, so `research.test.ts` fixtures are pages rather than snippets. + +Three more things follow from what a source is worth: + +- **Many sources has to mean many _different_ ones.** A search for a news story returns four pages of the same newspaper, and reading all four spends four reader requests to hear one newsroom repeat itself. The results are reordered so the first hit on each host comes first — reordered rather than filtered, because Wikipedia's results are all one host, and there the list refills with further articles instead of collapsing to a single source. +- **A page that will not open becomes its search snippet.** The reader's budget is shared, so a 429 on the fourth source is an ordinary event rather than a reason to discard the three that arrived. The header says how many sources are snippets rather than pages, because once both are quoted lines in a list nothing else distinguishes them. +- **Nothing readable is a failure, not an empty result.** A call that found pages and could read none of them throws. Reported as a result, a 0.8B model relays it as "there is nothing on this subject", which is the one answer a page that merely failed to load must never produce. + +**It costs six of the twenty requests a minute** the keyless reader allows — one search plus five pages — so three research questions in a minute is the honest ceiling. That is the real limit on "as many sources as possible", and it is why five is where this stops rather than fifteen. [LangSearch](#how-the-network-tools-work-without-a-server) takes the search off that budget and is worth the free key if you ask a lot of these. + +`research` and `web_search` therefore both ship, and the difference between them is cost: a single search is one request, a research call is six. Which one a turn sees is decided by [the skill that routes it](#skills) — `research-question` offers only `research`, `lookup-term` still searches — rather than by the model weighing that up for itself. + **`weather`** needs no key and no provider choice. It resolves the place with Open-Meteo's geocoder and then asks two unrelated services about that one point: Open-Meteo for DWD's ICON, NOAA's GFS and ECMWF's IFS, and wttr.in for an independent reading of the conditions right now. All three endpoints send `Access-Control-Allow-Origin: *` on the real request from the deployed origin. The geocoder matches names, and what a 0.8B model passes is often the whole question — `Wetter in Berlin`, `Hamburg heute`. Both found nothing, and the tool failed outright rather than approximately. So `placeCandidates` narrows the argument: the phrase as written first, then what follows a preposition, then the same with the subject and the time words removed. Whole-first is the safeguard, since In Salah is a town in Algeria and narrowing it would answer about somewhere else. @@ -448,13 +480,16 @@ Every reply says which skill answered it and how it was found: `weather skill · A skill fires on some requests. This runs on all of them. -Between the model settling on an answer and that answer reaching the screen, `src/agent/review.ts` reads it back against what the turn actually produced — the results the tools returned, and the URLs already in the conversation. Three things are checked: +Between the model settling on an answer and that answer reaching the screen, `src/agent/review.ts` reads it back against what the turn actually produced — the results the tools returned, and the URLs already in the conversation. Four things are checked: | Check | Fires when | | ----------------- | --------------------------------------------------------------------------- | | `wrong-number` | The calculator returned a value the answer states nowhere, at any precision | | `invented-source` | The answer cites a URL that no tool returned and nobody supplied | | `missing-source` | Tools returned sources and the answer cites none | +| `single-source` | Three or more sites were returned and the answer rests on one | + +The last one is what makes [`research`](#researching-a-question-across-several-sources) worth calling rather than merely expensive: five sources the reply never used are five reader requests spent on nothing. It is the shyest of the four. Three sites have to have been returned before it fires at all, because two is what an ordinary search-then-read turn produces — `web_search` finds a page and `read_page` opens it, and citing that page is correct. It counts sites rather than URLs, so two pages of one newspaper are one source however they are cited. And grounding is settled first: an answer citing nothing is asked for a source before it is asked for a second one. A failed check costs one further generation. The model is handed its own draft and told what to change — _The calculator returned 6748 \* 9 = 60732. Give that number, exactly as it came back._ — and the correction replaces the draft only if it leaves fewer problems behind. Otherwise the draft stands. That gate is the important half: the correction comes from the same 0.8B model, so a mechanism that could not tell an improvement from a regression would be a coin toss on every reply. diff --git a/src/agent/review.test.ts b/src/agent/review.test.ts index b7c6654..50f8d0a 100644 --- a/src/agent/review.test.ts +++ b/src/agent/review.test.ts @@ -162,6 +162,114 @@ describe('reviewAnswer', () => { ]) }) }) + + describe('breadth', () => { + const researched = evidence({ + toolResults: [ + { + tool: 'research', + result: [ + 'Researched "who runs it" across 3 sources, all read in full.', + '1. Leadership — https://fictionalairways.example/leadership', + ' "Ama Osei has led it since 2023."', + '2. Airtimes — https://airtimes.example/osei', + ' "The board appointed her in March 2023."', + '3. Encyclopedia — https://encyclopedia.example/fictional-airways', + ' "The airline was founded in 1974."', + ].join('\n'), + }, + ], + }) + + it('accepts an answer that cites two of the sources', () => { + expect( + checks( + 'Ama Osei, since March 2023.\n\nSources: https://fictionalairways.example/leadership https://airtimes.example/osei', + researched, + ), + ).toEqual([]) + }) + + it('catches an answer that cites one of three', () => { + expect(checks('Ama Osei.\n\nSource: https://fictionalairways.example/leadership', researched)).toEqual([ + 'single-source', + ]) + }) + + it('names a source on another host to check it against', () => { + const [finding] = reviewAnswer( + 'Ama Osei.\n\nSource: https://fictionalairways.example/leadership', + researched, + ) + + expect(finding?.instruction).toContain('https://airtimes.example/osei') + expect(finding?.instruction).toContain('3 sources were returned') + }) + + it('counts hosts rather than URLs, so two pages of one site are one source', () => { + const oneSite = evidence({ + toolResults: [ + { + tool: 'research', + result: + '1. A — https://fictionalairways.example/leadership\n2. B — https://fictionalairways.example/board\n3. C — https://fictionalairways.example/history', + }, + ], + }) + + expect(checks('Ama Osei.\n\nSource: https://fictionalairways.example/leadership', oneSite)).toEqual([]) + }) + + /** + * Search-then-read returns the page and then opens it, so the answer cites + * one host out of two by design. Asking for breadth there would fire on the + * commonest web turn there is. + */ + it('leaves an ordinary search and read alone', () => { + const searchThenRead = evidence({ + toolResults: [ + searchResult, + { tool: 'read_page', result: '# Leadership\nSource: https://fictionalairways.example/leadership' }, + ], + }) + + expect( + checks('Ama Osei.\n\nSource: https://fictionalairways.example/leadership', searchThenRead), + ).toEqual([]) + }) + + it('does not ask an answer that found nothing for a second source', () => { + expect(checks('I could not find who runs it.', researched)).toEqual([]) + }) + + it('asks for the missing source first when the answer cites none at all', () => { + expect(checks('Ama Osei runs it.', researched)).toEqual(['missing-source']) + }) + + it('asks about the invented source first when there is one', () => { + expect(checks('Ama Osei.\n\nSource: https://wikipedia.org/Osei', researched)).toEqual([ + 'invented-source', + ]) + }) + }) + + it('reports a dropped number and a thin citation together', () => { + const both = evidence({ + toolResults: [ + { tool: 'calculator', result: '2 + 2 = 4' }, + { + tool: 'research', + result: + '1. A — https://one.example/a\n2. B — https://two.example/b\n3. C — https://three.example/c', + }, + ], + }) + + expect(checks('About five.\n\nSource: https://one.example/a', both)).toEqual([ + 'wrong-number', + 'single-source', + ]) + }) }) describe('collectEvidence', () => { diff --git a/src/agent/review.ts b/src/agent/review.ts index 1e8d4fe..18cc4cf 100644 --- a/src/agent/review.ts +++ b/src/agent/review.ts @@ -15,7 +15,7 @@ import type { ChatTurn } from '@/llm/protocol' * costs a second generation and teaches the user to ignore the whole mechanism, * so every one of them prefers to miss a mistake over inventing one. */ -export type ReviewCheck = 'wrong-number' | 'invented-source' | 'missing-source' +export type ReviewCheck = 'wrong-number' | 'invented-source' | 'missing-source' | 'single-source' export interface ReviewFinding { check: ReviewCheck @@ -140,6 +140,23 @@ function preferredSource(evidence: ReviewEvidence): string | null { return null } +/** + * How many sources a turn has to have been given before citing one of them is + * treated as citing too few. + * + * Three, not two, because two is the shape an ordinary search-then-read turn + * produces: `web_search` returns a page and `read_page` opens it, and the answer + * cites the page it read. Nagging for a second source there would fire on the + * common case and teach the user to ignore the mechanism. `research` returns + * five independent sources, and an answer that quotes one of them has dropped + * the corroboration on purpose. + */ +const MIN_SOURCES_FOR_BREADTH = 3 + +function hostsOf(urls: string[]): Set { + return new Set(urls.flatMap((url) => locate(url)?.host ?? [])) +} + /** * Reads a draft answer against the evidence and returns what needs fixing. * @@ -162,13 +179,15 @@ export function reviewAnswer(answer: string, evidence: ReviewEvidence): ReviewFi } const source = preferredSource(evidence) - const known = [...evidence.knownUrls, ...evidence.toolResults.flatMap(({ result }) => findUrls(result))] + const returned = evidence.toolResults.flatMap(({ result }) => findUrls(result)) + const known = [...evidence.knownUrls, ...returned] .map(locate) .filter((entry): entry is Located => entry !== null) - const invented = findUrls(draft).find((url) => { - const cited = locate(url) - return cited !== null && !isGrounded(cited, known) + const cited = findUrls(draft) + const invented = cited.find((url) => { + const located = locate(url) + return located !== null && !isGrounded(located, known) }) if (invented) { @@ -178,11 +197,33 @@ export function reviewAnswer(answer: string, evidence: ReviewEvidence): ReviewFi ? `Nothing returned ${invented}. The source is ${source} — cite that one instead.` : `Nothing returned ${invented}. Drop that link; no source was fetched.`, }) - } else if (source && findUrls(draft).length === 0 && !NOTHING_TO_CITE.test(draft)) { + return findings + } + + if (source && cited.length === 0 && !NOTHING_TO_CITE.test(draft)) { findings.push({ check: 'missing-source', instruction: `The answer cites no source. End it with "Source: ${source}".`, }) + return findings + } + + // Breadth, once grounding is settled. A turn handed five independent sources + // and answering out of one has thrown away the only thing this app can offer + // in place of a bigger model: the same claim, found twice. + const offered = hostsOf(returned) + const citedHosts = hostsOf(cited) + if (citedHosts.size === 1 && offered.size >= MIN_SOURCES_FOR_BREADTH && !NOTHING_TO_CITE.test(draft)) { + const second = returned.find((url) => { + const host = locate(url)?.host + return host !== undefined && !citedHosts.has(host) + }) + if (second) { + findings.push({ + check: 'single-source', + instruction: `${offered.size} sources were returned and the answer cites one. Check it against ${second} and end with a "Sources:" line listing both.`, + }) + } } return findings diff --git a/src/components/MessageItem.tsx b/src/components/MessageItem.tsx index f67c240..352b3d9 100644 --- a/src/components/MessageItem.tsx +++ b/src/components/MessageItem.tsx @@ -36,6 +36,7 @@ const REVIEW_REASON: Record = { 'wrong-number': 'a number the calculator disagreed with', 'invented-source': 'a source no tool returned', 'missing-source': 'a missing source', + 'single-source': 'one source where several were found', } /** Reads as a phrase, so the same wording works before and after the fix. */ diff --git a/src/eval/scenarios.ts b/src/eval/scenarios.ts index 67d90ca..b793660 100644 --- a/src/eval/scenarios.ts +++ b/src/eval/scenarios.ts @@ -63,6 +63,30 @@ function searchQuery(calls: Invocation[]): string | null { return search ? String(search.arguments.query ?? '') : null } +const URL_IN_TEXT = /https?:\/\/[^\s<>"'`)\]}]+/g + +/** + * Passes when the answer rests on more than one site. + * + * `research` hands the model five independent sources, and an answer built from + * one of them is the failure the whole tool exists to avoid — invisible to + * `expectTool`, which sees the right call, and to a keyword `accept`, which sees + * the right fact. Hosts rather than URLs, because two pages of one newspaper are + * one source however they are cited. + */ +function citesSeveralSources(answer: string): boolean { + const hosts = new Set( + [...answer.matchAll(URL_IN_TEXT)].flatMap((match) => { + try { + return [new URL(match[0]).hostname.replace(/^www\./, '').toLowerCase()] + } catch { + return [] + } + }), + ) + return hosts.size >= 2 +} + /** * Passes when the place reached the weather tool. * @@ -261,11 +285,38 @@ export const SCENARIOS: Scenario[] = [ id: 'web-current-event', category: 'web', prompt: 'Who is the current secretary-general of the United Nations?', - expectTool: 'web_search', + expectTool: 'research', accept: matches(/guterres/i), - // The hardest of these under Wikipedia, whose lead extract describes the - // office and never names the incumbent: passing there needs a follow-up - // `read_page`. A web provider names him in the snippets. + // Used to be the hardest of these under Wikipedia, whose lead extract + // describes the office and never names the incumbent, so passing needed a + // second tool round. `research` reads the pages itself, so the name is in the + // first result whichever provider answered. + online: true, + }, + { + id: 'web-several-sources', + category: 'web', + // Breadth, measured on the answer rather than on the call: the tool always + // returns several sources, and whether the reply used more than one of them + // is the thing that is actually in question. + prompt: 'Who is the chief executive of Nvidia?', + expectTool: 'research', + accept: (answer) => /huang/i.test(answer) && citesSeveralSources(answer), + online: true, + }, + { + id: 'web-research-keeps-the-question', + category: 'web', + // The `1inch` failure in `research`'s clothing: the tool takes a question, so + // a model that boils it down to two words throws away what it was asked. + prompt: 'Who won the Formula 1 world championship in 2024?', + expectTool: 'research', + acceptCall: (calls) => { + const asked = calls.find((call) => call.name === 'research') + const question = String(asked?.arguments.question ?? '').toLowerCase() + return question.includes('2024') && /formula|f1/.test(question) + }, + accept: matches(/verstappen/i), online: true, }, { @@ -313,7 +364,7 @@ export const SCENARIOS: Scenario[] = [ // A price is looked up, never worked out. `how much is` was an arithmetic // keyword, so this reached the calculator with nothing to calculate. prompt: 'How much is a Big Mac in Japan?', - expectTool: 'web_search', + expectTool: 'research', accept: (answer) => /\d/.test(answer), online: true, }, @@ -322,7 +373,7 @@ export const SCENARIOS: Scenario[] = [ category: 'web', // `today` was a current-date trigger, which answered this with the date. prompt: "What's today's news?", - expectTool: 'web_search', + expectTool: 'research', accept: (answer) => answer.trim().length > 20, online: true, }, diff --git a/src/lib/tool-labels.test.ts b/src/lib/tool-labels.test.ts index 806ee7b..abf7b86 100644 --- a/src/lib/tool-labels.test.ts +++ b/src/lib/tool-labels.test.ts @@ -12,6 +12,13 @@ describe('describeTool', () => { expect(describeTool('read_page', 'error')).toBe('Read a page') }) + // The row is the only place the difference between one search and a call that + // spends six reader requests is visible before it is opened. + it('distinguishes researching several sources from a single search', () => { + expect(describeTool('research', 'running')).toBe('Researching several sources') + expect(describeTool('research', 'done')).toBe('Researched several sources') + }) + it('leaves a tool it does not ship under its own name', () => { expect(describeTool('acme_create_ticket', 'done')).toBe('acme_create_ticket') }) diff --git a/src/lib/tool-labels.ts b/src/lib/tool-labels.ts index 9970c32..5d17a4f 100644 --- a/src/lib/tool-labels.ts +++ b/src/lib/tool-labels.ts @@ -12,6 +12,7 @@ import type { ToolCall } from '@/types' * whoever wrote them, and inventing a verb for one would be a guess. */ const PHRASES: Record = { + research: { running: 'Researching several sources', done: 'Researched several sources' }, web_search: { running: 'Searching the web', done: 'Searched the web' }, read_page: { running: 'Reading a page', done: 'Read a page' }, calculator: { running: 'Calculating', done: 'Calculated' }, diff --git a/src/skills/library.test.ts b/src/skills/library.test.ts index d74fb86..9bd473a 100644 --- a/src/skills/library.test.ts +++ b/src/skills/library.test.ts @@ -33,7 +33,7 @@ describe('the shipped library', () => { ['current-date', 25, ['current_time']], ['summarize-url', 20, ['read_page']], ['lookup-term', 15, ['web_search', 'read_page']], - ['research-question', 10, ['web_search', 'read_page']], + ['research-question', 10, ['research']], ]) }) @@ -366,7 +366,10 @@ describe('activating each shipped skill', () => { ['What year is it?', 'current-date', ['current_time']], ['What does https://example.com/pricing say?', 'summarize-url', ['read_page']], ['What is Stripe?', 'lookup-term', ['web_search', 'read_page']], - ['Who is the current secretary-general of the UN?', 'research-question', ['web_search', 'read_page']], + // One tool and no choice about it, the way `weather` gets one: `research` + // already searches and reads, so offering the pieces alongside it would only + // give the model a decision it has nothing to decide with. + ['Who is the current secretary-general of the UN?', 'research-question', ['research']], ] it.each(cases)('materialises %s for %j with only its tools', (message, name, tools) => { diff --git a/src/skills/research-question/SKILL.md b/src/skills/research-question/SKILL.md index 6226e53..ddea6a7 100644 --- a/src/skills/research-question/SKILL.md +++ b/src/skills/research-question/SKILL.md @@ -1,11 +1,10 @@ --- name: research-question -description: Answers a question about current or verifiable facts by searching the web, opening the most promising result and citing it. Use for anything recent, anything about a named person or organisation, and anything you would otherwise be guessing at. +description: Answers a question about current or verifiable facts by researching it across several web sources at once and citing the ones it used. Use for anything recent, anything about a named person or organisation, and anything you would otherwise be guessing at. jarvis: priority: 10 tools: - - web_search - - read_page + - research keywords: - look it up - find out @@ -43,17 +42,22 @@ jarvis: exemplars: - user: Who is the chief executive of Fictional Airways? steps: - - tool: web_search + - tool: research arguments: - query: Fictional Airways chief executive + question: Who is the chief executive of Fictional Airways? result: | - 1. Leadership — Fictional Airways - https://fictionalairways.example/leadership - Ama Osei has led Fictional Airways as chief executive since 2023. + Researched "Who is the chief executive of Fictional Airways?" across 3 sources, all read in full. + + 1. Leadership — https://fictionalairways.example/leadership + "Ama Osei has led Fictional Airways as chief executive since 2023." + 2. Fictional Airways names new chief — https://airtimes.example/osei-appointed + "The board appointed Ama Osei in March 2023, succeeding Piet Hendriks." + 3. Fictional Airways — https://encyclopedia.example/fictional-airways + "The airline was founded in 1974 and is based in Accra." answer: | - Ama Osei, chief executive since 2023. + Ama Osei. She was appointed in March 2023, succeeding Piet Hendriks. - Source: https://fictionalairways.example/leadership + Sources: https://fictionalairways.example/leadership https://airtimes.example/osei-appointed --- -Search first, then answer from the results. Open a result with `read_page` only when the snippet is not enough. Always end with the source URL. +Call `research` once, with the question as it was asked. It searches and reads several sources for you, so do not search again afterwards. Answer from the quoted passages only, prefer what more than one source says, and end with a `Sources:` line listing every URL you used. diff --git a/src/tools/builtins.test.ts b/src/tools/builtins.test.ts index 072aa51..3467ad3 100644 --- a/src/tools/builtins.test.ts +++ b/src/tools/builtins.test.ts @@ -69,3 +69,26 @@ describe('web_search', () => { expect(description('langsearch')).toMatch(/Search the web/) }) }) + +describe('research', () => { + const research = () => toolNamed('research') + + it('refuses an empty question without spending a request', async () => { + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + + await expect(research().execute({ question: ' ' })).rejects.toThrow('question must not be empty') + expect(fetchMock).not.toHaveBeenCalled() + }) + + // Told it has the whole web, the model asks Wikipedia for the morning's news. + it('says which kind of source it is about to consult', () => { + const description = (provider: SearchProvider) => + createBuiltinTools({ provider }).find((tool) => tool.schema.function.name === 'research')!.schema + .function.description + + expect(description('wikipedia')).toMatch(/Wikipedia articles/) + expect(description('wikipedia')).toMatch(/does not cover current events/) + expect(description('duckduckgo')).toMatch(/independent web sources/) + }) +}) diff --git a/src/tools/builtins.ts b/src/tools/builtins.ts index f7f8a4a..faef820 100644 --- a/src/tools/builtins.ts +++ b/src/tools/builtins.ts @@ -1,5 +1,6 @@ import { evaluateExpression } from './calculator' import { memory } from './memory' +import { researchQuestion } from './research' import { defineTool, type Tool } from './types' import { DEFAULT_WEB_ACCESS, readPage, searchWeb, type SearchProvider, type WebAccessConfig } from './web' import { weatherReport } from './weather' @@ -56,6 +57,42 @@ function truncate(text: string): string { return `${text.slice(0, MAX_PAGE_CHARS)}\n\n[Truncated: the page continues beyond this point.]` } +/** + * The same distinction `searchDescription` draws, for the same reason: told it + * has the whole web, the model asks this for the morning's news, and under the + * Wikipedia provider every source it gets back is an encyclopedia article. + */ +function researchDescription(provider: SearchProvider): string { + if (provider === 'wikipedia') { + return 'Research a question across several Wikipedia articles at once and return quoted passages with the URL each came from. Use for facts, definitions, people, places and history; it does not cover current events.' + } + return 'Research a question across several independent web sources at once and return quoted passages with the URL each came from. Use for current events, people, organisations, prices, and anything where one source is not enough.' +} + +/** + * One call for a whole question, rather than a search the model then has to + * follow up. The fan-out and the narrowing both happen in `research.ts`; what + * arrives here is already short enough to hand to the model whole. + */ +function createResearch(config: WebAccessConfig): Tool { + return defineTool( + 'research', + researchDescription(config.provider), + { + type: 'object', + properties: { + question: { type: 'string', description: 'The question to research, in the words it was asked in' }, + }, + required: ['question'], + }, + async (args) => { + const question = String(args.question ?? '').trim() + if (!question) throw new Error('question must not be empty') + return researchQuestion(question, config) + }, + ) +} + function createReadPage(config: WebAccessConfig): Tool { return defineTool( 'read_page', @@ -125,13 +162,27 @@ export const currentTime = defineTool( * rebuilt when those change. Every tool ships in every deployment: none of them * needs a server, so a static host is no longer a reason to withhold one. * + * `research` and `web_search` overlap on purpose, and the difference is cost. + * One search is a single request through the reader; a research call is six, out + * of the twenty a minute the keyless tier allows. So a question that wants + * corroboration gets `research` and a term that wants a definition gets + * `web_search`, and which one a turn sees is decided by the skill that routes it + * rather than by the model weighing that up for itself. + * * `memory` is the exception, and is left out entirely when the user has turned * memory off. Offering a tool that then refuses would spend a tool round to * arrive at nothing, and would put the word "remember" in a prompt from someone * who asked not to be remembered. */ export function createBuiltinTools(config: WebAccessConfig, options: { memory?: boolean } = {}): Tool[] { - const tools = [createWebSearch(config), createReadPage(config), calculator, currentTime, weather] + const tools = [ + createResearch(config), + createWebSearch(config), + createReadPage(config), + calculator, + currentTime, + weather, + ] return options.memory === false ? tools : [...tools, memory] } diff --git a/src/tools/research.test.ts b/src/tools/research.test.ts new file mode 100644 index 0000000..257c73c --- /dev/null +++ b/src/tools/research.test.ts @@ -0,0 +1,379 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { digest, diverseFirst, paragraphsOf, passagesFor, researchQuestion } from './research' +import type { SearchResult, WebAccessConfig } from './web' + +function result(url: string, title = 'Title', snippet = 'A snippet.'): SearchResult { + return { url, title, snippet } +} + +describe('diverseFirst', () => { + it('puts the first hit on each host ahead of any second hit on one', () => { + const chosen = diverseFirst( + [ + result('https://news.example/one'), + result('https://news.example/two'), + result('https://other.example/a'), + result('https://third.example/b'), + ], + 3, + ) + + expect(chosen.map((entry) => entry.url)).toEqual([ + 'https://news.example/one', + 'https://other.example/a', + 'https://third.example/b', + ]) + }) + + it('treats www as the same host', () => { + const chosen = diverseFirst( + [result('https://www.news.example/one'), result('https://news.example/two')], + 2, + ) + + expect(chosen.map((entry) => entry.url)).toEqual([ + 'https://www.news.example/one', + 'https://news.example/two', + ]) + }) + + // Wikipedia returns every result on one host. Collapsing to a single source + // there would make the tool useless under that provider, so the list refills + // with further pages from the same host rather than being cut short. + it('fills the quota from one host when that is all there is', () => { + const chosen = diverseFirst( + [ + result('https://en.wikipedia.org/wiki/A'), + result('https://en.wikipedia.org/wiki/B'), + result('https://en.wikipedia.org/wiki/C'), + ], + 3, + ) + + expect(chosen).toHaveLength(3) + }) + + it('drops a URL the provider returned twice', () => { + const chosen = diverseFirst([result('https://news.example/one'), result('https://news.example/one')], 5) + + expect(chosen).toHaveLength(1) + }) + + it('skips a result with no URL at all', () => { + expect(diverseFirst([result(''), result('https://news.example/one')], 5)).toHaveLength(1) + }) +}) + +describe('paragraphsOf', () => { + it('keeps link text and drops the address', () => { + const [paragraph] = paragraphsOf( + 'The airline was founded by [Ama Osei](https://who.example/osei) in 1974 and still flies daily.', + ) + + expect(paragraph).toContain('Ama Osei') + expect(paragraph).not.toContain('who.example') + }) + + it('drops headings, bullets and images but keeps what they introduced', () => { + const paragraphs = paragraphsOf( + '# Leadership\n\n![Image 1: portrait](https://img.example/1.png)\n\n- Ama Osei has led the airline as chief executive since March 2023.', + ) + + expect(paragraphs).toEqual(['Ama Osei has led the airline as chief executive since March 2023.']) + }) + + it('leaves out nav items and bylines, which are too short to be prose', () => { + expect(paragraphsOf('Home\n\nBy our reporter\n\nShare')).toEqual([]) + }) + + it('joins the lines of one paragraph and separates two', () => { + expect( + paragraphsOf( + 'Ama Osei has led the airline as its chief\nexecutive since March 2023.\n\nThe airline was founded in 1974 and is based in Accra, Ghana.', + ), + ).toEqual([ + 'Ama Osei has led the airline as its chief executive since March 2023.', + 'The airline was founded in 1974 and is based in Accra, Ghana.', + ]) + }) +}) + +describe('passagesFor', () => { + /** + * As many paragraphs as a real page has, and that is the point rather than the + * prose: the term weights are measured over them. A four-paragraph fixture + * makes `the` exactly as rare as `executive` and ranks the wrong paragraph + * first, which is what pooling the weights across everything fetched fixes. + */ + const PAGE = [ + 'Skip to content and sign up for the newsletter so you never miss an update from us.', + 'The airline was founded in 1974 in Accra and now flies to thirty destinations.', + 'Ama Osei was appointed chief executive in March 2023, succeeding Piet Hendriks.', + 'The cabin refurbishment programme was completed across the whole fleet last summer.', + 'The company reported a modest profit for the financial year ending in December 2025.', + 'Cookies help the site deliver its services, and you may choose to accept all of them.', + 'The head office moved to a new building near the airport in the early part of 2024.', + 'Passengers may check two bags on the international routes without paying a fee.', + ].join('\n\n') + + function forOnePage(question: string, markdown: string): string[] { + return passagesFor(question, [paragraphsOf(markdown)])[0] ?? [] + } + + it('picks the paragraph that answers the question over one that merely shares its words', () => { + const [first] = forOnePage('Who is the chief executive of the airline?', PAGE) + + expect(first).toContain('Ama Osei') + }) + + it('returns at most two passages, however long the page is', () => { + expect(forOnePage('airline chief executive founded Accra 1974', PAGE)).toHaveLength(2) + }) + + it('gives every page its own list, in the order the sources were selected', () => { + const chosen = passagesFor('chief executive', [paragraphsOf(PAGE), [], paragraphsOf(PAGE)]) + + // The middle page is one that could not be read. Dropping it here would + // shift every source after it onto the wrong URL. + expect(chosen.map((passages) => passages.length)).toEqual([1, 0, 1]) + }) + + it('has nothing to say about a page with no prose on it', () => { + expect(forOnePage('who runs it', 'Home\n\nContact\n\nShare')).toEqual([]) + }) + + /** + * A definition is answered by a lead paragraph that repeats none of the words + * in the question, so a page whose score is zero everywhere still contributes + * its opening rather than going silent. + */ + it('falls back to the opening paragraph when nothing matches', () => { + const [first] = forOnePage( + 'zzz qqq', + 'Stripe is a payments company that builds financial infrastructure for online businesses.', + ) + + expect(first).toBe( + 'Stripe is a payments company that builds financial infrastructure for online businesses.', + ) + }) + + it('cuts a long paragraph down to the sentences that carry the question', () => { + const filler = 'The catering was reviewed in a report nobody read. '.repeat(6) + const [first] = forOnePage( + 'Who was appointed chief executive?', + `${filler}Ama Osei was appointed chief executive in March 2023.${filler}`, + ) + + expect(first).toContain('Ama Osei was appointed chief executive in March 2023.') + expect(first!.length).toBeLessThanOrEqual(280) + }) + + it('cuts a single sentence that is longer than the cap on its own', () => { + const [first] = forOnePage('chief executive', `Ama Osei ${'and others '.repeat(60)}is chief executive.`) + + expect(first!.length).toBeLessThanOrEqual(281) + expect(first).toContain('…') + }) + + it('does not quote the same claim twice', () => { + const twice = [ + 'Ama Osei was appointed chief executive of the airline in March 2023 after nine years.', + 'Ama Osei was appointed chief executive of the airline in March 2023 after nine years.', + ].join('\n\n') + + expect(forOnePage('who is the chief executive', twice)).toHaveLength(1) + }) +}) + +describe('digest', () => { + const sources = [ + { + url: 'https://fictionalairways.example/leadership', + title: 'Leadership', + passages: ['Ama Osei has led Fictional Airways as chief executive since 2023.'], + read: true, + }, + { + url: 'https://airtimes.example/osei', + title: 'Airtimes', + passages: ['The board appointed Ama Osei in March 2023.'], + read: true, + }, + ] + + it('numbers each source and puts its URL on the same line as its name', () => { + expect(digest('Who runs Fictional Airways?', sources)).toBe( + [ + 'Researched "Who runs Fictional Airways?" across 2 sources, all read in full.', + '', + '1. Leadership — https://fictionalairways.example/leadership', + ' "Ama Osei has led Fictional Airways as chief executive since 2023."', + '2. Airtimes — https://airtimes.example/osei', + ' "The board appointed Ama Osei in March 2023."', + ].join('\n'), + ) + }) + + // A snippet is weaker evidence than a page, and once both are quoted lines in + // a list nothing else says which is which. + it('says how many sources are snippets rather than pages', () => { + const mixed = [sources[0]!, { ...sources[1]!, read: false }] + + expect(digest('Who runs it?', mixed)).toContain( + 'across 2 sources; 1 read in full, 1 from the search snippet only', + ) + }) + + it('warns when no page could be opened at all', () => { + const snippets = sources.map((source) => ({ ...source, read: false })) + + expect(digest('Who runs it?', snippets)).toContain('none could be opened, so these are search snippets') + }) + + it('caps the whole result, because five whole pages is what this avoids', () => { + const long = Array.from({ length: 5 }, (_, at) => ({ + url: `https://example.com/${at}`, + title: 'T'.repeat(400), + passages: ['x'.repeat(600), 'y'.repeat(600)], + read: true, + })) + + const built = digest('anything', long) + + expect(built).toContain('[Truncated: further sources were dropped.]') + expect(built.length).toBeLessThan(4_100) + }) + + it('does not double the quotes around a passage that arrived with them', () => { + const quoted = [{ ...sources[0]!, passages: ['"Already quoted."'] }] + + expect(digest('q', quoted)).toContain(' "Already quoted."') + }) +}) + +/** + * The whole call, over a stubbed network: LangSearch answers the search and the + * reader answers each page, which is the shape every provider reduces to. + */ +describe('researchQuestion', () => { + const config: WebAccessConfig = { provider: 'langsearch', langsearchApiKey: 'key' } + const question = 'Who is the chief executive of Fictional Airways?' + + const LEADERSHIP = 'https://fictionalairways.example/leadership' + const AIRTIMES = 'https://airtimes.example/osei' + + const HITS = [ + { name: 'Leadership', url: LEADERSHIP, snippet: 'Ama Osei leads the airline.' }, + { name: 'Airtimes', url: AIRTIMES, snippet: 'The board appointed Ama Osei in 2023.' }, + ] + + /** A page body, or the status the reader should refuse it with. */ + type Reply = string | number + + function stubNetwork(pages: Record, hits = HITS) { + const calls: string[] = [] + + vi.stubGlobal( + 'fetch', + vi.fn(async (input: string) => { + const url = String(input) + calls.push(url) + + if (url.startsWith('https://api.langsearch.com')) { + return { ok: true, status: 200, json: async () => ({ data: { webPages: { value: hits } } }) } + } + + const target = url.replace('https://r.jina.ai/', '') + const reply = pages[target] + if (reply === undefined || typeof reply === 'number') { + return { ok: false, status: typeof reply === 'number' ? reply : 404 } + } + return { + ok: true, + status: 200, + json: async () => ({ data: { title: 'T', url: target, content: reply } }), + } + }), + ) + + return calls + } + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('quotes every source it could read, each against its own URL', async () => { + stubNetwork({ + [LEADERSHIP]: 'Ama Osei has led Fictional Airways as chief executive since March 2023 in Accra.', + [AIRTIMES]: 'The board appointed Ama Osei as chief executive in March 2023, succeeding Piet Hendriks.', + }) + + const result = await researchQuestion(question, config) + + expect(result).toContain('across 2 sources, all read in full') + expect(result).toContain(`1. T — ${LEADERSHIP}`) + expect(result).toContain( + '"Ama Osei has led Fictional Airways as chief executive since March 2023 in Accra."', + ) + expect(result).toContain(`2. T — ${AIRTIMES}`) + expect(result).toContain('succeeding Piet Hendriks') + }) + + it('reads the pages at the same time rather than one after another', async () => { + const calls = stubNetwork({ + [LEADERSHIP]: 'Ama Osei has led Fictional Airways as chief executive since March 2023 in Accra.', + [AIRTIMES]: 'The board appointed Ama Osei as chief executive in March 2023, succeeding Piet Hendriks.', + }) + + await researchQuestion(question, config) + + // One search and one read per source: the cost of the call is what the + // reader's per-minute budget is spent on, so it is worth pinning. + expect(calls).toHaveLength(3) + }) + + /** + * The reader's budget is shared between search and every page, so a 429 on one + * source is an ordinary event. Losing the two that arrived over it would turn a + * partial answer into no answer. + */ + it('stands a page that would not open in as its search snippet', async () => { + stubNetwork({ + [LEADERSHIP]: 'Ama Osei has led Fictional Airways as chief executive since March 2023 in Accra.', + [AIRTIMES]: 429, + }) + + const result = await researchQuestion(question, config) + + expect(result).toContain('1 read in full, 1 from the search snippet only') + expect(result).toContain('"The board appointed Ama Osei in 2023."') + }) + + it('says so when no page opened and only snippets are left', async () => { + stubNetwork({ [LEADERSHIP]: 500, [AIRTIMES]: 429 }) + + const result = await researchQuestion(question, config) + + expect(result).toContain('none could be opened, so these are search snippets only') + }) + + /** + * Nothing readable and nothing to quote is a failure, not an empty result. A + * 0.8B model relays "no sources" as "there is nothing on this subject", which + * is the one answer that must never come out of a page that simply would not + * load. + */ + it('fails rather than reporting that it found nothing', async () => { + stubNetwork({ [LEADERSHIP]: 500 }, [{ name: 'Leadership', url: LEADERSHIP, snippet: '' }]) + + await expect(researchQuestion(question, config)).rejects.toThrow('could not read any of them') + }) + + it('reports a search that genuinely matched nothing as such', async () => { + stubNetwork({}, []) + + await expect(researchQuestion(question, config)).resolves.toBe(`No results for "${question}".`) + }) +}) diff --git a/src/tools/research.ts b/src/tools/research.ts new file mode 100644 index 0000000..5bcb2fe --- /dev/null +++ b/src/tools/research.ts @@ -0,0 +1,361 @@ +/** + * Researching a question across several sources in one tool call. + * + * `web_search` and `read_page` can do this between them, and the model has to + * chain them to get there: search, pick a result, read it, and — if it wants a + * second opinion — read another. A turn is capped at `MAX_TOOL_ROUNDS` rounds, + * so that chain runs out of budget at roughly two sources, and every link in it + * is a decision a 0.8B model can get wrong. This is the `weather` shape applied + * to the web: the fan-out happens here, and what reaches the model is one + * compact result it did not have to assemble. + * + * The reason to do it here is not only the round budget. Reading more of the web + * makes answers worse if all of it reaches the context — function-calling + * accuracy falls by 7% to 91% as tool responses grow (arXiv:2505.10570), and + * five pages at `read_page`'s cap would be 40,000 characters. So the pages are + * read in full and quoted in part: the passages that bear on the question are + * selected here, verbatim, and the rest is never sent. Five sources cost less + * context than one whole page. + * + * Selection is lexical, for the same reason skill routing is: a dense retriever + * would mean shipping a second model into an app whose premise is one download. + * Paragraphs are scored by the question's terms, each weighted by how rare it is + * across every paragraph the turn fetched — which is what makes a stop list + * unnecessary, since a word that appears in all of them earns almost nothing + * without anyone having to write it down. + */ + +import { readPage, searchWeb, type SearchResult, type WebAccessConfig } from './web' + +/** + * How many results to ask for before narrowing them. Larger than `MAX_SOURCES` + * because the narrowing drops duplicate hosts, and a page of results from one + * newspaper should still leave five sources to read. + */ +const SEARCH_LIMIT = 8 + +/** + * How many sources are consulted. + * + * The ceiling is the reader's, not the model's: search and `read_page` share 20 + * requests a minute per IP without a Jina key, so one call already spends six of + * them and three questions in a minute is the honest limit. Raising this trades + * a rate limit the user cannot see for sources the answer does not need. + */ +const MAX_SOURCES = 5 + +/** Two passages carry a claim and its context. A third is usually the same claim again. */ +const MAX_PASSAGES_PER_SOURCE = 2 + +const MAX_PASSAGE_CHARS = 280 + +/** Shorter than this is a heading, a byline or a nav item rather than prose. */ +const MIN_PASSAGE_CHARS = 60 + +/** Words, not characters: a long run of link text can clear the character floor. */ +const MIN_PASSAGE_WORDS = 8 + +/** + * The whole result, well under `read_page`'s 8,000. Five sources are worth + * having because they are short; five sources at page length would be the + * failure this tool exists to avoid. + */ +const MAX_DIGEST_CHARS = 4000 + +const WORD = /[\p{L}\p{N}]+/gu + +function words(text: string): string[] { + return [...text.toLowerCase().matchAll(WORD)].map((match) => match[0]) +} + +function collapse(value: string): string { + return value.replace(/\s+/g, ' ').trim() +} + +function hostOf(url: string): string { + try { + return new URL(url).hostname.replace(/^www\./, '').toLowerCase() + } catch { + return url + } +} + +/** + * Orders the results so that the first hit on each host comes before any second + * hit on one, then takes the first `max`. + * + * "Many sources" has to mean many *different* sources: a search for a news story + * returns four pages of the same newspaper, and reading all four is four reader + * requests spent to hear one newsroom repeat itself. Reordering rather than + * discarding is what keeps this from being a special case for Wikipedia, whose + * results are all one host — there the list refills with further articles from + * that host instead of collapsing to a single source. + */ +export function diverseFirst(results: SearchResult[], max: number): SearchResult[] { + const hosts = new Set() + const urls = new Set() + const first: SearchResult[] = [] + const rest: SearchResult[] = [] + + for (const result of results) { + if (!result.url || urls.has(result.url)) continue + urls.add(result.url) + + const host = hostOf(result.url) + if (hosts.has(host)) { + rest.push(result) + continue + } + hosts.add(host) + first.push(result) + } + + return [...first, ...rest].slice(0, max) +} + +const IMAGE = /!\[[^\]]*\]\([^)]*\)/g +const LINK = /\[([^\]]*)\]\([^)]*\)/g +const LINE_FURNITURE = /^\s*(?:#{1,6}\s+|[-*+]\s+|>\s+|\d+\.\s+)/ + +/** + * Reads the reader's markdown back into candidate paragraphs. + * + * Link text is kept and the target dropped: a paragraph is prose whether or not + * the names in it were linked, and keeping the URLs would put addresses no tool + * returned in front of a model that is checked for citing exactly those. + */ +export function paragraphsOf(markdown: string): string[] { + return markdown + .replace(IMAGE, ' ') + .replace(LINK, '$1') + .split(/\n\s*\n/) + .map((block) => + collapse( + block + .split('\n') + .map((line) => line.replace(LINE_FURNITURE, '')) + .join(' '), + ), + ) + .filter((block) => block.length >= MIN_PASSAGE_CHARS && words(block).length >= MIN_PASSAGE_WORDS) +} + +/** BM25's idf without the length normalisation, as `skills/retrieve.ts` uses it. */ +function inverseFrequency(documentFrequency: number, total: number): number { + return Math.log((total - documentFrequency + 0.5) / (documentFrequency + 0.5) + 1) +} + +/** + * What each word of the question is worth, measured over every paragraph the + * turn fetched rather than over one page at a time. + * + * Pooling is what makes a stop list unnecessary, and it has to be pooled to + * work: across a hundred paragraphs *the* appears in nearly all of them and ends + * up worth about two per cent of *executive*, but within a single four-paragraph + * page it can be exactly as rare and score just as high. Measured on one page, + * "who is the chief executive of the airline" ranked the paragraph containing + * *the airline* level with the one naming the chief executive. + * + * A length floor would be the cheap way to drop *of* and *is*, and it is the + * wrong one: *UN*, *EU* and *AI* are two letters and are the whole question. + */ +function weigh(question: string, corpus: string[]): Map { + const terms = new Set(words(question).filter((term) => term.length > 1)) + const tokenized = corpus.map((paragraph) => new Set(words(paragraph))) + + const weights = new Map() + for (const term of terms) { + const seen = tokenized.filter((paragraph) => paragraph.has(term)).length + if (seen > 0) weights.set(term, inverseFrequency(seen, tokenized.length)) + } + return weights +} + +function score(text: string, weights: Map): number { + let total = 0 + for (const term of new Set(words(text))) total += weights.get(term) ?? 0 + return total +} + +const SENTENCE_END = /(?<=[.!?…])\s+/ + +/** + * Cuts a paragraph down to the run of sentences that carries the most of the + * question, so a long page contributes its relevant lines rather than its first + * ones. A single sentence over the cap is cut mid-way, which is the one case + * where this cannot avoid it. + */ +function condense(paragraph: string, weights: Map): string { + if (paragraph.length <= MAX_PASSAGE_CHARS) return paragraph + + const sentences = paragraph.split(SENTENCE_END) + let best = '' + let bestScore = -1 + + for (let start = 0; start < sentences.length; start += 1) { + let window = '' + for (let end = start; end < sentences.length; end += 1) { + const extended = window ? `${window} ${sentences[end]}` : (sentences[end] ?? '') + if (extended.length > MAX_PASSAGE_CHARS) break + window = extended + const windowScore = score(window, weights) + if (windowScore > bestScore || (windowScore === bestScore && window.length > best.length)) { + best = window + bestScore = windowScore + } + } + } + + return best || `${paragraph.slice(0, MAX_PASSAGE_CHARS).trimEnd()}…` +} + +/** Enough of a passage to recognise the same claim written out twice. */ +function fingerprint(passage: string): string { + return words(passage).slice(0, 8).join(' ') +} + +/** + * The passages from one page worth putting in front of the model. + * + * A page whose prose never repeats the question's words still falls back to its + * opening paragraph: *what is Stripe* is answered by a lead paragraph that says + * "Stripe is a payments company" and may never say "what" or "is" again. + */ +function choose(candidates: string[], weights: Map): string[] { + if (candidates.length === 0) return [] + + const ranked = candidates + .map((text, at) => ({ text, at, score: score(text, weights) })) + .sort((a, b) => b.score - a.score || a.at - b.at) + + const relevant = ranked.filter((candidate) => candidate.score > 0) + const passages: string[] = [] + const seen = new Set() + + for (const candidate of relevant.length > 0 ? relevant : ranked.slice(0, 1)) { + if (passages.length >= MAX_PASSAGES_PER_SOURCE) break + const passage = condense(candidate.text, weights) + const mark = fingerprint(passage) + if (mark === '' || seen.has(mark)) continue + seen.add(mark) + passages.push(passage) + } + + return passages +} + +/** + * Picks the passages for every page at once, so each is ranked against the same + * weights. Pages arrive as paragraph lists and leave as passage lists, index for + * index; a page nothing could be read off stays empty rather than being dropped, + * because its position still names a source. + */ +export function passagesFor(question: string, pages: string[][]): string[][] { + const weights = weigh(question, pages.flat()) + return pages.map((candidates) => choose(candidates, weights)) +} + +export interface Source { + url: string + title: string + /** Verbatim, best first. */ + passages: string[] + /** False when the page could not be opened and its search snippet stood in. */ + read: boolean +} + +/** Straight quotes wrap each passage, so the model is not handed its own edges to trip on. */ +function unquote(passage: string): string { + return passage.replace(/^["'“”]+|["'“”]+$/g, '').trim() +} + +function entry(source: Source, at: number): string { + const heading = `${at + 1}. ${source.title || hostOf(source.url)} — ${source.url}` + return [heading, ...source.passages.map((passage) => ` "${unquote(passage)}"`)].join('\n') +} + +/** + * Says what was consulted before quoting any of it. + * + * A source that could only be reached as a search snippet is named as one: a + * snippet is weaker evidence than a page, and the difference is invisible once + * both are quoted lines in a list. + */ +function header(question: string, sources: Source[]): string { + const read = sources.filter((source) => source.read).length + const subject = `Researched "${question}" across ${sources.length} source${sources.length === 1 ? '' : 's'}` + + if (read === 0) return `${subject}; none could be opened, so these are search snippets only.` + if (read === sources.length) return `${subject}, all read in full.` + return `${subject}; ${read} read in full, ${sources.length - read} from the search snippet only.` +} + +export function digest(question: string, sources: Source[]): string { + const body = [header(question, sources), '', ...sources.map(entry)].join('\n') + if (body.length <= MAX_DIGEST_CHARS) return body + return `${body.slice(0, MAX_DIGEST_CHARS)}\n\n[Truncated: further sources were dropped.]` +} + +function reasonsFrom(settled: PromiseSettledResult[]): string { + return settled + .flatMap((outcome) => + outcome.status === 'rejected' + ? [outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason)] + : [], + ) + .join('; ') +} + +/** + * Searches, reads the most promising results in parallel and returns the + * passages that bear on the question, each with the URL it came from. + * + * A page that fails is replaced by its search snippet rather than allowed to + * take the answer down with it — the reader's per-minute budget is shared, so a + * 429 on the fourth page is an ordinary event and not a reason to abandon the + * three that arrived. + */ +export async function researchQuestion(question: string, config: WebAccessConfig): Promise { + const results = await searchWeb(question, SEARCH_LIMIT, config) + if (results.length === 0) return `No results for "${question}".` + + const selected = diverseFirst(results, MAX_SOURCES) + const settled = await Promise.allSettled(selected.map((result) => readPage(result.url, config))) + + const chosen = passagesFor( + question, + settled.map((outcome) => (outcome.status === 'fulfilled' ? paragraphsOf(outcome.value.text) : [])), + ) + + const sources = selected.flatMap((result, at): Source[] => { + const outcome = settled[at] + const passages = chosen[at] ?? [] + + if (outcome?.status === 'fulfilled' && passages.length > 0) { + return [ + { + url: outcome.value.url, + title: outcome.value.title || result.title, + passages, + read: true, + }, + ] + } + + const snippet = collapse(result.snippet) + if (!snippet) return [] + return [{ url: result.url, title: result.title, passages: [snippet], read: false }] + }) + + // Every source silent means the search found pages and nothing could be read + // off any of them. Reporting that as a result would have the model relay it as + // "there is nothing on this", which is the one thing it must not say. + if (sources.length === 0) { + const reasons = reasonsFrom(settled) + throw new Error( + `Found ${results.length} results for "${question}" but could not read any of them${reasons ? `: ${reasons}` : '.'}`, + ) + } + + return digest(question, sources) +} From 98274dd14ec6ced730de8c29f4e10a2a164d1d4c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 25 Aug 2026 22:06:30 +0000 Subject: [PATCH 2/2] Quote prose from a researched page rather than its furniture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run against the live web, the tool returned five real sources per question and largely quoted the wrong parts of them. Each failure below is now a filter whose test fixture is the string that was actually returned. Two of five Nvidia sources contributed nothing but consent notices: a cookie banner is several sentences long and names the site it sits on, which is all the scoring had to go on. Wikipedia's reference list outranked the sentence that answered the question, because citations repeat the subject once per entry. A Sucuri block page counted as a source, since the reader answers 200 and nothing upstream could see a failure — it now falls back to the search snippet for the same URL, which turned out to be the best passage in the German run. A breadcrumb was quoted as a source on what a chancellor is, and needs no word list to catch: prose ends in a full stop and menus do not, in either language. And condense returned the FAQ heading "Who leads NVIDIA?" whole, a heading being the densest window a paragraph contains, so a window now has to clear the passage floor to win on score alone. The word lists have a cost and it is stated rather than hidden: a paragraph genuinely about cookies is dropped with the banners. The block page test pairs its wording against a length, because an article about Cloudflare is long and a page refusing to serve one is not. Two more came from removing markup without leaving a separator, which is the trap `unbold` already documents for snippets. `our@NVIDIATwitter account,NVIDIA Facebookpage` was three adjacent links whose brackets were deleted rather than replaced, and `[^)]*` stopped at the first bracket of `Betreuung_(Recht)`, stranding the reader's quoted title mid-sentence. Bare URLs and footnote anchors go too: `reviewAnswer` reads every URL in a tool result as a citable source, so a `#cite_note` anchor in a passage becomes a source that states nothing. Co-authored-by: Sebastian --- README.md | 14 ++++ src/tools/research.test.ts | 163 ++++++++++++++++++++++++++++++++++- src/tools/research.ts | 168 ++++++++++++++++++++++++++++++++----- 3 files changed, 324 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index cc988cb..673feca 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,20 @@ Which parts are quoted is decided lexically, for the same reason [skill retrieva **That weighting has to be pooled across the sources, and finding out why is what the tests pin.** Measured per page, _who is the chief executive of the airline_ scored the paragraph containing "the airline" exactly level with the one naming the chief executive: within four paragraphs `the` is as rare as `executive`. Across the hundred-odd paragraphs five pages actually produce, `the` appears in nearly all of them and ends up worth about two per cent of `executive`. The mechanism only works at the scale it runs at, so `research.test.ts` fixtures are pages rather than snippets. +**What a page offers and what a page contains are not the same thing, and the live web is what settled that.** The first run of this against real pages produced five sources whose passages were largely furniture, and each failure is now a filter with the observed string as its test fixture: + +| Quoted instead of the answer | Why it won | +| ------------------------------------------------------------ | ---------------------------------------------------------------- | +| `These cookies may store a unique ID…` — two of five sources | A consent notice is several sentences and names its own site | +| Wikipedia's reference list, `↑ Retrieved December 24, 2024` | Citations repeat the subject once per entry, so they outscore it | +| `Sucuri WebSite Firewall - Access Denied` | The reader answers 200, so nothing upstream saw a failure | +| `Sie befinden sich hier … \| Startseite` | A breadcrumb clears every length floor | +| `Who leads NVIDIA?` | A heading is the densest window a paragraph contains | + +The first two are word lists, and the cost is stated rather than hidden: a paragraph genuinely about cookies is dropped with the banners. The third pairs the wording against a length, because an article about Cloudflare is long and a page refusing to serve one is not. The fourth needs no word list at all — prose ends in a full stop and menus do not, which works in either language. The fifth is a floor on how short a passage may be to win on density. + +Two more came from **removing markup without leaving a separator behind**, which is the trap `unbold` in `web.ts` already documents for search snippets. `our@NVIDIATwitter account,NVIDIA Facebookpage` was three adjacent links whose brackets were deleted rather than replaced, and `[^)]*` stopped at the first bracket of `Betreuung_(Recht)` and left `"Betreuung (Recht)")` stranded mid-sentence. Bare URLs go too, footnote anchors included: `reviewAnswer` reads every URL in a tool result as a source the answer may cite, so a `#cite_note` anchor left in a passage becomes a citable source that states nothing. + Three more things follow from what a source is worth: - **Many sources has to mean many _different_ ones.** A search for a news story returns four pages of the same newspaper, and reading all four spends four reader requests to hear one newsroom repeat itself. The results are reordered so the first hit on each host comes first — reordered rather than filtered, because Wikipedia's results are all one host, and there the list refills with further articles instead of collapsing to a single source. diff --git a/src/tools/research.test.ts b/src/tools/research.test.ts index 257c73c..a0ca6bd 100644 --- a/src/tools/research.test.ts +++ b/src/tools/research.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from 'vitest' -import { digest, diverseFirst, paragraphsOf, passagesFor, researchQuestion } from './research' +import { digest, diverseFirst, looksBlocked, paragraphsOf, passagesFor, researchQuestion } from './research' import type { SearchResult, WebAccessConfig } from './web' function result(url: string, title = 'Title', snippet = 'A snippet.'): SearchResult { @@ -86,6 +86,120 @@ describe('paragraphsOf', () => { expect(paragraphsOf('Home\n\nBy our reporter\n\nShare')).toEqual([]) }) + it('strips a footnote link without leaving the anchor behind as a source', () => { + // Observed on Wikipedia. `reviewAnswer` reads every URL in a tool result as a + // source the answer may cite, so a citation anchor left in a passage becomes + // a citable source that states nothing. + const [paragraph] = paragraphsOf( + 'Huang has been Nvidia\'s chief executive for three decades, a tenure described as "almost unheard of".[[47]](https://en.wikipedia.org/wiki/Jensen_Huang#cite_note-fitch20240226-50) He owns 3.6% of Nvidia.', + ) + + expect(paragraph).not.toContain('http') + expect(paragraph).not.toContain('cite_note') + // The number goes with it: a bare `47` mid-sentence reads as part of the claim. + expect(paragraph).toBe( + 'Huang has been Nvidia\'s chief executive for three decades, a tenure described as "almost unheard of". He owns 3.6% of Nvidia.', + ) + }) + + it('strips emphasis and a heading mark that survived being joined into a line', () => { + const [paragraph] = paragraphsOf( + '### Sommer-Pressekonferenz des Bundeskanzlers\nRede von Bundeskanzler Merz in _Paderborn_ über **Europa** und Deutschland.', + ) + + expect(paragraph).toBe( + 'Sommer-Pressekonferenz des Bundeskanzlers Rede von Bundeskanzler Merz in Paderborn über Europa und Deutschland.', + ) + }) + + /** + * The reader drops the spaces around its own emphasis marks, so deleting them + * fuses the words either side. Observed as `,NVIDIA Facebookpage` in a passage + * that had read `,**NVIDIA Facebook**page`. + */ + it('separates words the reader fused with emphasis rather than joining them', () => { + const [paragraph] = paragraphsOf( + 'We intend to use our **NVIDIA** Twitter account,**NVIDIA Facebook**page and company **blog** as a means of disclosing information about the company.', + ) + + expect(paragraph).toContain('account, NVIDIA Facebook page') + expect(paragraph).not.toContain('Facebookpage') + }) + + /** + * The same fusion from the other direction: a page writes two links with + * nothing between them, so removing the brackets joins what they held. + * `our@NVIDIATwitter account,NVIDIA Facebookpage` was three adjacent links. + */ + it('separates adjacent links instead of running their text together', () => { + const [paragraph] = paragraphsOf( + 'We intend to use our [@NVIDIA](https://x.example/nvidia)[Twitter](https://x.example) account,[NVIDIA Facebook](https://fb.example)page as a means of disclosing information.', + ) + + expect(paragraph).toBe( + 'We intend to use our @NVIDIA Twitter account, NVIDIA Facebook page as a means of disclosing information.', + ) + }) + + it('strips a Wikipedia link whose target carries brackets of its own', () => { + // `[^)]*` stops at the bracket inside `Betreuung_(Recht)` and leaves the + // reader's quoted title stranded in the passage. + const [paragraph] = paragraphsOf( + 'Auch [Betreuung](https://de.wikipedia.org/wiki/Betreuung_(Recht) "Betreuung (Recht)") oder Unterbringung in einem Krankenhaus würden ihn disqualifizieren.', + ) + + expect(paragraph).toBe( + 'Auch Betreuung oder Unterbringung in einem Krankenhaus würden ihn disqualifizieren.', + ) + }) + + /** + * Nav rows, breadcrumbs and share links clear every length floor and answer + * nothing. Both of these were quoted as sources for *wer ist der Bundeskanzler*. + */ + it('drops a breadcrumb and a share row, which no full stop ends', () => { + expect( + paragraphsOf('Sie befinden sich hier Bundesregierung | Startseite Bundesregierung Bundeskabinett'), + ).toEqual([]) + expect( + paragraphsOf('Governance Management Team Board of Directors Governance Documents Contact the Board'), + ).toEqual([]) + }) + + /** + * Both of these outranked the sentence that answered the question when this ran + * against the live web, and they are why the filters exist rather than being a + * precaution. + */ + it('drops a consent notice, which is prose and answers nothing', () => { + expect( + paragraphsOf( + 'These cookies may store a unique ID so that our system will remember you when you return, and are used to improve website performance.', + ), + ).toEqual([]) + expect( + paragraphsOf( + 'Das Tool verwendet Cookies. Mit diesen Cookies können wir Besuche zählen und die Nutzung der Seite auswerten.', + ), + ).toEqual([]) + }) + + it('drops a citation list, which repeats the subject in every entry', () => { + expect( + paragraphsOf( + '"Here\'s how Nvidia CEO Jensen Huang won over his wife". Business Insider. Retrieved December 24, 2024. ↑ "#61 Jen-Hsun Huang". Forbes. Archived from the original on May 9, 2008.', + ), + ).toEqual([]) + }) + + it('keeps prose that merely mentions a year and a source', () => { + const kept = paragraphsOf( + 'Jensen Huang founded NVIDIA in 1993 and has served since its inception as president and chief executive officer.', + ) + + expect(kept).toHaveLength(1) + }) + it('joins the lines of one paragraph and separates two', () => { expect( paragraphsOf( @@ -98,6 +212,34 @@ describe('paragraphsOf', () => { }) }) +describe('looksBlocked', () => { + // The reader answers 200 with these, so nothing upstream can tell them from a + // page. One was quoted as a source for *wer ist der Bundeskanzler*. + it('recognises a firewall page the reader returned as a result', () => { + expect( + looksBlocked( + 'Sucuri WebSite Firewall - Access Denied', + 'Access Denied - Sucuri Website Firewall\n\nTime:2026-08-25 17:56:05 Server ID:20017', + ), + ).toBe(true) + }) + + it('recognises a JavaScript gate', () => { + expect(looksBlocked('Just a moment...', 'Enable JavaScript and cookies to continue')).toBe(true) + }) + + /** Length is half the test, or an article about Cloudflare would be discarded. */ + it('leaves a long article that happens to name a firewall vendor', () => { + const article = `Cloudflare reported revenue growth this quarter. ${'The company operates a global network. '.repeat(40)}` + + expect(looksBlocked('Cloudflare earnings', article)).toBe(false) + }) + + it('leaves an ordinary short page alone', () => { + expect(looksBlocked('Leadership', 'Ama Osei has led the airline since 2023.')).toBe(false) + }) +}) + describe('passagesFor', () => { /** * As many paragraphs as a real page has, and that is the point rather than the @@ -158,6 +300,25 @@ describe('passagesFor', () => { ) }) + /** + * The densest window is often a heading. Asked who runs Nvidia, this returned + * the FAQ question "Who leads NVIDIA?" — every word of it earning, and the + * answer underneath it left out. + */ + it('does not shrink a passage to a heading just because it scores densely', () => { + const faq = [ + 'Who leads NVIDIA?', + 'Jensen Huang founded NVIDIA in 1993 and has served since its inception as president and chief executive officer of the company.', + 'Who is part of the NVIDIA executive team?', + 'The executive staff includes Colette Kress as chief financial officer and Debora Shoquist in operations, alongside several others.', + ].join(' ') + + const [first] = forOnePage('Who leads NVIDIA?', faq) + + expect(first!.length).toBeGreaterThanOrEqual(60) + expect(first).toContain('Jensen Huang') + }) + it('cuts a long paragraph down to the sentences that carry the question', () => { const filler = 'The catering was reviewed in a report nobody read. '.repeat(6) const [first] = forOnePage( diff --git a/src/tools/research.ts b/src/tools/research.ts index 5bcb2fe..71a7b3f 100644 --- a/src/tools/research.ts +++ b/src/tools/research.ts @@ -72,6 +72,15 @@ function collapse(value: string): string { return value.replace(/\s+/g, ' ').trim() } +/** + * Closes the gap the separators above open in front of punctuation. A quote is + * only known to be a closing one when punctuation follows it, so that is the + * only case where a space in front of one is removed. + */ +function tidy(value: string): string { + return value.replace(/\s+(["'”’][.,;:!?])/g, '$1').replace(/\s+([.,;:!?])/g, '$1') +} + function hostOf(url: string): string { try { return new URL(url).hostname.replace(/^www\./, '').toLowerCase() @@ -114,30 +123,126 @@ export function diverseFirst(results: SearchResult[], max: number): SearchResult } const IMAGE = /!\[[^\]]*\]\([^)]*\)/g -const LINK = /\[([^\]]*)\]\([^)]*\)/g -const LINE_FURNITURE = /^\s*(?:#{1,6}\s+|[-*+]\s+|>\s+|\d+\.\s+)/ + +/** + * A markdown link target, tolerating one level of nesting inside it. + * + * `[^)]*` is the obvious pattern and stops at the first bracket of + * `Betreuung_(Recht)`, which left `"Betreuung (Recht)")` sitting in a German + * Wikipedia passage. Wikipedia URLs carry parenthesised disambiguators and the + * reader adds a quoted title beside them, so both have to survive being matched. + */ +const LINK_TARGET = /\]\((?:[^()]|\([^()]*\))*\)/g + +/** What is left of `[47]` once its target is gone: a footnote number mid-sentence. */ +const FOOTNOTE = /\[\d+\]/g + +const BARE_URL = /https?:\/\/\S+/g +const HEADING_MARK = /#{1,6}\s+/g +const EMPHASIS = /[*_`]+/g +const LINE_FURNITURE = /^\s*(?:[-*+]\s+|>\s+|\d+\.\s+)/ + +/** + * Prose ends in a full stop. Menus, breadcrumbs and share rows do not, and they + * clear every length floor: *Sie befinden sich hier Bundesregierung | Startseite + * Bundeskabinett Bundeskanzler* was quoted as a source on what a chancellor is. + * + * Cheaper and less parochial than naming the furniture — it needs no word list + * and works in either language. Trailing quotes and brackets are allowed through + * because a paragraph often ends inside them. + */ +const ENDS_A_SENTENCE = /[.!?…][)"'”’]*$/ + +/** + * Boilerplate that reads exactly like prose and answers nothing. + * + * Not a nicety. Run against the live web, two of five sources for *who is the + * chief executive of Nvidia* came back quoting consent notices — "these cookies + * may store a unique ID", "das Tool verwendet Cookies" — because a cookie banner + * is several sentences long and mentions the site it is on, which is all the + * scoring has to go on. + * + * The cost is that a paragraph genuinely about cookies or a privacy policy is + * dropped with them. That is the right way round: this tool is asked who runs a + * company far more often than it is asked what an HTTP cookie is, and the + * failure it prevents was happening on most commercial sites. + */ +const BOILERPLATE = + /\bcookies?\b|\bconsent\b|\bnewsletter\b|\bsubscribe\b|\bprivacy policy\b|\bterms of (use|service)\b|\ball rights reserved\b|\bdatenschutz\b|\beinwilligung\b|\bnutzungsbedingungen\b/i + +/** + * A citation list, which scores well and states nothing. + * + * Wikipedia's references were the top-ranked paragraph for *who is the chief + * executive of Nvidia*: they repeat the subject's name in every entry, so they + * out-score the sentence that answers the question. + */ +const REFERENCE_LIST = + /↑|\bretrieved\b\s+\w+\s+\d{1,2},?\s+\d{4}|\barchived from the original\b|\babgerufen am\b/i /** * Reads the reader's markdown back into candidate paragraphs. * - * Link text is kept and the target dropped: a paragraph is prose whether or not - * the names in it were linked, and keeping the URLs would put addresses no tool - * returned in front of a model that is checked for citing exactly those. + * Link text is kept and the target dropped, and any bare URL goes with it. That + * second part is not tidiness: `reviewAnswer` treats every URL in a tool result + * as a source the answer may cite, so a footnote anchor left in a passage would + * become a citable source that says nothing. A Wikipedia passage arrived carrying + * `#cite_note-fitch20240226-50`, which is exactly that. */ export function paragraphsOf(markdown: string): string[] { - return markdown - .replace(IMAGE, ' ') - .replace(LINK, '$1') - .split(/\n\s*\n/) - .map((block) => - collapse( - block - .split('\n') - .map((line) => line.replace(LINE_FURNITURE, '')) - .join(' '), - ), - ) - .filter((block) => block.length >= MIN_PASSAGE_CHARS && words(block).length >= MIN_PASSAGE_WORDS) + return ( + markdown + .replace(IMAGE, ' ') + .replace(LINK_TARGET, ']') + .replace(FOOTNOTE, '') + // Whatever brackets are left were a link's text. A nested `[[47]](url)` is why + // this strips them rather than matching a whole link in one pattern. + // + // A space, not nothing, and the same for the emphasis marks below: a page + // writes two links with nothing between them, so deleting the brackets fuses + // what they held. `our@NVIDIATwitter account,NVIDIA Facebookpage` was three + // adjacent links, and it is the trap `unbold` in `web.ts` already documents. + .replace(/[[\]]/g, ' ') + .replace(BARE_URL, ' ') + .split(/\n\s*\n/) + .map((block) => + tidy( + collapse( + block + .split('\n') + .map((line) => line.replace(LINE_FURNITURE, '')) + .join(' ') + .replace(HEADING_MARK, ' ') + .replace(EMPHASIS, ' '), + ), + ), + ) + .filter( + (block) => + block.length >= MIN_PASSAGE_CHARS && + words(block).length >= MIN_PASSAGE_WORDS && + ENDS_A_SENTENCE.test(block) && + !BOILERPLATE.test(block) && + !REFERENCE_LIST.test(block), + ) + ) +} + +/** + * What a firewall or a JavaScript gate serves instead of the page. + * + * The reader answers 200 with it, so nothing upstream can tell it from a result: + * *wer ist der Bundeskanzler* came back with "Sucuri WebSite Firewall — Access + * Denied" quoted as one of five sources. Length is half the test, because an + * article about Cloudflare is long and a page refusing to serve one is not. + */ +const BLOCKED = + /access denied|attention required|just a moment|enable javascript|are you a robot|verify you are human|cloudflare|sucuri|forbidden|zugriff verweigert/i + +const BLOCK_PAGE_CHARS = 1200 + +export function looksBlocked(title: string, text: string): boolean { + return text.length < BLOCK_PAGE_CHARS && BLOCKED.test(`${title} ${text.slice(0, 300)}`) } /** BM25's idf without the length normalisation, as `skills/retrieve.ts` uses it. */ @@ -184,6 +289,11 @@ const SENTENCE_END = /(?<=[.!?…])\s+/ * question, so a long page contributes its relevant lines rather than its first * ones. A single sentence over the cap is cut mid-way, which is the one case * where this cannot avoid it. + * + * A window has to clear `MIN_PASSAGE_CHARS` to win on score alone, because + * scoring rewards density and the densest window is often a heading. Asked who + * runs Nvidia, this returned the FAQ question "Who leads NVIDIA?" — every word of + * it earning, and the answer beneath it left out. */ function condense(paragraph: string, weights: Map): string { if (paragraph.length <= MAX_PASSAGE_CHARS) return paragraph @@ -191,6 +301,8 @@ function condense(paragraph: string, weights: Map): string { const sentences = paragraph.split(SENTENCE_END) let best = '' let bestScore = -1 + let substantial = '' + let substantialScore = -1 for (let start = 0; start < sentences.length; start += 1) { let window = '' @@ -198,15 +310,24 @@ function condense(paragraph: string, weights: Map): string { const extended = window ? `${window} ${sentences[end]}` : (sentences[end] ?? '') if (extended.length > MAX_PASSAGE_CHARS) break window = extended + const windowScore = score(window, weights) if (windowScore > bestScore || (windowScore === bestScore && window.length > best.length)) { best = window bestScore = windowScore } + if ( + window.length >= MIN_PASSAGE_CHARS && + (windowScore > substantialScore || + (windowScore === substantialScore && window.length > substantial.length)) + ) { + substantial = window + substantialScore = windowScore + } } } - return best || `${paragraph.slice(0, MAX_PASSAGE_CHARS).trimEnd()}…` + return substantial || best || `${paragraph.slice(0, MAX_PASSAGE_CHARS).trimEnd()}…` } /** Enough of a passage to recognise the same claim written out twice. */ @@ -324,7 +445,14 @@ export async function researchQuestion(question: string, config: WebAccessConfig const chosen = passagesFor( question, - settled.map((outcome) => (outcome.status === 'fulfilled' ? paragraphsOf(outcome.value.text) : [])), + settled.map((outcome) => { + if (outcome.status !== 'fulfilled') return [] + const { title, text } = outcome.value + // A page the site refused to serve arrives as a 200 with prose on it. Left + // in, it is a source that says nothing and cannot be told from one that + // does; the search snippet for the same URL at least came from the index. + return looksBlocked(title, text) ? [] : paragraphsOf(text) + }), ) const sources = selected.flatMap((result, at): Source[] => {