diff --git a/app/llms-full.txt/route.ts b/app/llms-full.txt/route.ts index 69755d83..905a7285 100644 --- a/app/llms-full.txt/route.ts +++ b/app/llms-full.txt/route.ts @@ -13,6 +13,10 @@ export async function GET() { const scanned = await Promise.all(scan); return new Response(AGENT_INSTRUCTIONS + scanned.join('\n\n'), { - headers: { 'Content-Type': 'text/markdown; charset=utf-8' }, + headers: { + 'Content-Type': 'text/markdown; charset=utf-8', + // A dump of every docs page: fetchable by agents, but not a search result. + 'X-Robots-Tag': 'noindex', + }, }); } diff --git a/app/llms.mdx/[[...slug]]/route.ts b/app/llms.mdx/[[...slug]]/route.ts index 4a1c41d7..3ea37257 100644 --- a/app/llms.mdx/[[...slug]]/route.ts +++ b/app/llms.mdx/[[...slug]]/route.ts @@ -20,7 +20,12 @@ export async function GET(_req: NextRequest, { params }: { params: Promise<{ slu const page = getPage(slug); if (!page) notFound(); - const headers = new Headers({ 'Content-Type': 'text/markdown; charset=utf-8' }); + // This markdown duplicates the canonical HTML page, so keep it out of search + // results. Crawlers may still fetch it: noindex only suppresses indexing. + const headers = new Headers({ + 'Content-Type': 'text/markdown; charset=utf-8', + 'X-Robots-Tag': 'noindex', + }); appendMarkdownVaryHeader(headers); return new NextResponse(await getLLMText(page, { indexPointer: true }), { diff --git a/next.config.mjs b/next.config.mjs index 36cbc579..e26c3c7e 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -220,6 +220,28 @@ const config = { }, ], }, + { + // Plain-text mirrors of the docs. Agents should fetch them; Google + // should not list them next to the HTML pages they duplicate. Matching + // happens on the requested path, so the homepage keeps its own headers + // when middleware rewrites it to /AGENTS.md for markdown clients. + source: "/:path*/llms.txt", + headers: [ + { + key: "X-Robots-Tag", + value: "noindex", + }, + ], + }, + { + source: "/AGENTS.md", + headers: [ + { + key: "X-Robots-Tag", + value: "noindex", + }, + ], + }, { source: "/(.*)", headers: [ diff --git a/tests/e2e/llm-endpoints.test.ts b/tests/e2e/llm-endpoints.test.ts index f2d737f6..50a81755 100644 --- a/tests/e2e/llm-endpoints.test.ts +++ b/tests/e2e/llm-endpoints.test.ts @@ -95,6 +95,34 @@ describe('.md suffix end-to-end', () => { expect(response.headers.get('location')).toBe('/AGENTS.md'); }); + test('marks a .md docs URL noindex while its canonical HTML page stays indexable', async () => { + const markdown = await fetch(`${BASE_URL}/overview/sessions-api/quickstart.md`, { + headers: BROWSER_HEADERS, + }); + expect(markdown.headers.get('x-robots-tag')).toContain('noindex'); + + const html = await fetch(`${BASE_URL}/overview/sessions-api/quickstart`, { + headers: BROWSER_HEADERS, + }); + expect(html.headers.get('x-robots-tag')).toBeNull(); + }); + + test('marks the plain-text agent endpoints noindex', async () => { + // Status is not asserted: /llms.txt is generated into public/ at build + // time, so it is absent when tests run. The header comes from the route + // config either way, which is what matters for crawlers. + for (const path of ['/llms.txt', '/llms-full.txt', '/AGENTS.md']) { + const response = await fetch(`${BASE_URL}${path}`, { headers: BROWSER_HEADERS }); + expect(response.headers.get('x-robots-tag')).toContain('noindex'); + } + }); + + test('keeps the homepage indexable when it negotiates markdown', async () => { + const response = await fetch(BASE_URL, { headers: { accept: 'text/markdown' } }); + expect(response.headers.get('content-type')).toStartWith('text/markdown'); + expect(response.headers.get('x-robots-tag')).toBeNull(); + }); + test('llms-full.txt does not repeat the index pointer', async () => { const response = await fetch(`${BASE_URL}/llms-full.txt`, { headers: BROWSER_HEADERS }); expect(response.status).toBe(200);