From 4ec65bc9e9c4a8f025eaced146934eb8dfee73e9 Mon Sep 17 00:00:00 2001
From: Nikola Balic <wave@nibzard.com>
Date: Thu, 30 Jul 2026 14:56:05 +0200
Subject: [PATCH] feat(seo): noindex LLM text endpoints

The plain-text mirrors of the docs were competing with the HTML pages they
duplicate: /llms-full.txt alone earned 37,527 Search impressions (18% of the
site's total) at position 17.9 with a 0.03% CTR, which drags the whole site's
CTR baseline down and puts a raw text dump in front of human searchers.

Send X-Robots-Tag: noindex on every alternate-format endpoint: /llms.txt and
its per-section copies plus /AGENTS.md via headers() in next.config.mjs, and
/llms-full.txt plus the /llms.mdx route (which also backs every .md-suffixed
docs URL) from the route handlers that build those responses. Crawling is
untouched: robots.txt still allows everything, so agents fetch these as before
and only indexing is suppressed.

Header matching happens on the requested path, so the homepage keeps its own
headers when middleware rewrites it to /AGENTS.md for markdown clients. No HTML
page gets noindex.
---
 app/llms-full.txt/route.ts        |  6 +++++-
 app/llms.mdx/[[...slug]]/route.ts |  7 ++++++-
 next.config.mjs                   | 22 ++++++++++++++++++++++
 tests/e2e/llm-endpoints.test.ts   | 28 ++++++++++++++++++++++++++++
 4 files changed, 61 insertions(+), 2 deletions(-)

diff --git a/app/llms-full.txt/route.ts b/app/llms-full.txt/route.ts
index 69755d83..905a7285 100644
--- a/app/llms-full.txt/route.ts
+++ b/app/llms-full.txt/route.ts
@@ -13,6 +13,10 @@ export async function GET() {
   const scanned = await Promise.all(scan);
 
   return new Response(AGENT_INSTRUCTIONS + scanned.join('\n\n'), {
-    headers: { 'Content-Type': 'text/markdown; charset=utf-8' },
+    headers: {
+      'Content-Type': 'text/markdown; charset=utf-8',
+      // A dump of every docs page: fetchable by agents, but not a search result.
+      'X-Robots-Tag': 'noindex',
+    },
   });
 }
diff --git a/app/llms.mdx/[[...slug]]/route.ts b/app/llms.mdx/[[...slug]]/route.ts
index 4a1c41d7..3ea37257 100644
--- a/app/llms.mdx/[[...slug]]/route.ts
+++ b/app/llms.mdx/[[...slug]]/route.ts
@@ -20,7 +20,12 @@ export async function GET(_req: NextRequest, { params }: { params: Promise<{ slu
   const page = getPage(slug);
   if (!page) notFound();
 
-  const headers = new Headers({ 'Content-Type': 'text/markdown; charset=utf-8' });
+  // This markdown duplicates the canonical HTML page, so keep it out of search
+  // results. Crawlers may still fetch it: noindex only suppresses indexing.
+  const headers = new Headers({
+    'Content-Type': 'text/markdown; charset=utf-8',
+    'X-Robots-Tag': 'noindex',
+  });
   appendMarkdownVaryHeader(headers);
 
   return new NextResponse(await getLLMText(page, { indexPointer: true }), {
diff --git a/next.config.mjs b/next.config.mjs
index 36cbc579..e26c3c7e 100644
--- a/next.config.mjs
+++ b/next.config.mjs
@@ -220,6 +220,28 @@ const config = {
           },
         ],
       },
+      {
+        // Plain-text mirrors of the docs. Agents should fetch them; Google
+        // should not list them next to the HTML pages they duplicate. Matching
+        // happens on the requested path, so the homepage keeps its own headers
+        // when middleware rewrites it to /AGENTS.md for markdown clients.
+        source: "/:path*/llms.txt",
+        headers: [
+          {
+            key: "X-Robots-Tag",
+            value: "noindex",
+          },
+        ],
+      },
+      {
+        source: "/AGENTS.md",
+        headers: [
+          {
+            key: "X-Robots-Tag",
+            value: "noindex",
+          },
+        ],
+      },
       {
         source: "/(.*)",
         headers: [
diff --git a/tests/e2e/llm-endpoints.test.ts b/tests/e2e/llm-endpoints.test.ts
index f2d737f6..50a81755 100644
--- a/tests/e2e/llm-endpoints.test.ts
+++ b/tests/e2e/llm-endpoints.test.ts
@@ -95,6 +95,34 @@ describe('.md suffix end-to-end', () => {
     expect(response.headers.get('location')).toBe('/AGENTS.md');
   });
 
+  test('marks a .md docs URL noindex while its canonical HTML page stays indexable', async () => {
+    const markdown = await fetch(`${BASE_URL}/overview/sessions-api/quickstart.md`, {
+      headers: BROWSER_HEADERS,
+    });
+    expect(markdown.headers.get('x-robots-tag')).toContain('noindex');
+
+    const html = await fetch(`${BASE_URL}/overview/sessions-api/quickstart`, {
+      headers: BROWSER_HEADERS,
+    });
+    expect(html.headers.get('x-robots-tag')).toBeNull();
+  });
+
+  test('marks the plain-text agent endpoints noindex', async () => {
+    // Status is not asserted: /llms.txt is generated into public/ at build
+    // time, so it is absent when tests run. The header comes from the route
+    // config either way, which is what matters for crawlers.
+    for (const path of ['/llms.txt', '/llms-full.txt', '/AGENTS.md']) {
+      const response = await fetch(`${BASE_URL}${path}`, { headers: BROWSER_HEADERS });
+      expect(response.headers.get('x-robots-tag')).toContain('noindex');
+    }
+  });
+
+  test('keeps the homepage indexable when it negotiates markdown', async () => {
+    const response = await fetch(BASE_URL, { headers: { accept: 'text/markdown' } });
+    expect(response.headers.get('content-type')).toStartWith('text/markdown');
+    expect(response.headers.get('x-robots-tag')).toBeNull();
+  });
+
   test('llms-full.txt does not repeat the index pointer', async () => {
     const response = await fetch(`${BASE_URL}/llms-full.txt`, { headers: BROWSER_HEADERS });
     expect(response.status).toBe(200);