From 0b2b5aabfa77809cf5ed5afc9843d0382ad91027 Mon Sep 17 00:00:00 2001 From: Nikola Balic Date: Wed, 29 Jul 2026 20:48:24 +0200 Subject: [PATCH] feat(llms): serve an /AGENTS.md agent guide Add a /AGENTS.md route that serves the shared AGENT_INSTRUCTIONS block (install, auth config, and usage code) as markdown with the llms.txt index pointer prepended. This satisfies the AGENTS.md convention and Vercel's agent-readability check; until now /AGENTS.md 404'd. The markdown-negotiation middleware previously rewrote /AGENTS.md to /llms.mdx/AGENTS (stripping the .md suffix), so /AGENTS and /AGENTS.md are now excluded from negotiation to let the route handler own the request. A /.well-known/agents.md -> /AGENTS.md permanent redirect covers the alternate location, mirroring the existing llms.txt one. --- app/AGENTS.md/route.ts | 12 ++++++++++++ lib/agent-instructions.ts | 4 ++-- lib/markdown-negotiation.ts | 2 ++ next.config.mjs | 5 +++++ tests/e2e/llm-endpoints.test.ts | 20 ++++++++++++++++++++ tests/markdown-negotiation.test.ts | 4 ++++ tests/middleware.test.ts | 11 +++++++++++ 7 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 app/AGENTS.md/route.ts diff --git a/app/AGENTS.md/route.ts b/app/AGENTS.md/route.ts new file mode 100644 index 00000000..a609eef3 --- /dev/null +++ b/app/AGENTS.md/route.ts @@ -0,0 +1,12 @@ +// ABOUTME: Serves the /AGENTS.md agent guide as markdown. +// ABOUTME: Content is the shared AGENT_INSTRUCTIONS block with the llms.txt index pointer. +import { AGENT_INSTRUCTIONS } from '@/lib/agent-instructions'; +import { LLMS_INDEX_POINTER } from '@/lib/get-llm-text'; + +export const revalidate = false; + +export async function GET() { + return new Response(`${LLMS_INDEX_POINTER}\n\n${AGENT_INSTRUCTIONS}`, { + headers: { 'Content-Type': 'text/markdown; charset=utf-8' }, + }); +} diff --git a/lib/agent-instructions.ts b/lib/agent-instructions.ts index 39c7ce1d..b0663ba7 100644 --- a/lib/agent-instructions.ts +++ b/lib/agent-instructions.ts @@ -1,5 +1,5 @@ -// ABOUTME: Quick reference and gotchas prepended to the agent-facing markdown -// ABOUTME: bundles, shared by public/llms.txt generation and the /llms-full.txt route. +// ABOUTME: Quick reference and gotchas served to agents via the /llms.txt index, +// ABOUTME: the /llms-full.txt bundle, and the /AGENTS.md route. export const AGENT_INSTRUCTIONS = `# Steel Documentation diff --git a/lib/markdown-negotiation.ts b/lib/markdown-negotiation.ts index 62981827..b8bc1f78 100644 --- a/lib/markdown-negotiation.ts +++ b/lib/markdown-negotiation.ts @@ -32,6 +32,8 @@ const MARKDOWN_VARY_HEADERS = ['Accept', 'User-Agent']; const EXCLUDED_EXACT_PATHS = new Set([ '/.well-known/llms.txt', + '/AGENTS', + '/AGENTS.md', '/favicon.ico', '/llms-full.txt', '/llms.txt', diff --git a/next.config.mjs b/next.config.mjs index ca0eee76..47d0341f 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -24,6 +24,11 @@ const config = { destination: "/llms.txt", permanent: true, // 301 redirect - tells crawlers this is the canonical location }, + { + source: "/.well-known/agents.md", + destination: "/AGENTS.md", + permanent: true, + }, { source: "/overview/llms-full.txt", destination: "/llms-full.txt", diff --git a/tests/e2e/llm-endpoints.test.ts b/tests/e2e/llm-endpoints.test.ts index 39114951..f6d53e5b 100644 --- a/tests/e2e/llm-endpoints.test.ts +++ b/tests/e2e/llm-endpoints.test.ts @@ -75,6 +75,26 @@ describe('.md suffix end-to-end', () => { expect(body).not.toContain(':::callout'); }); + test('serves /AGENTS.md as markdown with install, auth, and the index pointer', async () => { + const response = await fetch(`${BASE_URL}/AGENTS.md`, { headers: BROWSER_HEADERS }); + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toStartWith('text/markdown'); + const body = await response.text(); + expect(body).toStartWith('> Full docs index: https://docs.steel.dev/llms.txt'); + expect(body).toContain('npm install steel-sdk'); + expect(body).toContain('STEEL_API_KEY'); + }); + + test('redirects /.well-known/agents.md to /AGENTS.md', async () => { + const response = await fetch(`${BASE_URL}/.well-known/agents.md`, { + headers: BROWSER_HEADERS, + redirect: 'manual', + }); + // Next.js renders permanent config redirects as 308. + expect(response.status).toBe(308); + expect(response.headers.get('location')).toBe('/AGENTS.md'); + }); + test('llms-full.txt does not repeat the index pointer', async () => { const response = await fetch(`${BASE_URL}/llms-full.txt`, { headers: BROWSER_HEADERS }); expect(response.status).toBe(200); diff --git a/tests/markdown-negotiation.test.ts b/tests/markdown-negotiation.test.ts index ab034cc1..aec495c6 100644 --- a/tests/markdown-negotiation.test.ts +++ b/tests/markdown-negotiation.test.ts @@ -28,6 +28,10 @@ describe('resolveMarkdownPath', () => { expect(resolveMarkdownPath('/llms.txt.md')).toBeNull(); }); + test('returns null for /AGENTS.md so its route handler owns the request', () => { + expect(resolveMarkdownPath('/AGENTS.md')).toBeNull(); + }); + test('returns null when the stripped path is under an excluded prefix', () => { expect(resolveMarkdownPath('/llms.mdx/overview.md')).toBeNull(); expect(resolveMarkdownPath('/api/search.md')).toBeNull(); diff --git a/tests/middleware.test.ts b/tests/middleware.test.ts index b1415e5a..6f2a0a91 100644 --- a/tests/middleware.test.ts +++ b/tests/middleware.test.ts @@ -41,4 +41,15 @@ describe('middleware .md suffix handling', () => { const response = middleware(browserRequest('http://localhost/llms-full.txt.md')); expect(response.headers.get('x-middleware-rewrite')).toBeNull(); }); + + test('does not rewrite /AGENTS.md, even for markdown user agents', () => { + const browser = middleware(browserRequest('http://localhost/AGENTS.md')); + expect(browser.headers.get('x-middleware-rewrite')).toBeNull(); + const agent = middleware( + new NextRequest('http://localhost/AGENTS.md', { + headers: { 'user-agent': 'claude-code/1.0' }, + }), + ); + expect(agent.headers.get('x-middleware-rewrite')).toBeNull(); + }); });