diff --git a/middleware.ts b/middleware.ts index 315ae5c5..f9a91926 100644 --- a/middleware.ts +++ b/middleware.ts @@ -38,10 +38,13 @@ export default function middleware(request: NextRequest) { const wantsMarkdown = shouldServeMarkdown(request.headers); + // The homepage has no MDX page behind it, so markdown clients get the agent + // guide, which opens with a pointer to the /llms.txt index. Serving it in + // place keeps negotiation on the requested URL instead of redirecting away. if (pathname === '/' && (wantsMarkdown || isProgrammaticClient(request))) { - const redirectUrl = request.nextUrl.clone(); - redirectUrl.pathname = '/llms.txt'; - return withMarkdownVary(NextResponse.redirect(redirectUrl)); + const rewriteUrl = request.nextUrl.clone(); + rewriteUrl.pathname = '/AGENTS.md'; + return withMarkdownVary(NextResponse.rewrite(rewriteUrl)); } if (!isNegotiableDocsPath(pathname)) { diff --git a/tests/e2e/llm-endpoints.test.ts b/tests/e2e/llm-endpoints.test.ts index f6d53e5b..fca93528 100644 --- a/tests/e2e/llm-endpoints.test.ts +++ b/tests/e2e/llm-endpoints.test.ts @@ -101,6 +101,16 @@ describe('.md suffix end-to-end', () => { expect(await response.text()).not.toContain('Full docs index'); }); + test('negotiates markdown at the homepage without redirecting', async () => { + const response = await fetch(BASE_URL, { + headers: { accept: 'text/markdown' }, + redirect: 'manual', + }); + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toStartWith('text/markdown'); + expect(await response.text()).toStartWith('> Full docs index: https://docs.steel.dev/llms.txt'); + }); + test('returns 404 for a .md URL with no matching page', async () => { const response = await fetch(`${BASE_URL}/nonexistent-page.md`, { headers: BROWSER_HEADERS, diff --git a/tests/middleware.test.ts b/tests/middleware.test.ts index 6f2a0a91..cc6eea27 100644 --- a/tests/middleware.test.ts +++ b/tests/middleware.test.ts @@ -42,6 +42,35 @@ describe('middleware .md suffix handling', () => { expect(response.headers.get('x-middleware-rewrite')).toBeNull(); }); + test('serves the homepage as markdown in place, without a redirect', () => { + const response = middleware( + new NextRequest('http://localhost/', { headers: { accept: 'text/markdown' } }), + ); + const rewrite = response.headers.get('x-middleware-rewrite'); + expect(rewrite).not.toBeNull(); + expect(new URL(rewrite as string).pathname).toBe('/AGENTS.md'); + expect(response.status).toBe(200); + expect(response.headers.get('location')).toBeNull(); + expect(response.headers.get('Vary')).toContain('Accept'); + }); + + test('serves the homepage as markdown to programmatic clients', () => { + const response = middleware( + new NextRequest('http://localhost/', { + headers: { accept: 'text/html', 'user-agent': 'curl/8.7.1' }, + }), + ); + const rewrite = response.headers.get('x-middleware-rewrite'); + expect(rewrite).not.toBeNull(); + expect(new URL(rewrite as string).pathname).toBe('/AGENTS.md'); + }); + + test('leaves the homepage untouched for browsers', () => { + const response = middleware(browserRequest('http://localhost/')); + expect(response.headers.get('x-middleware-rewrite')).toBeNull(); + expect(response.headers.get('location')).toBeNull(); + }); + test('does not rewrite /AGENTS.md, even for markdown user agents', () => { const browser = middleware(browserRequest('http://localhost/AGENTS.md')); expect(browser.headers.get('x-middleware-rewrite')).toBeNull();