diff --git a/content/docs/cookbook/authors/aspectrr.mdx b/content/docs/cookbook/authors/aspectrr.mdx index 7876b1e5..6ae304ac 100644 --- a/content/docs/cookbook/authors/aspectrr.mdx +++ b/content/docs/cookbook/authors/aspectrr.mdx @@ -1,6 +1,7 @@ --- title: Collin Pfeifer description: 3 recipes contributed to the Steel Cookbook by Collin Pfeifer. +llm: false --- diff --git a/content/docs/cookbook/authors/bsparker.mdx b/content/docs/cookbook/authors/bsparker.mdx index 48deb13e..19abd8e3 100644 --- a/content/docs/cookbook/authors/bsparker.mdx +++ b/content/docs/cookbook/authors/bsparker.mdx @@ -1,6 +1,7 @@ --- title: Brian Sparker description: 1 recipe contributed to the Steel Cookbook by Brian Sparker. +llm: false --- diff --git a/content/docs/cookbook/authors/danew.mdx b/content/docs/cookbook/authors/danew.mdx index 9d34e563..84acf091 100644 --- a/content/docs/cookbook/authors/danew.mdx +++ b/content/docs/cookbook/authors/danew.mdx @@ -1,6 +1,7 @@ --- title: Dane Wilson description: 1 recipe contributed to the Steel Cookbook by Dane Wilson. +llm: false --- diff --git a/content/docs/cookbook/authors/hussufo.mdx b/content/docs/cookbook/authors/hussufo.mdx index 569eefc0..93bdece7 100644 --- a/content/docs/cookbook/authors/hussufo.mdx +++ b/content/docs/cookbook/authors/hussufo.mdx @@ -1,6 +1,7 @@ --- title: Hussien Hussien description: 4 recipes contributed to the Steel Cookbook by Hussien Hussien. +llm: false --- diff --git a/content/docs/cookbook/authors/jagadeshjai.mdx b/content/docs/cookbook/authors/jagadeshjai.mdx index c20b8bb7..dd38a828 100644 --- a/content/docs/cookbook/authors/jagadeshjai.mdx +++ b/content/docs/cookbook/authors/jagadeshjai.mdx @@ -1,6 +1,7 @@ --- title: Jagadesh Jai description: 4 recipes contributed to the Steel Cookbook by Jagadesh Jai. +llm: false --- diff --git a/content/docs/cookbook/authors/junhsss.mdx b/content/docs/cookbook/authors/junhsss.mdx index f4602527..8f6b5194 100644 --- a/content/docs/cookbook/authors/junhsss.mdx +++ b/content/docs/cookbook/authors/junhsss.mdx @@ -1,6 +1,7 @@ --- title: Jun Ryu description: 43 recipes contributed to the Steel Cookbook by Jun Ryu. +llm: false --- diff --git a/content/docs/cookbook/authors/nibzard.mdx b/content/docs/cookbook/authors/nibzard.mdx index 0e1f9639..64690e0c 100644 --- a/content/docs/cookbook/authors/nibzard.mdx +++ b/content/docs/cookbook/authors/nibzard.mdx @@ -1,6 +1,7 @@ --- title: Nikola Balic description: 3 recipes contributed to the Steel Cookbook by Nikola Balic. +llm: false --- diff --git a/lib/agent-instructions.ts b/lib/agent-instructions.ts index 3fd851ff..39c7ce1d 100644 --- a/lib/agent-instructions.ts +++ b/lib/agent-instructions.ts @@ -65,6 +65,7 @@ export const AGENT_INSTRUCTIONS = `# Steel Documentation - The Python package installs as \`pip install steel-sdk\` but imports as \`from steel import Steel\` - \`sessions.create()\` accepts an optional \`sessionId\` (Node) / \`session_id\` (Python) UUID when you need the ID before the session exists; omit it and Steel generates one - The Python \`sessions.create()\` session timeout param is \`api_timeout\` (not \`timeout\`, which is the HTTP request timeout) -- Individual doc pages are available as markdown at \`/llms.mdx/\` +- Any docs page is available as markdown by appending \`.md\` to its URL, for example \`https://docs.steel.dev/overview/steel-cli.md\`; AI user agents (Claude, Cursor, GPT) receive markdown automatically at the canonical URL +- \`/llms-full.txt\` concatenates every page into one file (large, roughly 230k tokens); prefer fetching the individual \`.md\` pages you need over reading the whole bundle `; diff --git a/scripts/sync-cookbook.ts b/scripts/sync-cookbook.ts index 68e0d7c9..dfc7ad7d 100644 --- a/scripts/sync-cookbook.ts +++ b/scripts/sync-cookbook.ts @@ -576,7 +576,9 @@ async function emitAuthorPage( const meta = authorMeta(handle, authors); const count = matching.length; const description = `${count} recipe${count === 1 ? '' : 's'} contributed to the Steel Cookbook by ${meta.name}.`; - const fm = frontmatter({ title: meta.name, description }); + // Author pages are ego listings with little value for coding agents; keep + // them out of the llms.txt index and the llms-full.txt bundle. + const fm = frontmatter({ title: meta.name, description, llm: 'false' }); const body = `${renderAuthorProfile(handle, authors)}\n\n${renderRecipeGrid(matching)}`; await fs.writeFile(path.join(AUTHORS_DIR, `${handle}.mdx`), `${fm}\n\n${body}\n`); } diff --git a/tests/cookbook-llm-flags.test.ts b/tests/cookbook-llm-flags.test.ts new file mode 100644 index 00000000..7ea5ddeb --- /dev/null +++ b/tests/cookbook-llm-flags.test.ts @@ -0,0 +1,33 @@ +// ABOUTME: Guards the LLM-visibility contract for generated cookbook hub pages: +// ABOUTME: author pages are excluded from agent surfaces, topic pages are kept. + +import { describe, expect, test } from 'bun:test'; +import { Glob } from 'bun'; +import matter from 'gray-matter'; + +async function frontmatter(file: string): Promise> { + return matter(await Bun.file(file).text()).data; +} + +const authors = [ + ...new Glob('*.mdx').scanSync({ cwd: 'content/docs/cookbook/authors', absolute: true }), +]; +const topics = [ + ...new Glob('*.mdx').scanSync({ cwd: 'content/docs/cookbook/topics', absolute: true }), +]; + +describe('cookbook hub LLM visibility', () => { + test('every author page is excluded from LLM surfaces', async () => { + expect(authors.length).toBeGreaterThan(0); + for (const file of authors) { + expect((await frontmatter(file)).llm).toBe(false); + } + }); + + test('topic pages remain visible to LLMs', async () => { + expect(topics.length).toBeGreaterThan(0); + for (const file of topics) { + expect((await frontmatter(file)).llm).not.toBe(false); + } + }); +});