diff --git a/public/robots.txt b/public/robots.txt index 846c9233..a2c03551 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,31 +1,52 @@ +# Content Signals (https://contentsignals.org) state how this site's content +# may be used once it has been crawled. Steel's docs are public and written to +# be read by agents, so all three signals are open: +# +# search: appearing in search results +# ai-input: use as input for an AI answer, such as RAG or grounding +# ai-train: use as training data for a model +# +# A crawler that matches a named group below ignores the wildcard group, so the +# declaration is repeated in every group rather than stated once. + User-agent: * +Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-agent: GPTBot +Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-agent: ChatGPT-User +Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-agent: OAI-SearchBot +Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-agent: PerplexityBot +Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-agent: Perplexity-User +Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-agent: ClaudeBot +Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-agent: Claude-User +Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-agent: Claude-SearchBot +Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-agent: Google-Extended +Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / Sitemap: https://docs.steel.dev/sitemap.xml diff --git a/tests/robots-txt.test.ts b/tests/robots-txt.test.ts new file mode 100644 index 00000000..7b3d40d2 --- /dev/null +++ b/tests/robots-txt.test.ts @@ -0,0 +1,79 @@ +// ABOUTME: Tests for public/robots.txt, covering the Content Signals declaration +// ABOUTME: that states how AI crawlers may use the docs, plus the crawl directives. +import { describe, expect, test } from 'bun:test'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +const ROBOTS = readFileSync( + fileURLToPath(new URL('../public/robots.txt', import.meta.url)), + 'utf8', +); + +/** Splits robots.txt into its User-agent groups, keyed by agent name. */ +function parseGroups(source: string): Map { + const groups = new Map(); + let current: string[] | null = null; + + for (const line of source.split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + + const [rawKey, ...rawValue] = trimmed.split(':'); + const key = rawKey.trim().toLowerCase(); + const value = rawValue.join(':').trim(); + + if (key === 'user-agent') { + current = []; + groups.set(value, current); + } else if (current) { + current.push(trimmed); + } + } + + return groups; +} + +const groups = parseGroups(ROBOTS); + +describe('robots.txt content signals', () => { + test('declares a Content-Signal in every User-agent group', () => { + // A crawler matching a specific group ignores the wildcard group entirely, + // so the declaration has to be repeated rather than stated once. + expect(groups.size).toBeGreaterThan(0); + + for (const [agent, directives] of groups) { + const signal = directives.find((directive) => + directive.toLowerCase().startsWith('content-signal:'), + ); + expect(signal, `${agent} is missing a Content-Signal directive`).toBeDefined(); + } + }); + + test('declares all three signals with valid values', () => { + for (const [agent, directives] of groups) { + const signal = directives.find((directive) => + directive.toLowerCase().startsWith('content-signal:'), + ); + const declared = new Map( + (signal ?? '') + .slice('content-signal:'.length) + .split(',') + .map((entry) => entry.trim().split('=') as [string, string]), + ); + + for (const name of ['search', 'ai-input', 'ai-train']) { + expect([...declared.keys()], `${agent} is missing the ${name} signal`).toContain(name); + expect(['yes', 'no'], `${agent} declares an invalid ${name} value`).toContain( + declared.get(name) ?? '', + ); + } + } + }); + + test('keeps the crawl directives, sitemap and host', () => { + expect(groups.get('*')).toContain('Allow: /'); + expect(groups.get('ClaudeBot')).toContain('Allow: /'); + expect(ROBOTS).toContain('Sitemap: https://docs.steel.dev/sitemap.xml'); + expect(ROBOTS).toContain('Host: docs.steel.dev'); + }); +});