From 0d153f7aa50af3b80b9a3109c3323d7f94161854 Mon Sep 17 00:00:00 2001
From: Nikola Balic <wave@nibzard.com>
Date: Thu, 30 Jul 2026 12:59:41 +0200
Subject: [PATCH] feat(robots): declare Content Signals for AI crawlers

robots.txt says which crawlers may fetch the docs but says nothing about
what they may do with the content once fetched. Content Signals
(contentsignals.org, draft-romm-aipref-contentsignals) covers that second
half, and agent-readiness scanners check for it.

Declare search=yes, ai-input=yes, ai-train=yes. That restates in
machine-readable form what robots.txt already grants: every AI crawler
here, including the training crawlers GPTBot and Google-Extended, is
allowed. Flip a value if that intent ever changes.

The declaration is repeated in every User-agent group because a crawler
matching a named group ignores the wildcard group entirely, so a single
declaration under `*` would not reach GPTBot or ClaudeBot.
---
 public/robots.txt        | 21 +++++++++++
 tests/robots-txt.test.ts | 79 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 100 insertions(+)
 create mode 100644 tests/robots-txt.test.ts

diff --git a/public/robots.txt b/public/robots.txt
index 846c9233..a2c03551 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -1,31 +1,52 @@
+# Content Signals (https://contentsignals.org) state how this site's content
+# may be used once it has been crawled. Steel's docs are public and written to
+# be read by agents, so all three signals are open:
+#
+#   search:   appearing in search results
+#   ai-input: use as input for an AI answer, such as RAG or grounding
+#   ai-train: use as training data for a model
+#
+# A crawler that matches a named group below ignores the wildcard group, so the
+# declaration is repeated in every group rather than stated once.
+
 User-agent: *
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: GPTBot
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: ChatGPT-User
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: OAI-SearchBot
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: PerplexityBot
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: Perplexity-User
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: ClaudeBot
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: Claude-User
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: Claude-SearchBot
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 User-agent: Google-Extended
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 Sitemap: https://docs.steel.dev/sitemap.xml
diff --git a/tests/robots-txt.test.ts b/tests/robots-txt.test.ts
new file mode 100644
index 00000000..7b3d40d2
--- /dev/null
+++ b/tests/robots-txt.test.ts
@@ -0,0 +1,79 @@
+// ABOUTME: Tests for public/robots.txt, covering the Content Signals declaration
+// ABOUTME: that states how AI crawlers may use the docs, plus the crawl directives.
+import { describe, expect, test } from 'bun:test';
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+
+const ROBOTS = readFileSync(
+  fileURLToPath(new URL('../public/robots.txt', import.meta.url)),
+  'utf8',
+);
+
+/** Splits robots.txt into its User-agent groups, keyed by agent name. */
+function parseGroups(source: string): Map<string, string[]> {
+  const groups = new Map<string, string[]>();
+  let current: string[] | null = null;
+
+  for (const line of source.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+
+    const [rawKey, ...rawValue] = trimmed.split(':');
+    const key = rawKey.trim().toLowerCase();
+    const value = rawValue.join(':').trim();
+
+    if (key === 'user-agent') {
+      current = [];
+      groups.set(value, current);
+    } else if (current) {
+      current.push(trimmed);
+    }
+  }
+
+  return groups;
+}
+
+const groups = parseGroups(ROBOTS);
+
+describe('robots.txt content signals', () => {
+  test('declares a Content-Signal in every User-agent group', () => {
+    // A crawler matching a specific group ignores the wildcard group entirely,
+    // so the declaration has to be repeated rather than stated once.
+    expect(groups.size).toBeGreaterThan(0);
+
+    for (const [agent, directives] of groups) {
+      const signal = directives.find((directive) =>
+        directive.toLowerCase().startsWith('content-signal:'),
+      );
+      expect(signal, `${agent} is missing a Content-Signal directive`).toBeDefined();
+    }
+  });
+
+  test('declares all three signals with valid values', () => {
+    for (const [agent, directives] of groups) {
+      const signal = directives.find((directive) =>
+        directive.toLowerCase().startsWith('content-signal:'),
+      );
+      const declared = new Map(
+        (signal ?? '')
+          .slice('content-signal:'.length)
+          .split(',')
+          .map((entry) => entry.trim().split('=') as [string, string]),
+      );
+
+      for (const name of ['search', 'ai-input', 'ai-train']) {
+        expect([...declared.keys()], `${agent} is missing the ${name} signal`).toContain(name);
+        expect(['yes', 'no'], `${agent} declares an invalid ${name} value`).toContain(
+          declared.get(name) ?? '',
+        );
+      }
+    }
+  });
+
+  test('keeps the crawl directives, sitemap and host', () => {
+    expect(groups.get('*')).toContain('Allow: /');
+    expect(groups.get('ClaudeBot')).toContain('Allow: /');
+    expect(ROBOTS).toContain('Sitemap: https://docs.steel.dev/sitemap.xml');
+    expect(ROBOTS).toContain('Host: docs.steel.dev');
+  });
+});