{
 "id": "ai-crawler-tokens",
 "group": null,
 "part_of": null,
 "name": "AI crawler tokens",
 "aliases": [
  "crawler user-agent tokens",
  "robots.txt AI bots"
 ],
 "definition_md": "**AI crawler tokens** are the `User-agent` names AI companies document for use in `robots.txt` under RFC 9309, split by purpose rather than treated as one identity. OpenAI documents OAI-SearchBot (search inclusion in ChatGPT), GPTBot (training data crawling), OAI-AdsBot (ad-landing-page checks) and ChatGPT-User (live fetches triggered by a user's own request, which is not governed by the search or training tokens). Anthropic and Perplexity publish an equivalent three-way split for their own products. Disallowing one token does not affect the others: a site can, for example, permit search indexing while refusing to have its content used for model training. `ChatGPT-User`- and `Perplexity-User`-class tokens are user-triggered fetches, not automated crawling, so robots.txt rules may not apply to them the way they do to a crawler. Used by: cloudflare-content-signals-policy.",
 "step": "permit",
 "related_protocols": [
  "cloudflare-content-signals-policy"
 ],
 "sources": [
  {
   "url": "https://developers.openai.com/api/docs/bots",
   "date": "2026-09-05",
   "label": "VERIFIED"
  },
  {
   "url": "https://support.claude.com/en/articles/8896518",
   "date": "2026-04-07",
   "label": "VERIFIED"
  },
  {
   "url": "https://docs.perplexity.ai/guides/bots",
   "date": "2026-09-05",
   "label": "REPORTED"
  }
 ],
 "url": "https://agenticweb.wiki/terms/ai-crawler-tokens/",
 "category": "permit",
 "type": "term"
}