[
  {
    "slug": "gptbot",
    "name": "GPTBot",
    "operator": "OpenAI",
    "tokens": [
      "GPTBot"
    ],
    "ua_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot",
    "purpose": "training",
    "respects_robots": "yes",
    "respects_robots_source": "OpenAI docs: 'Disallowing GPTBot indicates a site's content should not be used in training generative AI foundation models.' OpenAI states each robots.txt setting is independent.",
    "blocking_effect": "OpenAI says GPTBot and OAI-SearchBot settings are independent. A site can disallow GPTBot and still allow OAI-SearchBot to appear in ChatGPT search. Whether blocking GPTBot affects other OpenAI answers: not stated.",
    "robots_block": "User-agent: GPTBot\nDisallow: /",
    "robots_allow": "User-agent: GPTBot\nAllow: /",
    "ip_ranges_url": "https://openai.com/gptbot.json",
    "docs_url": "https://developers.openai.com/api/docs/bots",
    "sources": [
      {
        "url": "https://developers.openai.com/api/docs/bots",
        "title": "OpenAI: Overview of OpenAI Crawlers",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "UA version number may change (doc says 'the version number may change'). OpenAI may add a 'robots.txt' marker to the UA when fetching robots.txt. If both GPTBot and OAI-SearchBot are allowed, OpenAI says it may use one crawl for both uses."
  },
  {
    "slug": "oai-searchbot",
    "name": "OAI-SearchBot",
    "operator": "OpenAI",
    "tokens": [
      "OAI-SearchBot"
    ],
    "ua_string": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36; compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot",
    "purpose": "search index",
    "respects_robots": "yes",
    "respects_robots_source": "OpenAI docs list OAI-SearchBot as a robots.txt tag for managing search appearance; robots.txt changes can take ~24 hours to take effect.",
    "blocking_effect": "OpenAI: 'Sites that are opted out of OAI-SearchBot will not be shown in ChatGPT search answers, though can still appear as navigational links.'",
    "robots_block": "User-agent: OAI-SearchBot\nDisallow: /",
    "robots_allow": "User-agent: OAI-SearchBot\nAllow: /",
    "ip_ranges_url": "https://openai.com/searchbot.json",
    "docs_url": "https://developers.openai.com/api/docs/bots",
    "sources": [
      {
        "url": "https://developers.openai.com/api/docs/bots",
        "title": "OpenAI: Overview of OpenAI Crawlers",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "OpenAI recommends allowing OAI-SearchBot and its published IP ranges to appear in ChatGPT search. UA may carry a 'robots.txt' marker when fetching robots.txt."
  },
  {
    "slug": "chatgpt-user",
    "name": "ChatGPT-User",
    "operator": "OpenAI",
    "tokens": [
      "ChatGPT-User"
    ],
    "ua_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
    "purpose": "user-triggered fetch",
    "respects_robots": "no (operator: robots.txt rules may not apply)",
    "respects_robots_source": "OpenAI docs: 'Because these actions are initiated by a user, robots.txt rules may not apply.'",
    "blocking_effect": "OpenAI: 'ChatGPT-User is not used to determine whether content may appear in Search.' Effect on other ChatGPT answers: not stated.",
    "robots_block": "User-agent: ChatGPT-User\nDisallow: /",
    "robots_allow": "User-agent: ChatGPT-User\nAllow: /",
    "ip_ranges_url": "https://openai.com/chatgpt-user.json",
    "docs_url": "https://developers.openai.com/api/docs/bots",
    "sources": [
      {
        "url": "https://developers.openai.com/api/docs/bots",
        "title": "OpenAI: Overview of OpenAI Crawlers",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Used for certain user actions in ChatGPT and Custom GPTs, and GPT Actions. OpenAI says it is not used for automatic crawling. The block lines are valid syntax but OpenAI says they may not be honored."
  },
  {
    "slug": "oai-adsbot",
    "name": "OAI-AdsBot",
    "operator": "OpenAI",
    "tokens": [
      "OAI-AdsBot"
    ],
    "ua_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-AdsBot/1.0; +https://openai.com/adsbot",
    "purpose": "other (validates landing pages of ads submitted on ChatGPT)",
    "respects_robots": "not stated",
    "respects_robots_source": "OpenAI docs do not address robots.txt for OAI-AdsBot.",
    "blocking_effect": "not stated",
    "robots_block": "not stated",
    "robots_allow": "not stated",
    "ip_ranges_url": "https://openai.com/adsbot.json",
    "docs_url": "https://developers.openai.com/api/docs/bots",
    "sources": [
      {
        "url": "https://developers.openai.com/api/docs/bots",
        "title": "OpenAI: Overview of OpenAI Crawlers",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "OpenAI: only visits pages submitted as ads; data not used to train generative AI foundation models. No robots.txt token is documented, so block/allow lines are 'not stated'."
  },
  {
    "slug": "claudebot",
    "name": "ClaudeBot",
    "operator": "Anthropic",
    "tokens": [
      "ClaudeBot"
    ],
    "ua_string": "not stated",
    "purpose": "training",
    "respects_robots": "yes",
    "respects_robots_source": "Anthropic: 'Anthropic's Bots respect \"do not crawl\" signals by honoring industry standard directives in robots.txt.' Also supports Crawl-delay.",
    "blocking_effect": "Anthropic: restricting ClaudeBot 'signals that the site's future materials should be excluded from our AI model training datasets.'",
    "robots_block": "User-agent: ClaudeBot\nDisallow: /",
    "robots_allow": "User-agent: ClaudeBot\nAllow: /",
    "ip_ranges_url": "https://claude.com/crawling/bots.json",
    "docs_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
    "sources": [
      {
        "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
        "title": "Anthropic Help Center: Does Anthropic crawl data from the web, and how can site owners block the crawler?",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Full UA string not published on the help page. The IP list is shared across Anthropic bots. Anthropic warns blocking by IP may not work as it prevents reading robots.txt. Apply the rule on each subdomain."
  },
  {
    "slug": "claude-user",
    "name": "Claude-User",
    "operator": "Anthropic",
    "tokens": [
      "Claude-User"
    ],
    "ua_string": "not stated",
    "purpose": "user-triggered fetch",
    "respects_robots": "yes",
    "respects_robots_source": "Anthropic: bots honor industry-standard robots.txt directives (statement covers all three bots).",
    "blocking_effect": "Anthropic: disabling Claude-User 'prevents our system from retrieving your content in response to a user query, which may reduce your site's visibility for user-directed web search.'",
    "robots_block": "User-agent: Claude-User\nDisallow: /",
    "robots_allow": "User-agent: Claude-User\nAllow: /",
    "ip_ranges_url": "https://claude.com/crawling/bots.json",
    "docs_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
    "sources": [
      {
        "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
        "title": "Anthropic Help Center: Does Anthropic crawl data from the web, and how can site owners block the crawler?",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Full UA string not published on the help page."
  },
  {
    "slug": "claude-searchbot",
    "name": "Claude-SearchBot",
    "operator": "Anthropic",
    "tokens": [
      "Claude-SearchBot"
    ],
    "ua_string": "not stated",
    "purpose": "search index",
    "respects_robots": "yes",
    "respects_robots_source": "Anthropic: bots honor industry-standard robots.txt directives (statement covers all three bots).",
    "blocking_effect": "Anthropic: disabling Claude-SearchBot 'prevents our system from indexing your content for search optimization, which may reduce your site's visibility and accuracy in user search results.'",
    "robots_block": "User-agent: Claude-SearchBot\nDisallow: /",
    "robots_allow": "User-agent: Claude-SearchBot\nAllow: /",
    "ip_ranges_url": "https://claude.com/crawling/bots.json",
    "docs_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
    "sources": [
      {
        "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
        "title": "Anthropic Help Center: Does Anthropic crawl data from the web, and how can site owners block the crawler?",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Full UA string not published on the help page."
  },
  {
    "slug": "perplexitybot",
    "name": "PerplexityBot",
    "operator": "Perplexity",
    "tokens": [
      "PerplexityBot"
    ],
    "ua_string": "Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
    "purpose": "search index",
    "respects_robots": "yes (operator claim; disputed by third-party report)",
    "respects_robots_source": "Operator: documents PerplexityBot as controllable via robots.txt tags (changes take up to 24h); the page has no explicit compliance sentence. Contrary report: Cloudflare (Aug 2025) alleged Perplexity used undeclared crawlers with generic browser UAs to access sites that disallowed its declared bots; Perplexity disputed this.",
    "blocking_effect": "Perplexity: 'To ensure your site appears in search results, we recommend allowing PerplexityBot in your site's robots.txt file.' Doc says it is not used to crawl content for AI foundation models. Explicit statement that blocking removes the site from results: not stated.",
    "robots_block": "User-agent: PerplexityBot\nDisallow: /",
    "robots_allow": "User-agent: PerplexityBot\nAllow: /",
    "ip_ranges_url": "https://www.perplexity.ai/perplexitybot.json",
    "docs_url": "https://docs.perplexity.ai/guides/bots",
    "sources": [
      {
        "url": "https://docs.perplexity.ai/guides/bots",
        "title": "Perplexity Crawlers",
        "read_at": "2026-10-29"
      },
      {
        "url": "https://blog.cloudflare.com/perplexity-is-using-stealth-undeclared-crawlers-to-evade-website-no-crawl-directives/",
        "title": "Cloudflare Blog: Perplexity is using stealth, undeclared crawlers to evade website no-crawl directives",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Cloudflare's allegation and Perplexity's rebuttal are summarized in respects_robots_source; the Cloudflare post is listed in sources."
  },
  {
    "slug": "perplexity-user",
    "name": "Perplexity-User",
    "operator": "Perplexity",
    "tokens": [
      "Perplexity-User"
    ],
    "ua_string": "Mozilla/5.0 (compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)",
    "purpose": "user-triggered fetch",
    "respects_robots": "no (operator: generally ignores robots.txt)",
    "respects_robots_source": "Perplexity docs: 'Since a user requested the fetch, this fetcher generally ignores robots.txt rules.'",
    "blocking_effect": "not stated",
    "robots_block": "User-agent: Perplexity-User\nDisallow: /",
    "robots_allow": "User-agent: Perplexity-User\nAllow: /",
    "ip_ranges_url": "https://www.perplexity.ai/perplexity-user.json",
    "docs_url": "https://docs.perplexity.ai/guides/bots",
    "sources": [
      {
        "url": "https://docs.perplexity.ai/guides/bots",
        "title": "Perplexity Crawlers",
        "read_at": "2026-10-29"
      },
      {
        "url": "https://blog.cloudflare.com/perplexity-is-using-stealth-undeclared-crawlers-to-evade-website-no-crawl-directives/",
        "title": "Cloudflare Blog: Perplexity is using stealth, undeclared crawlers to evade website no-crawl directives",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Doc says it is not used for web crawling or to collect content for training AI foundation models. Block lines are valid syntax but the operator says they are generally ignored."
  },
  {
    "slug": "google-extended",
    "name": "Google-Extended",
    "operator": "Google",
    "tokens": [
      "Google-Extended"
    ],
    "ua_string": "No separate UA string (crawling uses existing Google user agents)",
    "purpose": "training (control token for Gemini training and grounding)",
    "respects_robots": "yes (robots.txt control token)",
    "respects_robots_source": "Google: 'Google-Extended is a standalone product token that web publishers can use to manage whether content Google crawls from their sites may be used for training future generations of Gemini models ... and for grounding' in Gemini Apps and Grounding with Google Search on Vertex AI.",
    "blocking_effect": "Google: 'Google-Extended does not impact a site's inclusion in Google Search nor is it used as a ranking signal in Google Search.' Effect on Gemini answers beyond training/grounding use: not stated.",
    "robots_block": "User-agent: Google-Extended\nDisallow: /",
    "robots_allow": "User-agent: Google-Extended\nAllow: /",
    "ip_ranges_url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
    "docs_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
    "sources": [
      {
        "url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
        "title": "Google Search Central: Google's common crawlers",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Googlebot relationship: Google-Extended is not a crawler and has no UA string. Fetching is done by Google's existing crawlers (such as Googlebot); the token only controls how Google may use content already crawled. Blocking Googlebot would remove the site from Google Search; blocking Google-Extended does not. IP ranges are those of Google's common crawlers."
  },
  {
    "slug": "googlebot",
    "name": "Googlebot (Google-Extended relationship)",
    "operator": "Google",
    "tokens": [
      "Googlebot"
    ],
    "ua_string": "Desktop: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/W.X.Y.Z Safari/537.36",
    "purpose": "search index",
    "respects_robots": "yes",
    "respects_robots_source": "Google lists a Googlebot robots.txt token on its common-crawlers page.",
    "blocking_effect": "Google lists affected products: Google Search (including Discover and Google Search features), Google Images, Google Video, Google News. Blocking it affects those, so do not block it to opt out of AI training.",
    "robots_block": "User-agent: Googlebot\nDisallow: /",
    "robots_allow": "User-agent: Googlebot\nAllow: /",
    "ip_ranges_url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
    "docs_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
    "sources": [
      {
        "url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
        "title": "Google Search Central: Google's common crawlers",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Reference row: shows how Googlebot relates to Google-Extended. Smartphone UA is also published on the doc page (Nexus 5X Android UA with Googlebot/2.1). W.X.Y.Z is Google's placeholder for the Chrome version."
  },
  {
    "slug": "google-cloudvertexbot",
    "name": "Google-CloudVertexBot",
    "operator": "Google",
    "tokens": [
      "Google-CloudVertexBot"
    ],
    "ua_string": "not stated",
    "purpose": "other (crawls requested by site owners to build Vertex AI Agents)",
    "respects_robots": "yes (robots.txt token documented)",
    "respects_robots_source": "Google lists a Google-CloudVertexBot robots.txt token and example group.",
    "blocking_effect": "Google: 'It has no effect on Google Search or other products.'",
    "robots_block": "User-agent: Google-CloudVertexBot\nDisallow: /",
    "robots_allow": "User-agent: Google-CloudVertexBot\nAllow: /",
    "ip_ranges_url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
    "docs_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
    "sources": [
      {
        "url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
        "title": "Google Search Central: Google's common crawlers",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Google documents a UA substring only, not a full string."
  },
  {
    "slug": "google-agent",
    "name": "Google-Agent",
    "operator": "Google",
    "tokens": [
      "Google-Agent"
    ],
    "ua_string": "Desktop: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent) Chrome/W.X.Y.Z Safari/537.36",
    "purpose": "user-triggered fetch (agents hosted on Google infrastructure act on user request)",
    "respects_robots": "no (Google: user-triggered fetchers generally ignore robots.txt)",
    "respects_robots_source": "Google user-triggered fetchers page: they 'generally ignore robots.txt rules' because the fetch was requested by a user.",
    "blocking_effect": "not stated",
    "robots_block": "not stated",
    "robots_allow": "not stated",
    "ip_ranges_url": "https://developers.google.com/static/crawling/ipranges/user-triggered-agents.json",
    "docs_url": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers",
    "sources": [
      {
        "url": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers",
        "title": "Google Search Central: Google user-triggered fetchers",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "A mobile UA is also published. Google is experimenting with Web Bot Auth using the https://agent.bot.goog identity. No robots.txt token documented."
  },
  {
    "slug": "google-gemininotebook",
    "name": "Google-GeminiNotebook",
    "operator": "Google",
    "tokens": [
      "Google-GeminiNotebook",
      "Google-NotebookLM (former, supported until August 2026)"
    ],
    "ua_string": "Desktop: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 (compatible; Google-GeminiNotebook; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-gemininotebook)",
    "purpose": "user-triggered fetch (URLs users add as notebook sources)",
    "respects_robots": "no (Google: user-triggered fetchers generally ignore robots.txt)",
    "respects_robots_source": "Google user-triggered fetchers page: they 'generally ignore robots.txt rules'.",
    "blocking_effect": "not stated",
    "robots_block": "not stated",
    "robots_allow": "not stated",
    "ip_ranges_url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json",
    "docs_url": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers",
    "sources": [
      {
        "url": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers",
        "title": "Google Search Central: Google user-triggered fetchers",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Google lists three user-triggered fetcher IP files; we matched this fetcher to the linked file and have not confirmed the match, so check it before relying on it. A mobile UA is also published."
  },
  {
    "slug": "bytespider",
    "name": "Bytespider",
    "operator": "ByteDance",
    "tokens": [
      "Bytespider"
    ],
    "ua_string": "not stated",
    "purpose": "not stated by operator",
    "respects_robots": "unverified",
    "respects_robots_source": "No ByteDance documentation found. The developer.bytedance.com/docs/en/spider URL returned 404 and bytedance.us/bytespider returned an empty page. We make no claim about whether it follows robots.txt.",
    "blocking_effect": "not stated",
    "robots_block": "User-agent: Bytespider\nDisallow: /\n(standard syntax; token seen in third-party sources, not operator docs)",
    "robots_allow": "User-agent: Bytespider\nAllow: /\n(standard syntax; token not confirmed by operator)",
    "ip_ranges_url": "not stated",
    "docs_url": "not stated",
    "sources": [
      {
        "url": "https://developer.bytedance.com/docs/en/spider",
        "title": "ByteDance developer docs (attempted; HTTP 404)",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Unverified. No operator documentation found. UA string, purpose and IP ranges are left as not stated."
  },
  {
    "slug": "ccbot",
    "name": "CCBot",
    "operator": "Common Crawl",
    "tokens": [
      "CCBot"
    ],
    "ua_string": "CCBot/2.0 (https://commoncrawl.org/faq/)",
    "purpose": "other (open web crawl repository; data is publicly available)",
    "respects_robots": "yes",
    "respects_robots_source": "Common Crawl gives robots.txt opt-out instructions for CCBot. The page has no explicit sentence saying CCBot obeys robots.txt.",
    "blocking_effect": "not stated",
    "robots_block": "User-agent: CCBot\nDisallow: /",
    "robots_allow": "User-agent: CCBot\nAllow: /",
    "ip_ranges_url": "https://index.commoncrawl.org/ccbot.json",
    "docs_url": "https://commoncrawl.org/ccbot",
    "sources": [
      {
        "url": "https://commoncrawl.org/ccbot",
        "title": "Common Crawl: CCBot",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Common Crawl says it is aware of crawlers falsely identifying as CCBot and recommends verifying (reverse DNS on *.crawl.commoncrawl.org, or the JSON IP list). It also has an Opt-out Registry. The page does not state AI training use; downstream use by third parties is outside operator docs."
  },
  {
    "slug": "amazonbot",
    "name": "Amazonbot",
    "operator": "Amazon",
    "tokens": [
      "Amazonbot"
    ],
    "ua_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1) Chrome/W.X.Y.Z Safari/537.36",
    "purpose": "training / other (improve Amazon products and services; may train Amazon AI models)",
    "respects_robots": "yes",
    "respects_robots_source": "Amazon: its user agents follow the Robots Exclusion Protocol, honoring user-agent and allow/disallow directives. Crawl-delay not supported.",
    "blocking_effect": "not stated",
    "robots_block": "User-agent: Amazonbot\nDisallow: /",
    "robots_allow": "User-agent: Amazonbot\nAllow: /",
    "ip_ranges_url": "https://developer.amazon.com/amazonbot/ip-addresses/",
    "docs_url": "https://developer.amazon.com/amazonbot",
    "sources": [
      {
        "url": "https://developer.amazon.com/amazonbot",
        "title": "Amazon: Amazon Web Search crawlers (Amazonbot)",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Amazon: respects rel=nofollow and robots meta noarchive (do not use for model training), noindex, none. Amazon mentions an Amazon Content Partners program for sites allowing Amazonbot (benefits not verified). Contact: amazonbot@amazon.com."
  },
  {
    "slug": "amzn-searchbot",
    "name": "Amzn-SearchBot",
    "operator": "Amazon",
    "tokens": [
      "Amzn-SearchBot"
    ],
    "ua_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-SearchBot/0.1) Chrome/W.X.Y.Z Safari/537.36",
    "purpose": "search index",
    "respects_robots": "yes",
    "respects_robots_source": "Amazon: honors robots.txt directives; if robots.txt does not mention it but allows other search bots, it follows those directives.",
    "blocking_effect": "Amazon: 'By permitting Amzn-SearchBot access to your website, your content is eligible to appear in search experiences such as Alexa.'",
    "robots_block": "User-agent: Amzn-SearchBot\nDisallow: /",
    "robots_allow": "User-agent: Amzn-SearchBot\nAllow: /",
    "ip_ranges_url": "https://developer.amazon.com/amazonbot/searchbot-ip-addresses/",
    "docs_url": "https://developer.amazon.com/amazonbot",
    "sources": [
      {
        "url": "https://developer.amazon.com/amazonbot",
        "title": "Amazon: Amazon Web Search crawlers (Amazonbot)",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Amazon: does not crawl content for generative AI model training."
  },
  {
    "slug": "amzn-user",
    "name": "Amzn-User",
    "operator": "Amazon",
    "tokens": [
      "Amzn-User"
    ],
    "ua_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-User/0.1) Chrome/W.X.Y.Z Safari/537.36",
    "purpose": "user-triggered fetch",
    "respects_robots": "partial (operator: may not follow all robots.txt directives)",
    "respects_robots_source": "Amazon: 'Because actions taken by Amzn-User can be initiated by a user, it may not follow all robots.txt directives.'",
    "blocking_effect": "not stated",
    "robots_block": "User-agent: Amzn-User\nDisallow: /",
    "robots_allow": "User-agent: Amzn-User\nAllow: /",
    "ip_ranges_url": "https://developer.amazon.com/amazonbot/live-ip-addresses/",
    "docs_url": "https://developer.amazon.com/amazonbot",
    "sources": [
      {
        "url": "https://developer.amazon.com/amazonbot",
        "title": "Amazon: Amazon Web Search crawlers (Amazonbot)",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Supports live answers such as Alexa queries. Amazon: does not crawl content for generative AI model training."
  },
  {
    "slug": "applebot-extended",
    "name": "Applebot-Extended",
    "operator": "Apple",
    "tokens": [
      "Applebot-Extended"
    ],
    "ua_string": "No separate UA string (it does not crawl; it controls use of data crawled by Applebot)",
    "purpose": "training (control token for Apple foundation models)",
    "respects_robots": "yes",
    "respects_robots_source": "Apple: 'In addition to following all robots.txt rules and directives, Apple has a secondary user agent, Applebot-Extended.' ",
    "blocking_effect": "Apple: 'Webpages that disallow Applebot-Extended can still be included in search results.' It is only used to determine how crawled data is used.",
    "robots_block": "User-agent: Applebot-Extended\nDisallow: /",
    "robots_allow": "User-agent: Applebot-Extended\nAllow: /",
    "ip_ranges_url": "https://search.developer.apple.com/applebot.json",
    "docs_url": "https://support.apple.com/en-us/119829",
    "sources": [
      {
        "url": "https://support.apple.com/en-us/119829",
        "title": "Apple Support: About Applebot",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Applebot-Extended does not crawl webpages. Opting out of use in broad world knowledge answers (Siri and Search) is done with the nosnippet meta tag, not Applebot-Extended. IP file is the Applebot CIDR list."
  },
  {
    "slug": "applebot",
    "name": "Applebot",
    "operator": "Apple",
    "tokens": [
      "Applebot"
    ],
    "ua_string": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)",
    "purpose": "search index (data may also be used to train Apple models unless Applebot-Extended is disallowed)",
    "respects_robots": "yes",
    "respects_robots_source": "Apple: 'Applebot respects standard robots.txt directives in general search crawls that are targeted at Applebot.' If robots.txt does not mention Applebot but mentions Googlebot, Applebot follows Googlebot instructions. No crawl-delay.",
    "blocking_effect": "Apple: 'Enabling Applebot in robots.txt allows website content to appear in search results for Apple users' in Spotlight, Siri and Safari.",
    "robots_block": "User-agent: Applebot\nDisallow: /",
    "robots_allow": "User-agent: Applebot\nAllow: /",
    "ip_ranges_url": "https://search.developer.apple.com/applebot.json",
    "docs_url": "https://support.apple.com/en-us/119829",
    "sources": [
      {
        "url": "https://support.apple.com/en-us/119829",
        "title": "Apple Support: About Applebot",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Apple also documents an 'iTMS' user agent for Apple Podcasts that does not follow robots.txt. Verify via reverse DNS *.applebot.apple.com."
  },
  {
    "slug": "meta-externalagent",
    "name": "Meta-ExternalAgent",
    "operator": "Meta",
    "tokens": [
      "Meta-ExternalAgent"
    ],
    "ua_string": "meta-externalagent/1.1 (+/documentation/sharing/webmasters/web-crawlers) (as printed on the doc page; the link is relative) ; also meta-externalagent/1.1",
    "purpose": "training / other (training foundation AI models or improving products by indexing content directly)",
    "respects_robots": "yes (operator gives robots.txt instructions)",
    "respects_robots_source": "Meta: 'In order to block these crawlers, add a disallow for the relevant crawler to robots.txt.' Changes may take up to 24 hours.",
    "blocking_effect": "not stated",
    "robots_block": "User-agent: meta-externalagent\nDisallow: /",
    "robots_allow": "User-agent: meta-externalagent\nAllow: /",
    "ip_ranges_url": "not stated",
    "docs_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
    "sources": [
      {
        "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
        "title": "Meta for Developers: Web Crawlers",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Meta's page does not publish IP ranges. Meta's own example uses the lowercase token meta-externalagent; robots.txt matching is case-insensitive by convention."
  },
  {
    "slug": "meta-externalfetcher",
    "name": "Meta-ExternalFetcher",
    "operator": "Meta",
    "tokens": [
      "Meta-ExternalFetcher"
    ],
    "ua_string": "meta-externalfetcher/1.1 (+/documentation/sharing/webmasters/web-crawlers) (as printed on the doc page; the link is relative) ; also meta-externalfetcher/1.1",
    "purpose": "user-triggered fetch",
    "respects_robots": "no (operator: may bypass robots.txt)",
    "respects_robots_source": "Meta: 'this crawler may bypass robots.txt rules' / 'may bypass robots.txt because it performs fetches that were requested by the user.'",
    "blocking_effect": "not stated",
    "robots_block": "User-agent: meta-externalfetcher\nDisallow: /",
    "robots_allow": "User-agent: meta-externalfetcher\nAllow: /",
    "ip_ranges_url": "not stated",
    "docs_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
    "sources": [
      {
        "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
        "title": "Meta for Developers: Web Crawlers",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Meta's sample block groups only show meta-externalagent and meta-externalads; the meta-externalfetcher lines here follow the same syntax. Supports agentic AI capabilities."
  },
  {
    "slug": "meta-webindexer",
    "name": "Meta-WebIndexer",
    "operator": "Meta",
    "tokens": [
      "Meta-WebIndexer"
    ],
    "ua_string": "meta-webindexer/1.1 (+/documentation/sharing/webmasters/web-crawlers) (as printed on the doc page; the link is relative) ; also meta-webindexer/1.1",
    "purpose": "search index",
    "respects_robots": "not stated (robots.txt control implied)",
    "respects_robots_source": "Meta: 'Allowing Meta-WebIndexer in your robots.txt file helps us cite and link to your content in Meta AI's responses.'",
    "blocking_effect": "Meta: allowing it 'helps us cite and link to your content in Meta AI's responses.' Explicit statement that blocking removes citations: not stated.",
    "robots_block": "User-agent: meta-webindexer\nDisallow: /",
    "robots_allow": "User-agent: meta-webindexer\nAllow: /",
    "ip_ranges_url": "not stated",
    "docs_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
    "sources": [
      {
        "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
        "title": "Meta for Developers: Web Crawlers",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Meta also documents Meta-ExternalAds (advertising/business products) and FacebookExternalHit (link previews, may bypass robots.txt for integrity checks); not given rows."
  },
  {
    "slug": "duckassistbot",
    "name": "DuckAssistBot",
    "operator": "DuckDuckGo",
    "tokens": [
      "DuckAssistBot"
    ],
    "ua_string": "DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)",
    "purpose": "user-triggered fetch / search (real-time crawling for AI-assisted answers)",
    "respects_robots": "yes",
    "respects_robots_source": "DuckDuckGo: publishers can opt out by disallowing the DuckAssistBot user agent in robots.txt; takes effect after 72 hours.",
    "blocking_effect": "DuckDuckGo: 'Opting out of DuckAssistBot does not impact organic search rankings and will not affect whether or not websites appear in our search results.' Opting out removes the site as a potential source for AI-assisted answers.",
    "robots_block": "User-agent: DuckAssistBot\nDisallow: /",
    "robots_allow": "User-agent: DuckAssistBot\nAllow: /",
    "ip_ranges_url": "https://duckduckgo.com/duckassistbot.json",
    "docs_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
    "sources": [
      {
        "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
        "title": "DuckDuckGo Help Pages: About DuckAssistBot",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "DuckDuckGo: data is not used to train AI models. Opt-out problems: crawling@duckduckgo.com."
  },
  {
    "slug": "cohere-ai",
    "name": "cohere-ai / Coherebot",
    "operator": "Cohere",
    "tokens": [
      "Coherebot",
      "cohere-ai"
    ],
    "ua_string": "not stated",
    "purpose": "not stated (Cohere says it currently does not crawl to train generative models)",
    "respects_robots": "not stated",
    "respects_robots_source": "Cohere: 'It is Cohere's policy to require that crawlers be designed to respect robots.txt.' This is a general policy statement; no specific bot is listed.",
    "blocking_effect": "not stated",
    "robots_block": "User-agent: Coherebot\nDisallow: /\n(operator's own example token; no active Cohere bot is listed)",
    "robots_allow": "not stated",
    "ip_ranges_url": "not stated",
    "docs_url": "https://docs.cohere.com/docs/cohere-web-crawlers",
    "sources": [
      {
        "url": "https://docs.cohere.com/docs/cohere-web-crawlers",
        "title": "Cohere Docs: Cohere Web Crawlers",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Unverified token. Cohere's page says it does not use bots or user agents to crawl to train generative AI foundation models 'at this time'; its bot table is 'N/A'. The tokens cohere-ai and cohere-training-data-crawler appear in third-party lists but not in Cohere docs, so no row for cohere-training-data-crawler and no allow line for cohere-ai."
  },
  {
    "slug": "diffbot",
    "name": "Diffbot",
    "operator": "Diffbot",
    "tokens": [
      "Diffbot"
    ],
    "ua_string": "not stated",
    "purpose": "search index (general crawling for Diffbot Knowledge Graph and web search)",
    "respects_robots": "yes",
    "respects_robots_source": "Diffbot docs: 'By default Diffbot's web crawls adhere to a site's robots.txt instructions, including the disallow and crawl-delay directives.'",
    "blocking_effect": "Diffbot: crawling allows sites 'to be discovered and cited from the Diffbot Knowledge Graph and web search services'. Effect of blocking on that: not stated beyond this.",
    "robots_block": "User-agent: Diffbot\nDisallow: /",
    "robots_allow": "User-agent: Diffbot\nAllow: /",
    "ip_ranges_url": "not stated",
    "docs_url": "https://docs.diffbot.com/docs/does-crawl-respect-robotstxt",
    "sources": [
      {
        "url": "https://docs.diffbot.com/docs/does-crawl-respect-robotstxt",
        "title": "Diffbot Docs: Does Diffbot respect robots.txt?",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Diffbot: not used for AI training; Diffbot does not crawl pages for training generative AI foundation models. UA string not on the docs page (a third-party-reported string exists). Customer-run Crawlbot/Extract jobs use customer-set UAs."
  },
  {
    "slug": "diffbot-user",
    "name": "Diffbot-User",
    "operator": "Diffbot",
    "tokens": [
      "Diffbot-User"
    ],
    "ua_string": "not stated",
    "purpose": "user-triggered fetch",
    "respects_robots": "not stated",
    "respects_robots_source": "Diffbot docs describe Diffbot-User as requests 'on behalf of a human user browsing a URL using Diffbot software' and do not state its robots.txt behavior.",
    "blocking_effect": "not stated",
    "robots_block": "User-agent: Diffbot-User\nDisallow: /",
    "robots_allow": "User-agent: Diffbot-User\nAllow: /",
    "ip_ranges_url": "not stated",
    "docs_url": "https://docs.diffbot.com/docs/does-crawl-respect-robotstxt",
    "sources": [
      {
        "url": "https://docs.diffbot.com/docs/does-crawl-respect-robotstxt",
        "title": "Diffbot Docs: Does Diffbot respect robots.txt?",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "No further detail published by the operator."
  },
  {
    "slug": "youbot",
    "name": "YouBot",
    "operator": "You.com",
    "tokens": [
      "YouBot"
    ],
    "ua_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; YouBot/1.0; +https://docs.you.com/youbot; env:prod) Chrome/X.X.X.X Safari/537.36",
    "purpose": "search index",
    "respects_robots": "yes",
    "respects_robots_source": "You.com docs: 'It respects robots.txt directives and crawl rate preferences'; honors crawl-delay; caches robots.txt for 30 minutes.",
    "blocking_effect": "not stated",
    "robots_block": "User-agent: YouBot\nDisallow: /",
    "robots_allow": "User-agent: YouBot\nAllow: /",
    "ip_ranges_url": "No JSON list. Docs page states a single range: 68.67.112.0/24 (https://docs.you.com/youbot)",
    "docs_url": "https://docs.you.com/youbot.md",
    "sources": [
      {
        "url": "https://docs.you.com/youbot.md",
        "title": "You.com Docs: YouBot, You.com's Web Crawler",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Verification options per docs: Cloudflare Web Bot Auth signatures (keys at https://you.com/.well-known/http-message-signatures-directory), reverse DNS youbot-{ip}.search.you.com, and the IP range. Contact spider@you.com. 'X.X.X.X' is the Chrome version placeholder."
  },
  {
    "slug": "timpibot",
    "name": "Timpibot",
    "operator": "Timpi",
    "tokens": [
      "Timpibot"
    ],
    "ua_string": "not stated",
    "purpose": "not stated by operator",
    "respects_robots": "unverified",
    "respects_robots_source": "No Timpi documentation found (timpi.io homepage read; /timpibot and /faq returned 404). Third-party sources conflict on compliance and are not credible enough to cite as fact.",
    "blocking_effect": "not stated",
    "robots_block": "User-agent: Timpibot\nDisallow: /\n(standard syntax; token seen in third-party sources, not operator docs)",
    "robots_allow": "User-agent: Timpibot\nAllow: /\n(standard syntax; token not confirmed by operator)",
    "ip_ranges_url": "not stated",
    "docs_url": "not stated",
    "sources": [
      {
        "url": "https://timpi.io/",
        "title": "Timpi homepage (no crawler documentation found)",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Unverified. UA string, purpose and IPs are left as not stated."
  },
  {
    "slug": "omgili",
    "name": "omgili / omgilibot",
    "operator": "Webz.io",
    "tokens": [
      "omgili",
      "omgilibot"
    ],
    "ua_string": "omgili/0.5 +https://omgili.com (as quoted by Webz.io)",
    "purpose": "other (web data feed sold via the Webz.io API; the page does not mention AI training)",
    "respects_robots": "yes (operator claim)",
    "respects_robots_source": "Webz.io: 'If you don't like our bot hanging around your site, you can tell us directly or through your robots.txt file.' No per-token syntax is given.",
    "blocking_effect": "not stated",
    "robots_block": "User-agent: omgili\nDisallow: /\n(standard syntax; Webz.io does not publish exact robots lines)",
    "robots_allow": "User-agent: omgili\nAllow: /\n(standard syntax)",
    "ip_ranges_url": "not stated",
    "docs_url": "https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/",
    "sources": [
      {
        "url": "https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/",
        "title": "Webz.io blog: What is the Omgili Bot, and why is it Crawling Your Website?",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "The older URL http://omgili.com/Crawler.html redirects (HTTP 302) to a Webz.io blog page. Webz.io calls it Omgilibot in prose but quotes UA 'omgili/0.5'; which robots.txt token it matches (omgili vs omgilibot) is not stated, so block both to be safe. Indexed content is made available via the Webz.io API; operator names no AI-training use."
  },
  {
    "slug": "mistralai-user",
    "name": "MistralAI-User",
    "operator": "Mistral AI",
    "tokens": [
      "MistralAI-User"
    ],
    "ua_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)",
    "purpose": "user-triggered fetch (user actions in Vibe)",
    "respects_robots": "not stated",
    "respects_robots_source": "Mistral: MistralAI-User 'governs which sites these user requests can be made to'; an explicit statement about honoring robots.txt was not captured.",
    "blocking_effect": "not stated",
    "robots_block": "User-agent: MistralAI-User\nDisallow: /",
    "robots_allow": "User-agent: MistralAI-User\nAllow: /",
    "ip_ranges_url": "https://mistral.ai/mistralai-user-ips.json",
    "docs_url": "https://docs.mistral.ai/robots",
    "sources": [
      {
        "url": "https://docs.mistral.ai/robots",
        "title": "Mistral Docs: Crawlers (robots)",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Mistral: not used for automatic crawling or AI training."
  },
  {
    "slug": "mistralai-index",
    "name": "MistralAI-Index",
    "operator": "Mistral AI",
    "tokens": [
      "MistralAI-Index"
    ],
    "ua_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Index/1.0; +https://docs.mistral.ai/robots)",
    "purpose": "search index",
    "respects_robots": "not stated",
    "respects_robots_source": "Mistral doc names MistralAI-Index as a robots.txt user agent for indexing; explicit compliance sentence not captured.",
    "blocking_effect": "Mistral: indexes content for Mistral search, which helps answer user questions in Vibe. Explicit effect of blocking: not stated.",
    "robots_block": "User-agent: MistralAI-Index\nDisallow: /",
    "robots_allow": "User-agent: MistralAI-Index\nAllow: /",
    "ip_ranges_url": "https://mistral.ai/mistralai-index-ips.json",
    "docs_url": "https://docs.mistral.ai/robots",
    "sources": [
      {
        "url": "https://docs.mistral.ai/robots",
        "title": "Mistral Docs: Crawlers (robots)",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "Mistral: content not used for generative AI training of any kind."
  },
  {
    "slug": "mistralai-training",
    "name": "MistralAI-Training",
    "operator": "Mistral AI",
    "tokens": [
      "MistralAI-Training"
    ],
    "ua_string": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Training/1.0; +https://docs.mistral.ai/robots)",
    "purpose": "training",
    "respects_robots": "yes (operator: webmasters can disallow it in robots.txt)",
    "respects_robots_source": "Mistral: 'Webmasters can disallow this user agent in their robots.txt file.'",
    "blocking_effect": "Mistral: not used for search indexing or to answer live user queries in Vibe.",
    "robots_block": "User-agent: MistralAI-Training\nDisallow: /",
    "robots_allow": "User-agent: MistralAI-Training\nAllow: /",
    "ip_ranges_url": "not stated",
    "docs_url": "https://docs.mistral.ai/robots",
    "sources": [
      {
        "url": "https://docs.mistral.ai/robots",
        "title": "Mistral Docs: Crawlers (robots)",
        "read_at": "2026-10-29"
      }
    ],
    "last_verified": "2026-10-29",
    "notes": "No IP list link found for this agent."
  }
]