Merge pull request #98 from kylebuckingham/main

ai.robots.txt · ai.robots.txt · commit 7c5389f4a0c5 · 2025-05-21T19:00:23.000Z
Updating Claude Bots
diff --git a/.htaccess b/.htaccess
@@ -1,3 +1,3 @@
 RewriteEngine On
-RewriteCond %{HTTP_USER_AGENT} (AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|imgproxy|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|QualifiedBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot) [NC]
+RewriteCond %{HTTP_USER_AGENT} (AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|ClaudeBot|Claude\-User|Claude\-SearchBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|imgproxy|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|QualifiedBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot) [NC]
 RewriteRule !^/?robots\.txt$ - [F,L]
diff --git a/Caddyfile b/Caddyfile
@@ -1,3 +1,3 @@
 @aibots {
-        header_regexp User-Agent "(AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|imgproxy|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|QualifiedBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot)"
+        header_regexp User-Agent "(AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|ClaudeBot|Claude\-User|Claude\-SearchBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|imgproxy|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|QualifiedBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot)"
 }
diff --git a/haproxy-block-ai-bots.txt b/haproxy-block-ai-bots.txt
@@ -9,8 +9,9 @@ Brightbot 1.0
 Bytespider
 CCBot
 ChatGPT-User
-Claude-Web
 ClaudeBot
+Claude-User
+Claude-SearchBot
 cohere-ai
 cohere-training-data-crawler
 Cotoyogi
diff --git a/nginx-block-ai-bots.conf b/nginx-block-ai-bots.conf
@@ -1,3 +1,3 @@
-if ($http_user_agent ~* "(AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|imgproxy|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|QualifiedBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot)") {
+if ($http_user_agent ~* "(AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|ClaudeBot|Claude\-User|Claude\-SearchBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|imgproxy|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|QualifiedBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot)") {
     return 403;
 }
diff --git a/table-of-bot-metrics.md b/table-of-bot-metrics.md
@@ -11,8 +11,9 @@
 | Bytespider | ByteDance | No | LLM training. | Unclear at this time. | Downloads data to train LLMS, including ChatGPT competitors. |
 | CCBot | [Common Crawl Foundation](https://commoncrawl.org) | [Yes](https://commoncrawl.org/ccbot) | Provides open crawl dataset, used for many purposes, including Machine Learning/AI. | Monthly at present. | Web archive going back to 2008. [Cited in thousands of research papers per year](https://commoncrawl.org/research-papers). |
 | ChatGPT\-User | [OpenAI](https://openai.com) | Yes | Takes action based on user prompts. | Only when prompted by a user. | Used by plugins in ChatGPT to answer queries based on user input. |
-| Claude\-Web | [Anthropic](https://www.anthropic.com) | Unclear at this time. | Scrapes data to train Anthropic's AI products. | No information provided. | Scrapes data to train LLMs and AI products offered by Anthropic. |
 | ClaudeBot | [Anthropic](https://www.anthropic.com) | [Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler) | Scrapes data to train Anthropic's AI products. | No information provided. | Scrapes data to train LLMs and AI products offered by Anthropic. |
+| Claude\-User | [Anthropic](https://www.anthropic.com) | [Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler) | Claude-User supports Claude AI users. When individuals ask questions to Claude, it may access websites using a Claude-User agent. | No information provided. | Claude-User supports Claude AI users. When individuals ask questions to Claude, it may access websites using a Claude-User agent. |
+| Claude\-SearchBot | [Anthropic](https://www.anthropic.com) | [Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler) | Claude-SearchBot navigates the web to improve search result quality for users. It analyzes online content specifically to enhance the relevance and accuracy of search responses. | No information provided. | Claude-SearchBot navigates the web to improve search result quality for users. It analyzes online content specifically to enhance the relevance and accuracy of search responses. |
 | cohere\-ai | [Cohere](https://cohere.com) | Unclear at this time. | Retrieves data to provide responses to user-initiated prompts. | Takes action based on user prompts. | Retrieves data based on user prompts. |
 | cohere\-training\-data\-crawler | Cohere to download training data for its LLMs (Large Language Models) that power its enterprise AI products | Unclear at this time. | AI Data Scrapers | Unclear at this time. | cohere-training-data-crawler is a web crawler operated by Cohere to download training data for its LLMs (Large Language Models) that power its enterprise AI products. More info can be found at https://darkvisitors.com/agents/agents/cohere-training-data-crawler |
 | Cotoyogi | [ROIS](https://ds.rois.ac.jp/en_center8/en_crawler/) | Yes | AI LLM Scraper. | No information provided. | Scrapes data for AI training in Japanese language. |

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,3 @@`
`1`	`1`	`@aibots {`
`2`		- header_regexp User-Agent "(AI2Bot\|Ai2Bot\-Dolma\|aiHitBot\|Amazonbot\|anthropic\-ai\|Applebot\|Applebot\-Extended\|Brightbot\ 1\.0\|Bytespider\|CCBot\|ChatGPT\-User\|Claude\-Web\|ClaudeBot\|cohere\-ai\|cohere\-training\-data\-crawler\|Cotoyogi\|Crawlspace\|Diffbot\|DuckAssistBot\|FacebookBot\|Factset_spyderbot\|FirecrawlAgent\|FriendlyCrawler\|Google\-CloudVertexBot\|Google\-Extended\|GoogleOther\|GoogleOther\-Image\|GoogleOther\-Video\|GPTBot\|iaskspider/2\.0\|ICC\-Crawler\|ImagesiftBot\|img2dataset\|imgproxy\|ISSCyberRiskCrawler\|Kangaroo\ Bot\|meta\-externalagent\|Meta\-ExternalAgent\|meta\-externalfetcher\|Meta\-ExternalFetcher\|MistralAI\-User/1\.0\|NovaAct\|OAI\-SearchBot\|omgili\|omgilibot\|Operator\|PanguBot\|Perplexity\-User\|PerplexityBot\|PetalBot\|QualifiedBot\|Scrapy\|SemrushBot\-OCOB\|SemrushBot\-SWA\|Sidetrade\ indexer\ bot\|TikTokSpider\|Timpibot\|VelenPublicWebCrawler\|Webzio\-Extended\|YouBot)"
	`2`	+ header_regexp User-Agent "(AI2Bot\|Ai2Bot\-Dolma\|aiHitBot\|Amazonbot\|anthropic\-ai\|Applebot\|Applebot\-Extended\|Brightbot\ 1\.0\|Bytespider\|CCBot\|ChatGPT\-User\|ClaudeBot\|Claude\-User\|Claude\-SearchBot\|cohere\-ai\|cohere\-training\-data\-crawler\|Cotoyogi\|Crawlspace\|Diffbot\|DuckAssistBot\|FacebookBot\|Factset_spyderbot\|FirecrawlAgent\|FriendlyCrawler\|Google\-CloudVertexBot\|Google\-Extended\|GoogleOther\|GoogleOther\-Image\|GoogleOther\-Video\|GPTBot\|iaskspider/2\.0\|ICC\-Crawler\|ImagesiftBot\|img2dataset\|imgproxy\|ISSCyberRiskCrawler\|Kangaroo\ Bot\|meta\-externalagent\|Meta\-ExternalAgent\|meta\-externalfetcher\|Meta\-ExternalFetcher\|MistralAI\-User/1\.0\|NovaAct\|OAI\-SearchBot\|omgili\|omgilibot\|Operator\|PanguBot\|Perplexity\-User\|PerplexityBot\|PetalBot\|QualifiedBot\|Scrapy\|SemrushBot\-OCOB\|SemrushBot\-SWA\|Sidetrade\ indexer\ bot\|TikTokSpider\|Timpibot\|VelenPublicWebCrawler\|Webzio\-Extended\|YouBot)"
`3`	`3`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,3 @@`
`1`		-if ($http_user_agent ~* "(AI2Bot\|Ai2Bot\-Dolma\|aiHitBot\|Amazonbot\|anthropic\-ai\|Applebot\|Applebot\-Extended\|Brightbot\ 1\.0\|Bytespider\|CCBot\|ChatGPT\-User\|Claude\-Web\|ClaudeBot\|cohere\-ai\|cohere\-training\-data\-crawler\|Cotoyogi\|Crawlspace\|Diffbot\|DuckAssistBot\|FacebookBot\|Factset_spyderbot\|FirecrawlAgent\|FriendlyCrawler\|Google\-CloudVertexBot\|Google\-Extended\|GoogleOther\|GoogleOther\-Image\|GoogleOther\-Video\|GPTBot\|iaskspider/2\.0\|ICC\-Crawler\|ImagesiftBot\|img2dataset\|imgproxy\|ISSCyberRiskCrawler\|Kangaroo\ Bot\|meta\-externalagent\|Meta\-ExternalAgent\|meta\-externalfetcher\|Meta\-ExternalFetcher\|MistralAI\-User/1\.0\|NovaAct\|OAI\-SearchBot\|omgili\|omgilibot\|Operator\|PanguBot\|Perplexity\-User\|PerplexityBot\|PetalBot\|QualifiedBot\|Scrapy\|SemrushBot\-OCOB\|SemrushBot\-SWA\|Sidetrade\ indexer\ bot\|TikTokSpider\|Timpibot\|VelenPublicWebCrawler\|Webzio\-Extended\|YouBot)") {
	`1`	+if ($http_user_agent ~* "(AI2Bot\|Ai2Bot\-Dolma\|aiHitBot\|Amazonbot\|anthropic\-ai\|Applebot\|Applebot\-Extended\|Brightbot\ 1\.0\|Bytespider\|CCBot\|ChatGPT\-User\|ClaudeBot\|Claude\-User\|Claude\-SearchBot\|cohere\-ai\|cohere\-training\-data\-crawler\|Cotoyogi\|Crawlspace\|Diffbot\|DuckAssistBot\|FacebookBot\|Factset_spyderbot\|FirecrawlAgent\|FriendlyCrawler\|Google\-CloudVertexBot\|Google\-Extended\|GoogleOther\|GoogleOther\-Image\|GoogleOther\-Video\|GPTBot\|iaskspider/2\.0\|ICC\-Crawler\|ImagesiftBot\|img2dataset\|imgproxy\|ISSCyberRiskCrawler\|Kangaroo\ Bot\|meta\-externalagent\|Meta\-ExternalAgent\|meta\-externalfetcher\|Meta\-ExternalFetcher\|MistralAI\-User/1\.0\|NovaAct\|OAI\-SearchBot\|omgili\|omgilibot\|Operator\|PanguBot\|Perplexity\-User\|PerplexityBot\|PetalBot\|QualifiedBot\|Scrapy\|SemrushBot\-OCOB\|SemrushBot\-SWA\|Sidetrade\ indexer\ bot\|TikTokSpider\|Timpibot\|VelenPublicWebCrawler\|Webzio\-Extended\|YouBot)") {
`2`	`2`	`return 403;`
`3`	`3`	`}`