Merge pull request #140 from ai-robots-txt/yandex-bots

ai.robots.txt · ai.robots.txt · commit 080946c36046 · 2025-06-03T19:51:25.000Z
chore(robots.json): adds YandexAdditional crawlers
diff --git a/.htaccess b/.htaccess
@@ -1,3 +1,3 @@
 RewriteEngine On
-RewriteCond %{HTTP_USER_AGENT} (AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|QualifiedBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|wpbot|YouBot) [NC]
+RewriteCond %{HTTP_USER_AGENT} (AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|QualifiedBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|wpbot|YandexAdditional|YandexAdditionalBot|YouBot) [NC]
 RewriteRule !^/?robots\.txt$ - [F,L]
diff --git a/Caddyfile b/Caddyfile
@@ -1,3 +1,3 @@
 @aibots {
-        header_regexp User-Agent "(AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|QualifiedBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|wpbot|YouBot)"
+        header_regexp User-Agent "(AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|QualifiedBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|wpbot|YandexAdditional|YandexAdditionalBot|YouBot)"
 }
diff --git a/haproxy-block-ai-bots.txt b/haproxy-block-ai-bots.txt
@@ -61,4 +61,6 @@ Timpibot
 VelenPublicWebCrawler
 Webzio-Extended
 wpbot
+YandexAdditional
+YandexAdditionalBot
 YouBot
diff --git a/nginx-block-ai-bots.conf b/nginx-block-ai-bots.conf
@@ -1,3 +1,3 @@
-if ($http_user_agent ~* "(AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|QualifiedBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|wpbot|YouBot)") {
+if ($http_user_agent ~* "(AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|QualifiedBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|wpbot|YandexAdditional|YandexAdditionalBot|YouBot)") {
     return 403;
 }
diff --git a/robots.txt b/robots.txt
@@ -61,5 +61,7 @@ User-agent: Timpibot
 User-agent: VelenPublicWebCrawler
 User-agent: Webzio-Extended
 User-agent: wpbot
+User-agent: YandexAdditional
+User-agent: YandexAdditionalBot
 User-agent: YouBot
 Disallow: /
diff --git a/table-of-bot-metrics.md b/table-of-bot-metrics.md
@@ -63,4 +63,6 @@
 | VelenPublicWebCrawler | [Velen Crawler](https://velen.io) | [Yes](https://velen.io) | Scrapes data for business data sets and machine learning models. | No information. | "Our goal with this crawler is to build business datasets and machine learning models to better understand the web." |
 | Webzio\-Extended | Unclear at this time. | Unclear at this time. | AI Data Scrapers | Unclear at this time. | Webzio-Extended is a web crawler used by Webz.io to maintain a repository of web crawl data that it sells to other companies, including those using it to train AI models. More info can be found at https://darkvisitors.com/agents/agents/webzio-extended |
 | wpbot | [QuantumCloud](https://www.quantumcloud.com) | Unclear at this time; opt out provided via [Google Form](https://forms.gle/ajBaxygz9jSR8p8G9) | Live chat support and lead generation. | Unclear at this time. | wpbot is a used to support the functionality of the AI Chatbot for WordPress plugin. It supports the use of customer models, data collection and customer support. |
+| YandexAdditional | [Yandex](https://yandex.ru) | [Yes](https://yandex.ru/support/webmaster/en/search-appearance/fast.html?lang=en) | Scrapes/analyzes data for the YandexGPT LLM. | No information. | Retrieves data used for YandexGPT quick answers features. |
+| YandexAdditionalBot | [Yandex](https://yandex.ru) | [Yes](https://yandex.ru/support/webmaster/en/search-appearance/fast.html?lang=en) | Scrapes/analyzes data for the YandexGPT LLM. | No information. | Retrieves data used for YandexGPT quick answers features. |
 | YouBot | [You](https://about.you.com/youchat/) | [Yes](https://about.you.com/youbot/) | Scrapes data for search engine and LLMs. | No information. | Retrieves data used for You.com web search engine and LLMs. |

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,3 @@`
`1`	`1`	`@aibots {`
`2`		- header_regexp User-Agent "(AI2Bot\|Ai2Bot\-Dolma\|aiHitBot\|Amazonbot\|Andibot\|anthropic\-ai\|Applebot\|Applebot\-Extended\|Brightbot\ 1\.0\|Bytespider\|CCBot\|ChatGPT\-User\|Claude\-SearchBot\|Claude\-User\|Claude\-Web\|ClaudeBot\|cohere\-ai\|cohere\-training\-data\-crawler\|Cotoyogi\|Crawlspace\|Diffbot\|DuckAssistBot\|FacebookBot\|Factset_spyderbot\|FirecrawlAgent\|FriendlyCrawler\|Google\-CloudVertexBot\|Google\-Extended\|GoogleOther\|GoogleOther\-Image\|GoogleOther\-Video\|GPTBot\|iaskspider/2\.0\|ICC\-Crawler\|ImagesiftBot\|img2dataset\|ISSCyberRiskCrawler\|Kangaroo\ Bot\|meta\-externalagent\|Meta\-ExternalAgent\|meta\-externalfetcher\|Meta\-ExternalFetcher\|MistralAI\-User/1\.0\|NovaAct\|OAI\-SearchBot\|omgili\|omgilibot\|Operator\|PanguBot\|Perplexity\-User\|PerplexityBot\|PetalBot\|PhindBot\|QualifiedBot\|Scrapy\|SemrushBot\-OCOB\|SemrushBot\-SWA\|Sidetrade\ indexer\ bot\|TikTokSpider\|Timpibot\|VelenPublicWebCrawler\|Webzio\-Extended\|wpbot\|YouBot)"
	`2`	+ header_regexp User-Agent "(AI2Bot\|Ai2Bot\-Dolma\|aiHitBot\|Amazonbot\|Andibot\|anthropic\-ai\|Applebot\|Applebot\-Extended\|Brightbot\ 1\.0\|Bytespider\|CCBot\|ChatGPT\-User\|Claude\-SearchBot\|Claude\-User\|Claude\-Web\|ClaudeBot\|cohere\-ai\|cohere\-training\-data\-crawler\|Cotoyogi\|Crawlspace\|Diffbot\|DuckAssistBot\|FacebookBot\|Factset_spyderbot\|FirecrawlAgent\|FriendlyCrawler\|Google\-CloudVertexBot\|Google\-Extended\|GoogleOther\|GoogleOther\-Image\|GoogleOther\-Video\|GPTBot\|iaskspider/2\.0\|ICC\-Crawler\|ImagesiftBot\|img2dataset\|ISSCyberRiskCrawler\|Kangaroo\ Bot\|meta\-externalagent\|Meta\-ExternalAgent\|meta\-externalfetcher\|Meta\-ExternalFetcher\|MistralAI\-User/1\.0\|NovaAct\|OAI\-SearchBot\|omgili\|omgilibot\|Operator\|PanguBot\|Perplexity\-User\|PerplexityBot\|PetalBot\|PhindBot\|QualifiedBot\|Scrapy\|SemrushBot\-OCOB\|SemrushBot\-SWA\|Sidetrade\ indexer\ bot\|TikTokSpider\|Timpibot\|VelenPublicWebCrawler\|Webzio\-Extended\|wpbot\|YandexAdditional\|YandexAdditionalBot\|YouBot)"
`3`	`3`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,3 @@`
`1`		-if ($http_user_agent ~* "(AI2Bot\|Ai2Bot\-Dolma\|aiHitBot\|Amazonbot\|Andibot\|anthropic\-ai\|Applebot\|Applebot\-Extended\|Brightbot\ 1\.0\|Bytespider\|CCBot\|ChatGPT\-User\|Claude\-SearchBot\|Claude\-User\|Claude\-Web\|ClaudeBot\|cohere\-ai\|cohere\-training\-data\-crawler\|Cotoyogi\|Crawlspace\|Diffbot\|DuckAssistBot\|FacebookBot\|Factset_spyderbot\|FirecrawlAgent\|FriendlyCrawler\|Google\-CloudVertexBot\|Google\-Extended\|GoogleOther\|GoogleOther\-Image\|GoogleOther\-Video\|GPTBot\|iaskspider/2\.0\|ICC\-Crawler\|ImagesiftBot\|img2dataset\|ISSCyberRiskCrawler\|Kangaroo\ Bot\|meta\-externalagent\|Meta\-ExternalAgent\|meta\-externalfetcher\|Meta\-ExternalFetcher\|MistralAI\-User/1\.0\|NovaAct\|OAI\-SearchBot\|omgili\|omgilibot\|Operator\|PanguBot\|Perplexity\-User\|PerplexityBot\|PetalBot\|PhindBot\|QualifiedBot\|Scrapy\|SemrushBot\-OCOB\|SemrushBot\-SWA\|Sidetrade\ indexer\ bot\|TikTokSpider\|Timpibot\|VelenPublicWebCrawler\|Webzio\-Extended\|wpbot\|YouBot)") {
	`1`	+if ($http_user_agent ~* "(AI2Bot\|Ai2Bot\-Dolma\|aiHitBot\|Amazonbot\|Andibot\|anthropic\-ai\|Applebot\|Applebot\-Extended\|Brightbot\ 1\.0\|Bytespider\|CCBot\|ChatGPT\-User\|Claude\-SearchBot\|Claude\-User\|Claude\-Web\|ClaudeBot\|cohere\-ai\|cohere\-training\-data\-crawler\|Cotoyogi\|Crawlspace\|Diffbot\|DuckAssistBot\|FacebookBot\|Factset_spyderbot\|FirecrawlAgent\|FriendlyCrawler\|Google\-CloudVertexBot\|Google\-Extended\|GoogleOther\|GoogleOther\-Image\|GoogleOther\-Video\|GPTBot\|iaskspider/2\.0\|ICC\-Crawler\|ImagesiftBot\|img2dataset\|ISSCyberRiskCrawler\|Kangaroo\ Bot\|meta\-externalagent\|Meta\-ExternalAgent\|meta\-externalfetcher\|Meta\-ExternalFetcher\|MistralAI\-User/1\.0\|NovaAct\|OAI\-SearchBot\|omgili\|omgilibot\|Operator\|PanguBot\|Perplexity\-User\|PerplexityBot\|PetalBot\|PhindBot\|QualifiedBot\|Scrapy\|SemrushBot\-OCOB\|SemrushBot\-SWA\|Sidetrade\ indexer\ bot\|TikTokSpider\|Timpibot\|VelenPublicWebCrawler\|Webzio\-Extended\|wpbot\|YandexAdditional\|YandexAdditionalBot\|YouBot)") {
`2`	`2`	`return 403;`
`3`	`3`	`}`