diff --git a/public/robots.txt b/public/robots.txt index c963f454..db4a11b3 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -26,20 +26,97 @@ Disallow: /404 Disallow: /422 Disallow: /500 +# Block Specific Crawlers & Bots + User-agent: AhrefsBot Disallow: / -User-agent: SemrushBot +User-agent: BLEXBot Disallow: / -User-agent: PetalBot +User-agent: DataForSeoBot Disallow: / -User-agent: BLEXBot +User-agent: DOC Disallow: / User-agent: DotBot Disallow: / -User-agent: DataForSeoBot +User-agent: Download Ninja +Disallow: / + +User-agent: fast +Disallow: / + +User-agent: Fetch +Disallow: / + +User-agent: grub-client +Disallow: / + +User-agent: HTTrack +Disallow: / + +User-agent: k2spider +Disallow: / + +User-agent: libwww +Disallow: / + +User-agent: linko +Disallow: / + +User-agent: Mediapartners-Google* +Disallow: / + +User-agent: Microsoft.URL.Control +Disallow: / + +User-agent: MSIECrawler +Disallow: / + +User-agent: NPBot +Disallow: / + +User-agent: PetalBot +Disallow: / + +User-agent: SemrushBot +Disallow: / + +User-agent: SiteSnagger +Disallow: / + +User-agent: sitecheck.internetseer.com +Disallow: / + +User-agent: Teleport +Disallow: / + +User-agent: TeleportPro +Disallow: / + +User-agent: WebCopier +Disallow: / + +User-agent: WebReaper +Disallow: / + +User-agent: WebStripper +Disallow: / + +User-agent: WebZIP +Disallow: / + +User-agent: Xenu +Disallow: / + +User-agent: Zealbot +Disallow: / + +User-agent: ZyBORG +Disallow: / + +User-agent: Zao Disallow: /