From dc08e2a0d919be33f2ec438203080a06e3a2e5f0 Mon Sep 17 00:00:00 2001
From: Brian Kelly <polymonic@gmail.com>
Date: Fri, 26 Jul 2024 08:48:25 -0500
Subject: [PATCH] Adds robots.txt entries from Faculty Digital Archive

---
 public/robots.txt | 85 ++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 81 insertions(+), 4 deletions(-)

diff --git a/public/robots.txt b/public/robots.txt
index c963f454..db4a11b3 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -26,20 +26,97 @@ Disallow: /404
 Disallow: /422
 Disallow: /500
 
+# Block Specific Crawlers & Bots
+
 User-agent: AhrefsBot
 Disallow: /
 
-User-agent: SemrushBot
+User-agent: BLEXBot
 Disallow: /
 
-User-agent: PetalBot
+User-agent: DataForSeoBot
 Disallow: /
 
-User-agent: BLEXBot
+User-agent: DOC
 Disallow: /
 
 User-agent: DotBot
 Disallow: /
 
-User-agent: DataForSeoBot
+User-agent: Download Ninja
+Disallow: /
+
+User-agent: fast
+Disallow: /
+
+User-agent: Fetch
+Disallow: /
+
+User-agent: grub-client
+Disallow: /
+
+User-agent: HTTrack
+Disallow: /
+
+User-agent: k2spider
+Disallow: /
+
+User-agent: libwww
+Disallow: /
+
+User-agent: linko
+Disallow: /
+
+User-agent: Mediapartners-Google*
+Disallow: /
+
+User-agent: Microsoft.URL.Control
+Disallow: /
+
+User-agent: MSIECrawler
+Disallow: /
+
+User-agent: NPBot
+Disallow: /
+
+User-agent: PetalBot
+Disallow: /
+
+User-agent: SemrushBot
+Disallow: /
+
+User-agent: SiteSnagger
+Disallow: /
+
+User-agent: sitecheck.internetseer.com
+Disallow: /
+
+User-agent: Teleport
+Disallow: /
+
+User-agent: TeleportPro
+Disallow: /
+
+User-agent: WebCopier
+Disallow: /
+
+User-agent: WebReaper
+Disallow: /
+
+User-agent: WebStripper
+Disallow: /
+
+User-agent: WebZIP
+Disallow: /
+
+User-agent: Xenu
+Disallow: /
+
+User-agent: Zealbot
+Disallow: /
+
+User-agent: ZyBORG
+Disallow: /
+
+User-agent: Zao
 Disallow: /