From bac5c66b9aa861edcee4b65f903de2ffadc2f9ab Mon Sep 17 00:00:00 2001
From: Brian Kelly <polymonic@gmail.com>
Date: Thu, 25 Jul 2024 11:48:07 -0500
Subject: [PATCH] Add GeoBlacklight suggested robots.txt entries

---
 public/robots.txt | 46 +++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 45 insertions(+), 1 deletion(-)

diff --git a/public/robots.txt b/public/robots.txt
index c19f78ab..c963f454 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -1 +1,45 @@
-# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
+User-agent: *
+Disallow: /?q=
+Disallow: /*?q=*
+Disallow: /?f
+Disallow: /*?f*
+Disallow: /?_
+Disallow: /?bbox
+Disallow: /?page=
+Disallow: /bookmarks
+Disallow: /catalog.html?f
+Disallow: /catalog.html?_
+Disallow: /catalog.atom
+Disallow: /catalog.rss
+Disallow: /catalog/*/relations
+Disallow: /catalog/facet/*
+Disallow: /catalog/*/web_services
+Disallow: /catalog/email
+Disallow: /catalog/opensearch
+Disallow: /catalog/range_limit
+Disallow: /catalog/sms
+Disallow: /saved_searches
+Disallow: /search_history
+Disallow: /suggest
+Disallow: /users
+Disallow: /404
+Disallow: /422
+Disallow: /500
+
+User-agent: AhrefsBot
+Disallow: /
+
+User-agent: SemrushBot
+Disallow: /
+
+User-agent: PetalBot
+Disallow: /
+
+User-agent: BLEXBot
+Disallow: /
+
+User-agent: DotBot
+Disallow: /
+
+User-agent: DataForSeoBot
+Disallow: /