serve an empty robots.txt instead of redirecting to stdlib page

Author: tayloraswift

Assets/robots.txt

@@ -18,6 +18,8 @@ extension Site
 
         case main_js            = "main.js"
         case main_js_map        = "main.js.map"
+
+        case robots_txt         = "robots.txt"
     }
 }
 extension Site.Asset:FixedRoot

Sources/UnidocPages/Templates/Site.Asset.swift

@@ -14,7 +14,8 @@ extension Site.Asset:CacheKey
         case    .literata45_woff2,
                 .literata47_woff2,
                 .literata75_woff2,
-                .literata77_woff2:  return .cold
+                .literata77_woff2,
+                .robots_txt:        return .cold
         case    .main_css,
                 .main_js:           return nil
         case    .main_css_map,
@@ -30,6 +31,7 @@ extension Site.Asset:CacheKey
         case .literata47_woff2: return ["woff2", "Literata_24pt-Italic.woff2"]
         case .literata75_woff2: return ["woff2", "Literata_24pt-Bold.woff2"]
         case .literata77_woff2: return ["woff2", "Literata_24pt-BoldItalic.woff2"]
+        case .robots_txt:       return ["robots.txt"]
         case .main_css:         return ["css", "Main.css"]
         case .main_css_map:     return ["css", "Main.css.map"]
         case .main_js:          return ["js", "Main.js"]
@@ -41,6 +43,7 @@ extension Site.Asset:CacheKey
     {
         switch self
         {
+        case    .robots_txt:        return .text(.plain, charset: .utf8)
         case    .literata45_woff2,
                 .literata47_woff2,
                 .literata75_woff2,

Sources/UnidocServer/Caching/Site.Asset (ext).swift

@@ -40,6 +40,7 @@ extension AnyOperation
         switch root
         {
         case Site.Admin.root:   return .database(AdminOperation.status)
+        case "robots.txt":      return .datafile(.init(.robots_txt, tag: tag))
         case _:                 return nil
         }
     }