draft implementation of github tag crawler

Author: tayloraswift

Package.swift

@@ -75,6 +75,8 @@ let package:Package = .init(
         .package(url: "https://github.com/tayloraswift/swift-mongodb", .upToNextMinor(
             from: "0.8.2")),
 
+        .package(url: "https://github.com/apple/swift-atomics", .upToNextMinor(
+            from: "1.1.0")),
         .package(url: "https://github.com/apple/swift-nio", .upToNextMinor(
             from: "2.57.0")),
         .package(url: "https://github.com/apple/swift-nio-http2", .upToNextMinor(
@@ -379,6 +381,7 @@ let package:Package = .init(
                 .target(name: "GitHubIntegration"),
                 .target(name: "UnidocAnalysis"),
                 .target(name: "UnidocLinker"),
+                .target(name: "UnixTime"),
                 .product(name: "MongoDB", package: "swift-mongodb"),
             ]),
 
@@ -454,6 +457,8 @@ let package:Package = .init(
                 .target(name: "Multiparts"),
                 .target(name: "System"),
                 .target(name: "UnidocPages"),
+
+                .product(name: "Atomics", package: "swift-atomics"),
             ]),
 
 

Sources/UnidocDB/Packages/PackageDatabase.Packages.swift

@@ -40,9 +40,56 @@ extension PackageDatabase.Packages:DatabaseCollection
                 $0[PackageRecord[.cell]] = (+)
             }
         },
+        .init
+        {
+            $0[.unique] = false
+            $0[.name] = "crawled"
+            $0[.key] = .init
+            {
+                $0[PackageRecord[.crawled]] = (+)
+            }
+        },
     ]
 }
 extension PackageDatabase.Packages
+{
+    public
+    func recode(with session:Mongo.Session) async throws -> (modified:Int, of:Int)
+    {
+        try await self.recode(through: PackageRecord.self,
+            with: session,
+            by: .now.advanced(by: .seconds(30)))
+    }
+}
+extension PackageDatabase.Packages
+{
+    public
+    func stalest(_ limit:Int, with session:Mongo.Session) async throws -> [PackageRecord]
+    {
+        try await session.run(
+            command: Mongo.Find<Mongo.SingleBatch<PackageRecord>>.init(Self.name,
+                limit: limit)
+            {
+                $0[.sort] = .init
+                {
+                    $0[PackageRecord[.crawled]] = (+)
+                }
+                $0[.hint] = .init
+                {
+                    $0[PackageRecord[.crawled]] = (+)
+                }
+            },
+            against: self.database)
+    }
+
+    public
+    func update(record:PackageRecord,
+        with session:Mongo.Session) async throws -> Bool?
+    {
+        try await self.update(record, with: session)
+    }
+}
+extension PackageDatabase.Packages
 {
     /// Registers the given package identifier in the database, returning its package
     /// coordinate. This is really just a glorified string internment system.
@@ -157,19 +204,6 @@ extension PackageDatabase.Packages
         return placement.first ?? .first
     }
 
-    @discardableResult
-    public
-    func update(_ package:Int32,
-        repo:PackageRepo,
-        with session:Mongo.Session) async throws -> Bool?
-    {
-        try await self.update(field: PackageRecord[.repo],
-            by: PackageRecord[.cell],
-            of: package,
-            to: repo,
-            with: session)
-    }
-
     private
     func scan(with session:Mongo.Session) async throws -> SearchIndex<Int32>
     {

Sources/UnidocDB/Packages/PackageRecord.swift

@@ -3,6 +3,7 @@ import BSONEncoding
 import ModuleGraphs
 import MongoQL
 import SymbolGraphs
+import UnixTime
 
 @frozen public
 struct PackageRecord:Identifiable, Equatable, Sendable
@@ -13,15 +14,24 @@ struct PackageRecord:Identifiable, Equatable, Sendable
     /// but packages that track remote repositories will always have positive cell numbers.
     public
     let cell:Int32
+
+    /// When this package *record* was last crawled. This is different from the time when the
+    /// package itself was last updated.
+    public
+    var crawled:BSON.Millisecond
     /// The repo this package tracks. Currently only GitHub repos are supported.
     public
     var repo:PackageRepo?
 
     @inlinable public
-    init(id:PackageIdentifier, cell:Int32, repo:PackageRepo? = nil)
+    init(id:PackageIdentifier,
+        cell:Int32,
+        crawled:BSON.Millisecond = .now(),
+        repo:PackageRepo? = nil)
     {
         self.id = id
         self.cell = cell
+        self.crawled = crawled
         self.repo = repo
     }
 }
@@ -32,6 +42,7 @@ extension PackageRecord:MongoMasterCodingModel
     {
         case id = "_id"
         case cell = "P"
+        case crawled = "T"
         case repo = "R"
     }
 }
@@ -42,6 +53,7 @@ extension PackageRecord:BSONDocumentEncodable
     {
         bson[.id] = self.id
         bson[.cell] = self.cell
+        bson[.crawled] = self.crawled
         bson[.repo] = self.repo
     }
 }
@@ -52,6 +64,7 @@ extension PackageRecord:BSONDocumentDecodable
     {
         self.init(id: try bson[.id].decode(),
             cell: try bson[.cell].decode(),
+            crawled: try bson[.crawled]?.decode() ?? .now(),
             repo: try bson[.repo]?.decode())
     }
 }

Sources/UnidocPages/Templates/Site.Admin.Action.swift

@@ -10,6 +10,7 @@ extension Site.Admin
         case lintPackageEditions = "lint-package-editions"
 
         case recodePackageEditions = "recode-package-editions"
+        case recodePackageRecords = "recode-package-records"
         case recodeUnidocVertices = "recode-unidoc-vertices"
 
         case rebuild = "rebuild"
@@ -27,6 +28,7 @@ extension Site.Admin.Action
         case .dropUnidocDB:             return "Drop Unidoc Database"
         case .lintPackageEditions:      return "Lint Package Editions"
         case .recodePackageEditions:    return "Recode Package Editions"
+        case .recodePackageRecords:     return "Recode Package Records"
         case .recodeUnidocVertices:     return "Recode Unidoc Vertices"
         case .rebuild:                  return "Rebuild Collections"
         case .upload:                   return "Upload Snapshots"

Sources/UnidocPages/Templates/Site.Admin.Confirm.swift

@@ -41,6 +41,9 @@ extension Site.Admin.Confirm
         case .recodePackageEditions:
             prompt = "This will recode all package editions. Are you sure?"
 
+        case .recodePackageRecords:
+            prompt = "This will recode all package records. Are you sure?"
+
         case .recodeUnidocVertices:
             prompt = "This will recode all Unidoc vertices. Are you sure?"
 

Sources/UnidocPages/Templates/Site.Admin.swift

@@ -11,15 +11,41 @@ extension Site
     {
         public
         let configuration:Mongo.ReplicaSetConfiguration
+
+        public
+        let crawlingErrors:Int
+        public
+        let packagesCrawled:Int
+        public
+        let packagesUpdated:Int
+        public
+        let tagsCrawled:Int
+        public
+        let tagsUpdated:Int
+
         public
         let tour:ServerTour
         public
         let real:Bool
 
         @inlinable public
-        init(configuration:Mongo.ReplicaSetConfiguration, tour:ServerTour, real:Bool)
+        init(configuration:Mongo.ReplicaSetConfiguration,
+            crawlingErrors:Int,
+            packagesCrawled:Int,
+            packagesUpdated:Int,
+            tagsCrawled:Int,
+            tagsUpdated:Int,
+            tour:ServerTour,
+            real:Bool)
         {
             self.configuration = configuration
+
+            self.crawlingErrors = crawlingErrors
+            self.packagesCrawled = packagesCrawled
+            self.packagesUpdated = packagesUpdated
+            self.tagsCrawled = tagsCrawled
+            self.tagsUpdated = tagsUpdated
+
             self.tour = tour
             self.real = real
         }
@@ -148,6 +174,7 @@ extension Site.Admin:AdministrativePage
         [
             .lintPackageEditions,
             .recodePackageEditions,
+            .recodePackageRecords,
             .recodeUnidocVertices,
 
             .dropUnidocDB,
@@ -186,6 +213,21 @@ extension Site.Admin:AdministrativePage
 
             $0[.dt] = "bytes transferred (content only)"
             $0[.dd] = "\(self.tour.stats.bytes.total)"
+
+            $0[.dt] = "crawling errors"
+            $0[.dd] = "\(self.crawlingErrors)"
+
+            $0[.dt] = "packages crawled"
+            $0[.dd] = "\(self.packagesCrawled)"
+
+            $0[.dt] = "packages updated"
+            $0[.dd] = "\(self.packagesUpdated)"
+
+            $0[.dt] = "tags crawled"
+            $0[.dd] = "\(self.tagsCrawled)"
+
+            $0[.dt] = "tags updated"
+            $0[.dd] = "\(self.tagsUpdated)"
         }
 
         main += ServerTour.StatsBreakdown.init(self.tour.stats)

Sources/UnidocServer/Caching/Cache.swift

@@ -13,7 +13,7 @@ actor Cache<Key> where Key:CacheKey
     private
     var table:[Key: ServerResource]
 
-    init(source assets:FilePath, reload:Bool)
+    init(source assets:FilePath = "Assets", reload:Bool)
     {
         self.assets = assets
         self.reload = reload

Sources/UnidocServer/DatabaseOperation.swift

@@ -1,14 +1,14 @@
 import HTTP
 import UnidocPages
 
-protocol StatefulOperation:Sendable
+protocol InteractiveOperation:Sendable
 {
     var statisticalType:WritableKeyPath<ServerTour.Stats.ByType, Int> { get }
 
-    func load(from state:ServerState,
+    func load(from server:Server.State,
         with cookies:Server.Request.Cookies) async throws -> ServerResponse?
 }
-extension StatefulOperation
+extension InteractiveOperation
 {
     var statisticalType:WritableKeyPath<ServerTour.Stats.ByType, Int>
     {