add a stats counter for average crawl staleness

Author: tayloraswift

Sources/SwiftinitPages/Surfaces/Administration/Swiftinit.AdminPage.swift

@@ -17,6 +17,8 @@ extension Swiftinit
         public
         let requestsDropped:Int
         public
+        let averagePackageStaleness:Duration
+        public
         let errorsCrawling:Int
         public
         let reposCrawled:Int
@@ -35,6 +37,7 @@ extension Swiftinit
         @inlinable public
         init(configuration:Mongo.ReplicaSetConfiguration,
             requestsDropped:Int,
+            averagePackageStaleness:Duration,
             errorsCrawling:Int,
             reposCrawled:Int,
             reposUpdated:Int,
@@ -46,6 +49,7 @@ extension Swiftinit
             self.configuration = configuration
 
             self.requestsDropped = requestsDropped
+            self.averagePackageStaleness = averagePackageStaleness
             self.errorsCrawling = errorsCrawling
             self.reposCrawled = reposCrawled
             self.reposUpdated = reposUpdated
@@ -250,19 +254,48 @@ extension Swiftinit.AdminPage:Swiftinit.AdministrativePage
 
         main[.dl]
         {
+            let uptime:Age = .init(self.tour.started.duration(to: .now))
+
             $0[.dt] = "Uptime"
-            $0[.dd] = "\(self.tour.started.duration(to: .now))"
+            $0[.dd] = uptime.long
 
             let requests:Int =
                 self.tour.profile.requests.http1.total +
                 self.tour.profile.requests.http2.total
+
             $0[.dt] = "Requests"
             $0[.dd] = "\(requests)"
 
             $0[.dt] = "Requests (Barbies)"
             $0[.dd] = "\(self.tour.profile.requests.http2.barbie)"
 
+            $0[.dt] = "requests dropped"
+            $0[.dd] = "\(self.requestsDropped)"
+
+            let staleness:Duration = self.averagePackageStaleness
+            $0[.dt] = "Average package staleness"
+            $0[.dd] = "\(staleness)"
+
+            $0[.dt] = "GitHub crawling errors"
+            $0[.dd] = "\(self.errorsCrawling)"
+
+            $0[.dt] = "GitHub repos crawled"
+            $0[.dd] = "\(self.reposCrawled)"
+
+            $0[.dt] = "GitHub repos updated"
+            $0[.dd] = "\(self.reposUpdated)"
+
+            $0[.dt] = "GitHub tags crawled"
+            $0[.dd] = "\(self.tagsCrawled)"
+
+            $0[.dt] = "GitHub tags updated"
+            $0[.dd] = "\(self.tagsUpdated)"
+        }
 
+        main[.h2] = "Performance"
+
+        main[.dl]
+        {
             if  let last:ServerTour.Request = self.tour.lastImpression
             {
                 $0[.h3] = "Last Impression"
@@ -321,24 +354,6 @@ extension Swiftinit.AdminPage:Swiftinit.AdministrativePage
 
             $0[.dt] = "bytes transferred (content only)"
             $0[.dd] = "\(self.tour.profile.requests.bytes.total)"
-
-            $0[.dt] = "requests dropped"
-            $0[.dd] = "\(self.requestsDropped)"
-
-            $0[.dt] = "GitHub crawling errors"
-            $0[.dd] = "\(self.errorsCrawling)"
-
-            $0[.dt] = "GitHub repos crawled"
-            $0[.dd] = "\(self.reposCrawled)"
-
-            $0[.dt] = "GitHub repos updated"
-            $0[.dd] = "\(self.reposUpdated)"
-
-            $0[.dt] = "GitHub tags crawled"
-            $0[.dd] = "\(self.tagsCrawled)"
-
-            $0[.dt] = "GitHub tags updated"
-            $0[.dd] = "\(self.tagsUpdated)"
         }
 
         main += self.tour.profile

Sources/SwiftinitServer/Endpoints/Interactive/Swiftinit.AdminDashboardEndpoint.swift

@@ -23,6 +23,8 @@ extension Swiftinit.AdminDashboardEndpoint:RestrictedEndpoint
 
             .init(configuration: configuration,
                 requestsDropped: counters.requestsDropped.load(ordering: .relaxed),
+                averagePackageStaleness: .milliseconds(counters.averagePackageStaleness.load(
+                    ordering: .relaxed)),
                 errorsCrawling: counters.errorsCrawling.load(ordering: .relaxed),
                 reposCrawled: counters.reposCrawled.load(ordering: .relaxed),
                 reposUpdated: counters.reposUpdated.load(ordering: .relaxed),

Sources/SwiftinitServer/Plugins/GitHubCrawler.swift

@@ -12,12 +12,14 @@ protocol GitHubCrawler
 
     var api:GitHub.Client<GitHub.API> { get }
 
+    mutating
     func crawl(updating server:Swiftinit.ServerLoop,
         over connection:GitHub.Client<GitHub.API>.Connection,
         with session:Mongo.Session) async throws
 }
 extension GitHubCrawler
 {
+    mutating
     func run(alongside server:Swiftinit.ServerLoop) async throws
     {
         while true

Sources/SwiftinitServer/Plugins/GitHubPlugin.RepoMonitor.swift

@@ -1,10 +1,41 @@
+import BSON
 import GitHubAPI
 import GitHubClient
 import MongoDB
 import SemanticVersions
 import UnidocDB
 import UnidocRecords
 
+extension GitHubPlugin
+{
+    struct Average
+    {
+        private(set)
+        var total:Double
+        private(set)
+        var count:Int
+
+        init()
+        {
+            self.total = 0
+            self.count = 0
+        }
+    }
+}
+extension GitHubPlugin.Average
+{
+    mutating
+    func insert(_ value:Double)
+    {
+        self.total += value
+        self.count += 1
+    }
+
+    var value:Double?
+    {
+        self.count > 0 ? self.total / Double.init(self.count) : nil
+    }
+}
 extension GitHubPlugin
 {
     struct RepoMonitor
@@ -14,10 +45,15 @@ extension GitHubPlugin
         private
         let pat:String
 
+        private
+        var staleness:Average
+
         init(api:GitHub.Client<GitHub.API>, pat:String)
         {
             self.api = api
             self.pat = pat
+
+            self.staleness = .init()
         }
     }
 }
@@ -26,6 +62,7 @@ extension GitHubPlugin.RepoMonitor:GitHubCrawler
     static
     var interval:Duration { .seconds(30) }
 
+    mutating
     func crawl(updating server:Swiftinit.ServerLoop,
         over connection:GitHub.Client<GitHub.API>.Connection,
         with session:Mongo.Session) async throws
@@ -53,8 +90,27 @@ extension GitHubPlugin.RepoMonitor:GitHubCrawler
                 repo: origin.name,
                 pat: self.pat)
 
+            let now:BSON.Millisecond = .now()
+
+            staleness:
+            if  package.crawled != 0
+            {
+                //  Not entirely accurate (leap seconds!!!), but good enough for stats.
+                self.staleness.insert(Double.init(now.value - package.crawled.value))
+
+                guard
+                let average:Double = self.staleness.value
+                else
+                {
+                    break staleness
+                }
+
+                server.atomics.averagePackageStaleness.store(Int.init(average),
+                    ordering: .relaxed)
+            }
+
             package.repo = try .github(response.repo)
-            package.crawled = .now()
+            package.crawled = now
 
             switch try await server.db.packages.update(metadata: package, with: session)
             {

Sources/SwiftinitServer/Server/Swiftinit.Counters.swift

@@ -6,6 +6,8 @@ extension Swiftinit
     {
         let requestsDropped:UnsafeAtomic<Int>
 
+        /// Has units of milliseconds, but we don’t have a way to type that with `UnsafeAtomic`.
+        let averagePackageStaleness:UnsafeAtomic<Int>
         let errorsCrawling:UnsafeAtomic<Int>
         let reposCrawled:UnsafeAtomic<Int>
         let reposUpdated:UnsafeAtomic<Int>
@@ -15,6 +17,8 @@ extension Swiftinit
         init()
         {
             self.requestsDropped = .create(0)
+
+            self.averagePackageStaleness = .create(0)
             self.errorsCrawling = .create(0)
             self.reposCrawled = .create(0)
             self.reposUpdated = .create(0)
@@ -25,6 +29,8 @@ extension Swiftinit
         deinit
         {
             self.requestsDropped.destroy()
+
+            self.averagePackageStaleness.destroy()
             self.errorsCrawling.destroy()
             self.reposCrawled.destroy()
             self.reposUpdated.destroy()

Sources/SwiftinitServer/Swiftinit.ServerLoop.swift

@@ -148,11 +148,13 @@ extension Swiftinit.ServerLoop
             {
                 tasks.addTask
                 {
-                    try await plugin.telescope.run(alongside: self)
+                    var telescope:GitHubPlugin.RepoTelescope = plugin.telescope
+                    try await telescope.run(alongside: self)
                 }
                 tasks.addTask
                 {
-                    try await plugin.monitor.run(alongside: self)
+                    var monitor:GitHubPlugin.RepoMonitor = plugin.monitor
+                    try await monitor.run(alongside: self)
                 }
             }
             if  let plugin:Swiftinit.PluginIntegration<PolicyPlugin> = self.plugins.policy