factor the GitHub crawling functions into a type GitHubPlugin.Crawler

Author: tayloraswift

README.md

@@ -1,6 +1,6 @@
 <div align="center">
 
-<strong><em><code>unidoc</code></em></strong><br><small><code>0.2.2</code></small>
+<strong><em><code>unidoc</code></em></strong><br><small><code>0.2.3</code></small>
 
 [![ci build status](https://github.com/kelvin13/swift-unidoc/actions/workflows/build.yml/badge.svg)](https://github.com/kelvin13/swift-unidoc/actions/workflows/build.yml)
 

Sources/UnidocPages/Templates/Site.Admin.swift

@@ -13,11 +13,11 @@ extension Site
         let configuration:Mongo.ReplicaSetConfiguration
 
         public
-        let crawlingErrors:Int
+        let errorsCrawling:Int
         public
-        let packagesCrawled:Int
+        let reposCrawled:Int
         public
-        let packagesUpdated:Int
+        let reposUpdated:Int
         public
         let tagsCrawled:Int
         public
@@ -30,19 +30,19 @@ extension Site
 
         @inlinable public
         init(configuration:Mongo.ReplicaSetConfiguration,
-            crawlingErrors:Int,
-            packagesCrawled:Int,
-            packagesUpdated:Int,
+            errorsCrawling:Int,
+            reposCrawled:Int,
+            reposUpdated:Int,
             tagsCrawled:Int,
             tagsUpdated:Int,
             tour:ServerTour,
             real:Bool)
         {
             self.configuration = configuration
 
-            self.crawlingErrors = crawlingErrors
-            self.packagesCrawled = packagesCrawled
-            self.packagesUpdated = packagesUpdated
+            self.errorsCrawling = errorsCrawling
+            self.reposCrawled = reposCrawled
+            self.reposUpdated = reposUpdated
             self.tagsCrawled = tagsCrawled
             self.tagsUpdated = tagsUpdated
 
@@ -214,19 +214,19 @@ extension Site.Admin:AdministrativePage
             $0[.dt] = "bytes transferred (content only)"
             $0[.dd] = "\(self.tour.stats.bytes.total)"
 
-            $0[.dt] = "crawling errors"
-            $0[.dd] = "\(self.crawlingErrors)"
+            $0[.dt] = "GitHub crawling errors"
+            $0[.dd] = "\(self.errorsCrawling)"
 
-            $0[.dt] = "packages crawled"
-            $0[.dd] = "\(self.packagesCrawled)"
+            $0[.dt] = "GitHub repos crawled"
+            $0[.dd] = "\(self.reposCrawled)"
 
-            $0[.dt] = "packages updated"
-            $0[.dd] = "\(self.packagesUpdated)"
+            $0[.dt] = "GitHub repos updated"
+            $0[.dd] = "\(self.reposUpdated)"
 
-            $0[.dt] = "tags crawled"
+            $0[.dt] = "GitHub tags crawled"
             $0[.dd] = "\(self.tagsCrawled)"
 
-            $0[.dt] = "tags updated"
+            $0[.dt] = "GitHub tags updated"
             $0[.dd] = "\(self.tagsUpdated)"
         }
 

Sources/UnidocServer/Operations/Server.Operation.AdminDashboard.swift

@@ -14,15 +14,15 @@ extension Server.Operation.AdminDashboard:RestrictedOperation
     func load(from server:Server.State) async throws -> ServerResponse?
     {
         let page:Site.Admin = .init(configuration: try await server.db.sessions.run(
-                    command: Mongo.ReplicaSetGetConfiguration.init(),
-                    against: .admin),
-                crawlingErrors: server._crawlingErrors.load(ordering: .relaxed),
-                packagesCrawled: server._packagesCrawled.load(ordering: .relaxed),
-                packagesUpdated: server._packagesUpdated.load(ordering: .relaxed),
-                tagsCrawled: server._tagsCrawled.load(ordering: .relaxed),
-                tagsUpdated: server._tagsUpdated.load(ordering: .relaxed),
-                tour: server.tour,
-                real: server.mode == .secured)
+                command: Mongo.ReplicaSetGetConfiguration.init(),
+                against: .admin),
+            errorsCrawling: server.github?.errors ?? 0,
+            reposCrawled: server.github?.reposCrawled ?? 0,
+            reposUpdated: server.github?.reposUpdated ?? 0,
+            tagsCrawled: server.github?.tagsCrawled ?? 0,
+            tagsUpdated: server.github?.tagsUpdated ?? 0,
+            tour: server.tour,
+            real: server.mode == .secured)
 
         return .resource(page.rendered())
     }

Sources/UnidocServer/Server/GitHubPlugin.Counters.swift

@@ -0,0 +1,32 @@
+import Atomics
+
+extension GitHubPlugin
+{
+    final
+    class Counters:Sendable
+    {
+        let reposCrawled:UnsafeAtomic<Int>
+        let reposUpdated:UnsafeAtomic<Int>
+        let tagsCrawled:UnsafeAtomic<Int>
+        let tagsUpdated:UnsafeAtomic<Int>
+        let errors:UnsafeAtomic<Int>
+
+        init()
+        {
+            self.reposCrawled = .create(0)
+            self.reposUpdated = .create(0)
+            self.tagsCrawled = .create(0)
+            self.tagsUpdated = .create(0)
+            self.errors = .create(0)
+        }
+
+        deinit
+        {
+            self.reposCrawled.destroy()
+            self.reposUpdated.destroy()
+            self.tagsCrawled.destroy()
+            self.tagsUpdated.destroy()
+            self.errors.destroy()
+        }
+    }
+}

Sources/UnidocServer/Server/GitHubPlugin.Crawler.swift

@@ -0,0 +1,124 @@
+import GitHubAPI
+import GitHubClient
+import MongoDB
+import SemanticVersions
+import UnidocDB
+import UnixTime
+
+extension GitHubPlugin
+{
+    struct Crawler
+    {
+        private
+        let count:Counters
+        private
+        let api:GitHubClient<GitHubOAuth.API>
+        private
+        let db:Server.DB
+
+        init(count:Counters, api:GitHubClient<GitHubOAuth.API>, db:Server.DB)
+        {
+            self.count = count
+            self.api = api
+            self.db = db
+        }
+    }
+}
+extension GitHubPlugin.Crawler
+{
+    func run() async throws
+    {
+        while true
+        {
+            async
+            let cooldown:Void = Task.sleep(for: .seconds(30))
+
+            let session:Mongo.Session = try await .init(from: self.db.sessions)
+            do
+            {
+                try await self.api.connect
+                {
+                    try await self.refresh(stalest: 10, from: $0, with: session)
+                }
+            }
+            catch let error as any GitHubRateLimitError
+            {
+                try await Task.sleep(for: error.until - .now())
+            }
+            catch let error
+            {
+                print("Crawling error: \(error)")
+                self.count.errors.wrappingIncrement(ordering: .relaxed)
+            }
+
+            try await cooldown
+        }
+    }
+
+    private
+    func refresh(stalest count:Int,
+        from github:GitHubClient<GitHubOAuth.API>.Connection,
+        with session:Mongo.Session) async throws
+    {
+        let stale:[PackageRecord] = try await self.db.package.packages.stalest(count,
+            with: session)
+
+        for package:PackageRecord in stale
+        {
+            guard case .github(let old) = package.repo
+            else
+            {
+                fatalError("unreachable: non-GitHub package was marked as stale!")
+            }
+
+            let repo:GitHub.Repo = try await github.get(
+                from: "/repos/\(old.owner.login)/\(old.name)")
+
+            switch try await self.db.package.packages.update(record: .init(id: package.id,
+                    cell: package.cell,
+                    repo: .github(repo)),
+                with: session)
+            {
+            case nil:
+                //  Might happen if package database is dropped while crawling.
+                continue
+
+            case _?:
+                //  To MongoDB, all repo updates look like modifications, since the package
+                //  record contains a timestamp.
+                self.count.reposCrawled.wrappingIncrement(ordering: .relaxed)
+            }
+            if  repo != old
+            {
+                self.count.reposUpdated.wrappingIncrement(ordering: .relaxed)
+            }
+
+            let tags:[GitHub.Tag] = try await github.get(
+                from: "/repos/\(repo.owner.login)/\(repo.name)/tags")
+
+            //  Import tags in chronological order.
+            for tag:GitHub.Tag in tags.reversed()
+            {
+                guard
+                let _:SemanticVersion = .init(refname: tag.name)
+                else
+                {
+                    //  We don’t care about non-semver tags.
+                    continue
+                }
+
+                switch try await self.db.package.editions.register(tag,
+                    package: package.cell,
+                    with: session)
+                {
+                case _?:
+                    self.count.tagsUpdated.wrappingIncrement(ordering: .relaxed)
+                    fallthrough
+
+                case nil:
+                    self.count.tagsCrawled.wrappingIncrement(ordering: .relaxed)
+                }
+            }
+        }
+    }
+}

Sources/UnidocServer/Server/GitHubPlugin.Partner.swift

@@ -0,0 +1,34 @@
+import Atomics
+import GitHubClient
+import GitHubAPI
+
+extension GitHubPlugin
+{
+    @dynamicMemberLookup
+    struct Partner
+    {
+        private
+        let count:Counters
+
+        let oauth:GitHubClient<GitHubOAuth>
+        // let app:GitHubClient<GitHubApp>
+        let api:GitHubClient<GitHubOAuth.API>
+
+        init(count:Counters,
+            oauth:GitHubClient<GitHubOAuth>,
+            api:GitHubClient<GitHubOAuth.API>)
+        {
+            self.count = count
+            self.oauth = oauth
+            self.api = api
+        }
+    }
+}
+extension GitHubPlugin.Partner
+{
+    subscript<Int>(
+        dynamicMember keyPath:KeyPath<GitHubPlugin.Counters, UnsafeAtomic<Int>>) -> Int
+    {
+        self.count[keyPath: keyPath].load(ordering: .relaxed)
+    }
+}

Sources/UnidocServer/Server/GitHubPlugin.swift

@@ -0,0 +1,53 @@
+import Atomics
+import GitHubClient
+import GitHubAPI
+import HTTPClient
+import NIOPosix
+import NIOSSL
+
+struct GitHubPlugin:Sendable
+{
+    let niossl:NIOSSLContext
+    let count:Counters
+
+    let oauth:GitHubOAuth
+    let app:GitHubApp
+
+    init(niossl:NIOSSLContext, oauth:GitHubOAuth, app:GitHubApp)
+    {
+        self.niossl = niossl
+        self.count = .init()
+
+        self.oauth = oauth
+        self.app = app
+    }
+}
+extension GitHubPlugin
+{
+    func partner(on threads:MultiThreadedEventLoopGroup) throws -> Partner
+    {
+        let root:HTTP2Client = .init(threads: threads,
+            niossl: niossl,
+            remote: "github.com")
+        let api:HTTP2Client = .init(threads: threads,
+            niossl: niossl,
+            remote: "api.github.com")
+
+        return .init(count: self.count,
+            oauth: .init(http2: root, app: self.oauth),
+            api: .init(http2: api, app: self.oauth.api))
+    }
+
+    func crawl(on threads:MultiThreadedEventLoopGroup, db:Server.DB) async throws
+    {
+        let api:HTTP2Client = .init(threads: threads,
+            niossl: niossl,
+            remote: "api.github.com")
+
+        let crawler:Crawler = .init(count: self.count,
+            api: .init(http2: api, app: self.oauth.api),
+            db: db)
+
+        try await crawler.run()
+    }
+}