overhaul package repo encoding, enable indexing them by creation date

Author: tayloraswift

Sources/GitHubAPI/GitHub.Repo.License.swift

@@ -32,7 +32,6 @@ extension GitHub.Repo.License:JSONObjectDecodable
     public
     init(json:JSON.ObjectDecoder<CodingKey>) throws
     {
-        self.init(id: try json[.id].decode(),
-            name: try json[.name].decode())
+        self.init(id: try json[.id].decode(), name: try json[.name].decode())
     }
 }

Sources/SwiftinitPages/Surfaces/Editions/Swiftinit.TagsPage.swift

@@ -92,28 +92,28 @@ extension Swiftinit.TagsPage:Swiftinit.ApplicationPage
 
         main[.section, { $0.class = "details" }]
         {
-            if  let repo:Unidoc.PackageMetadata.Repo = self.package.repo
+            if  let repo:Unidoc.PackageRepo = self.package.repo
             {
                 $0[.h2] = "Package Repository"
 
                 $0[.dl]
                 {
-                    switch repo
+                    switch repo.origin
                     {
-                    case .github(let repo):
+                    case .github(let origin):
                         let now:UnixInstant = .now()
 
                         $0[.dt] = "Provider"
                         $0[.dd]
                         {
                             $0[.a]
                             {
-                                $0.href = "https://github.com/\(repo.owner.login)/\(repo.name)"
+                                $0.href = origin.https
                                 $0.target = "_blank"
                             } = "GitHub"
                         }
 
-                        if  let license:GitHub.Repo.License = repo.license
+                        if  let license:Unidoc.PackageLicense = repo.license
                         {
                             $0[.dt] = "License"
                             $0[.dd] = license.name
@@ -125,10 +125,10 @@ extension Swiftinit.TagsPage:Swiftinit.ApplicationPage
                         }
 
                         $0[.dt] = "Owner"
-                        $0[.dd] = repo.owner
+                        $0[.dd] = origin.owner
 
                         $0[.dt] = "Watchers"
-                        $0[.dd] = "\(repo.watchers)"
+                        $0[.dd] = "\(origin.watchers)"
 
                         $0[.dt] = "Forks"
                         $0[.dd] = "\(repo.forks)"
@@ -137,23 +137,21 @@ extension Swiftinit.TagsPage:Swiftinit.ApplicationPage
                         $0[.dd] = "\(repo.stars)"
 
                         $0[.dt] = "Archived?"
-                        $0[.dd] = repo.archived ? "yes" : "no"
+                        $0[.dd] = origin.archived ? "yes" : "no"
 
-                        if  let created:Timestamp.Components = .init(iso8601: repo.created)
-                        {
-                            let created:Timestamp.Date = created.date
+                        let created:UnixInstant = .millisecond(repo.created.value)
+                        let updated:UnixInstant = .millisecond(repo.updated.value)
 
+                        if  let created:Timestamp.Date = created.timestamp?.date
+                        {
                             $0[.dt] = "Created"
                             $0[.dd] = "\(created.month(.en)) \(created.day), \(created.year)"
                         }
-                        if  let updated:Timestamp.Components = .init(iso8601: repo.updated),
-                            let updated:UnixInstant = .init(utc: updated)
-                        {
-                            let age:Age = .init(now - updated)
 
-                            $0[.dt] = "Last Pushed"
-                            $0[.dd] = "\(age)"
-                        }
+                        let age:Age = .init(now - updated)
+
+                        $0[.dt] = "Last Pushed"
+                        $0[.dd] = "\(age)"
                     }
                 }
             }

Sources/SwiftinitPages/Surfaces/Editions/Unidoc.PackageStatus (ext).swift

@@ -7,7 +7,7 @@ extension Unidoc.PackageStatus
     init?(from output:borrowing Unidoc.PackageQuery.Output)
     {
         guard
-        let repo:Unidoc.PackageMetadata.Repo = output.package.repo,
+        let repo:Unidoc.PackageRepo = output.package.repo,
         let release:Unidoc.PackageQuery.Tag = output.releases.first,
         let release:Edition = .init(from: release)
         else
@@ -17,7 +17,7 @@ extension Unidoc.PackageStatus
 
         self.init(
             coordinate: output.package.id,
-            repo: "https://\(repo.origin)",
+            repo: repo.origin.https,
             release: release,
             prerelease: output.prereleases.first.flatMap(Edition.init(from:)))
     }

Sources/SwiftinitServer/Endpoints/Interactive/Swiftinit.PackageIndexTagEndpoint.swift

@@ -43,7 +43,7 @@ extension Swiftinit.PackageIndexTagEndpoint:RestrictedEndpoint
         }
 
         guard
-        case .github(let repo) = package.repo
+        case .github(let origin) = package.repo?.origin
         else
         {
             return .notFound("Not a GitHub repository")
@@ -52,8 +52,8 @@ extension Swiftinit.PackageIndexTagEndpoint:RestrictedEndpoint
         let response:GitHubPlugin.TagResponse = try await github.api.connect
         {
             try await $0.inspect(tag: self.tag,
-                owner: repo.owner.login,
-                repo: repo.name,
+                owner: origin.owner,
+                repo: origin.name,
                 pat: github.plugin.pat)
         }
 

Sources/SwiftinitServer/Plugins/GitHubPlugin.RepoMonitor.swift

@@ -35,18 +35,25 @@ extension GitHubPlugin.RepoMonitor:GitHubCrawler
 
         for var package:Unidoc.PackageMetadata in stale
         {
-            guard case .github(let old) = package.repo
+            guard
+            let old:Unidoc.PackageRepo = package.repo
             else
             {
-                fatalError("unreachable: non-GitHub package was marked as stale!")
+                fatalError("unreachable: repoless package was marked as stale!")
+            }
+
+            let origin:Unidoc.PackageRepo.GitHubOrigin
+            switch old.origin
+            {
+            case .github(let github):   origin = github
             }
 
             let response:GitHubPlugin.RepoMonitorResponse = try await connection.crawl(
-                owner: old.owner.login,
-                repo: old.name,
+                owner: origin.owner,
+                repo: origin.name,
                 pat: self.pat)
 
-            package.repo = .github(response.repo)
+            package.repo = try .github(response.repo)
             package.crawled = .now()
 
             switch try await server.db.packages.update(metadata: package, with: session)
@@ -60,7 +67,7 @@ extension GitHubPlugin.RepoMonitor:GitHubCrawler
                 //  record contains a timestamp.
                 server.atomics.reposCrawled.wrappingIncrement(ordering: .relaxed)
             }
-            if  response.repo != old
+            if  package.repo != old
             {
                 server.atomics.reposUpdated.wrappingIncrement(ordering: .relaxed)
             }
@@ -112,8 +119,7 @@ extension GitHubPlugin.RepoMonitor:GitHubCrawler
             {
                 let activity:UnidocDatabase.RepoFeed.Activity = .init(discovered: .now(),
                     package: package.symbol,
-                    refname: interesting,
-                    origin: .github(response.repo.owner.login, response.repo.name))
+                    refname: interesting)
 
                 try await server.db.repoFeed.push(activity, with: session)
             }

Sources/SwiftinitServer/Plugins/GitHubPlugin.RepoTelescope.swift

@@ -68,7 +68,7 @@ extension GitHubPlugin.RepoTelescope:GitHubCrawler
         {
             let symbol:Symbol.Package = "\(repo.owner.login).\(repo.name)"
             switch try await server.db.unidoc.index(package: symbol,
-                repo: .github(repo),
+                repo: try .github(repo),
                 mode: .automatic,
                 with: session)
             {

Sources/UnidocDB/Activity/UnidocDatabase.RepoFeed.Activity.swift

@@ -16,19 +16,15 @@ extension UnidocDatabase.RepoFeed
         let package:Symbol.Package
         public
         let refname:String
-        public
-        let origin:Unidoc.PackageMetadata.Repo.Origin
 
         @inlinable public
         init(discovered id:BSON.Millisecond,
             package:Symbol.Package,
-            refname:String,
-            origin:Unidoc.PackageMetadata.Repo.Origin)
+            refname:String)
         {
             self.id = id
             self.package = package
             self.refname = refname
-            self.origin = origin
         }
     }
 }
@@ -41,6 +37,8 @@ extension UnidocDatabase.RepoFeed.Activity:MongoMasterCodingModel
 
         case package = "P"
         case refname = "G"
+
+        @available(*, unavailable)
         case origin = "O"
     }
 }
@@ -53,7 +51,6 @@ extension UnidocDatabase.RepoFeed.Activity:BSONDocumentEncodable
 
         bson[.package] = self.package
         bson[.refname] = self.refname
-        bson[.origin] = self.origin
     }
 }
 extension UnidocDatabase.RepoFeed.Activity:BSONDocumentDecodable
@@ -63,7 +60,6 @@ extension UnidocDatabase.RepoFeed.Activity:BSONDocumentDecodable
     {
         self.init(discovered: try bson[.id].decode(),
             package: try bson[.package].decode(),
-            refname: try bson[.refname].decode(),
-            origin: try bson[.origin].decode())
+            refname: try bson[.refname].decode())
     }
 }

Sources/UnidocDB/Packages/Unidoc.PackageLicense.swift

@@ -0,0 +1,47 @@
+import BSON
+import MongoQL
+
+extension Unidoc
+{
+    @frozen public
+    struct PackageLicense:Equatable, Sendable
+    {
+        public
+        let spdx:String
+        public
+        let name:String
+
+        @inlinable public
+        init(spdx:String, name:String)
+        {
+            self.spdx = spdx
+            self.name = name
+        }
+    }
+}
+extension Unidoc.PackageLicense:MongoMasterCodingModel
+{
+    @frozen public
+    enum CodingKey:String, Sendable
+    {
+        case spdx = "I"
+        case name = "N"
+    }
+}
+extension Unidoc.PackageLicense:BSONDocumentEncodable
+{
+    public
+    func encode(to bson:inout BSON.DocumentEncoder<CodingKey>)
+    {
+        bson[.spdx] = self.spdx
+        bson[.name] = self.name
+    }
+}
+extension Unidoc.PackageLicense:BSONDocumentDecodable
+{
+    @inlinable public
+    init(bson:BSON.DocumentDecoder<CodingKey, some RandomAccessCollection<UInt8>>) throws
+    {
+        self.init(spdx: try bson[.spdx].decode(), name: try bson[.name].decode())
+    }
+}

Sources/UnidocDB/Packages/Unidoc.PackageMetadata.swift

@@ -40,7 +40,7 @@ extension Unidoc
         ///
         /// Currently only GitHub repos are supported.
         public
-        var repo:Repo?
+        var repo:PackageRepo?
 
         /// When this package *record* was last crawled. This is different from the time when
         /// the package itself was last updated.
@@ -53,7 +53,7 @@ extension Unidoc
             hidden:Bool = false,
             realm:Unidoc.Realm? = nil,
             realmAligning:Bool = false,
-            repo:Repo? = nil,
+            repo:PackageRepo? = nil,
             crawled:BSON.Millisecond = 0)
         {
             self.id = id
@@ -76,7 +76,10 @@ extension Unidoc.PackageMetadata:MongoMasterCodingModel
         case hidden = "H"
         case realm = "r"
         case realmAligning = "A"
-        case repo = "R"
+
+        case repo = "G"
+        case repoLegacy = "R"
+
         case crawled = "T"
     }
 }
@@ -106,5 +109,15 @@ extension Unidoc.PackageMetadata:BSONDocumentDecodable
             realmAligning: try bson[.realmAligning]?.decode() ?? false,
             repo: try bson[.repo]?.decode(),
             crawled: try bson[.crawled]?.decode() ?? 0)
+
+        if  case nil = self.repo,
+            let legacy:Repo = try bson[.repoLegacy]?.decode()
+        {
+            switch legacy
+            {
+            case .github(let repo):
+                self.repo = try .github(repo)
+            }
+        }
     }
 }

Sources/UnidocDB/Packages/Unidoc.PackageRepo.AnyOrigin.swift

@@ -0,0 +1,30 @@
+import Symbols
+
+extension Unidoc.PackageRepo
+{
+    @frozen public
+    enum AnyOrigin:Equatable, Sendable
+    {
+        case github(GitHubOrigin)
+    }
+}
+extension Unidoc.PackageRepo.AnyOrigin
+{
+    @inlinable public
+    var https:String
+    {
+        switch self
+        {
+        case .github(let self): self.https
+        }
+    }
+
+    @inlinable public
+    func blob(refname:String, file:Symbol.File) -> String
+    {
+        switch self
+        {
+        case .github(let self): "\(self.https)/blob/\(refname)/\(file)"
+        }
+    }
+}