count number of SPIs, and add some link resolver logic for translating external URLs

Author: tayloraswift

Sources/CodelinkResolution/CodelinkResolver.Overloads.swift

@@ -34,6 +34,29 @@ extension CodelinkResolver.Overloads
         }
     }
 
+    @inlinable mutating
+    func overload(with overloads:Self)
+    {
+        switch (consume self, overloads)
+        {
+        case    (.one(let one), .one(let other)):
+            self = .some([one, other])
+
+        case    (.one(let one), .some([])),
+                (.some([]), .one(let one)):
+            self = .one(one)
+
+        case    (.some(var some), .one(let one)),
+                (.one(let one), .some(var some)):
+            some.append(one)
+            self = .some(some)
+
+        case    (.some(var some), .some(let others)):
+            some += others
+            self = .some(some)
+        }
+    }
+
     @inlinable public mutating
     func overload(with overload:CodelinkResolver<Scalar>.Overload)
     {

Sources/CodelinkResolution/CodelinkResolver.Table.swift

@@ -22,6 +22,23 @@ extension CodelinkResolver.Table:Sendable where Scalar:Sendable
 {
 }
 extension CodelinkResolver.Table
+{
+    public
+    func caseless() -> Self
+    {
+        var copy:Self = self
+            copy.entries.removeAll(keepingCapacity: true)
+
+        for (path, overloads) in self.entries
+        {
+            copy.entries[.init(string: path.string.lowercased()), default: .some([])].overload(
+                with: overloads)
+        }
+
+        return copy
+    }
+}
+extension CodelinkResolver.Table
 {
     @inlinable public
     subscript(namespace:Symbol.Module) -> CodelinkResolver<Scalar>.Overloads

Sources/SwiftinitPages/Contexts/Swiftinit.VertexPageContext (ext).swift

@@ -90,10 +90,30 @@ extension Swiftinit.VertexPageContext
         switch repo.origin
         {
         case .github(let origin):
-            return """
-            https://raw.githubusercontent.com\
-            /\(origin.owner)/\(origin.name)/\(refname)/\(file.symbol)
-            """
+            //  Files that lack a valid extension will not carry the correct `Content-Type`
+            //  header, and won’t display correctly in the browser. There is no simple way to
+            //  override this behavior, so files will just need to have the correct extension.
+            guard
+            let type:Substring = file.symbol.type
+            else
+            {
+                return nil
+            }
+
+            let prefix:String
+
+            switch type
+            {
+            case "gif":     prefix = "https://raw.githubusercontent.com"
+            case "jpg":     prefix = "https://raw.githubusercontent.com"
+            case "jpeg":    prefix = "https://raw.githubusercontent.com"
+            case "png":     prefix = "https://raw.githubusercontent.com"
+            case "svg":     prefix = "https://raw.githubusercontent.com"
+            case "webp":    prefix = "https://media.githubusercontent.com/media"
+            default:        return nil
+            }
+
+            return "\(prefix)/\(origin.owner)/\(origin.name)/\(refname)/\(file.symbol)"
         }
     }
 }

Sources/UnidocLinker/Modules/SymbolGraph.ModuleContext.swift

@@ -19,18 +19,32 @@ extension SymbolGraph
 
         private(set)
         var codelinks:CodelinkResolver<Unidoc.Scalar>.Table
+        /// This is needed to support URL translation from other package indexes.
+        private(set)
+        var caseless:CodelinkResolver<Unidoc.Scalar>.Table
         private(set)
         var imports:[Symbol.Module]
 
+        private
         init()
         {
             self.conformances = []
             self.codelinks = .init()
+            self.caseless = .init()
             self.imports = []
         }
     }
 }
 extension SymbolGraph.ModuleContext
+{
+    init(with build:(inout Self) throws -> ()) rethrows
+    {
+        self.init()
+        try build(&self)
+        self.caseless = self.codelinks.caseless()
+    }
+}
+extension SymbolGraph.ModuleContext
 {
     private mutating
     func remember(conforms t:Unidoc.Scalar, to p:Unidoc.Scalar)

Sources/UnidocLinker/Resolver/Codelink (ext).swift

@@ -35,4 +35,54 @@ extension Codelink
             self.init(v3: unidocV3)
         }
     }
+
+    /// The `domain` must share indices with `link`.
+    init?(translating link:__shared String, to domain:__shared Substring)
+    {
+        guard domain.endIndex < link.endIndex
+        else
+        {
+            return nil
+        }
+
+        let i:String.Index = link.index(after: domain.endIndex)
+        //  Does this look like a link to Swift documentation? If so, we probably already have a
+        //  local copy of it.
+        switch domain
+        {
+        case "developer.apple.com":
+            //  https://developer.apple.com/documentation/swift/uint16
+            let path:[Substring] = link[i...].split(separator: "/")
+            if  let c:Int = path.firstIndex(of: "documentation")
+            {
+                let d:Int = path.index(after: c)
+                self.init(
+                    base: .qualified,
+                    path: .init(components: path[d...].map { $0.lowercased() }))
+            }
+            else
+            {
+                return nil
+            }
+
+        case "swiftpackageindex.com":
+            //  https://swiftpackageindex.com/apple/swift-syntax/509.1.1/documentation/
+            let path:[Substring] = link[i...].split(separator: "/")
+            if  let c:Int = path.firstIndex(of: "documentation")
+            {
+                let d:Int = path.index(after: c)
+                self.init(
+                    base: .qualified,
+                    path: .init(components: path[d...].map { $0.lowercased() }))
+            }
+            else
+            {
+                return nil
+            }
+
+        default:
+            //  We don't know how to translate other URLs yet.
+            return nil
+        }
+    }
 }

Sources/UnidocLinker/Resolver/Unidoc.Resolver.swift

@@ -14,12 +14,18 @@ extension Unidoc
     {
         private
         let codelinks:CodelinkResolver<Scalar>
+        private
+        let caseless:CodelinkResolver<Scalar>
         private(set)
         var context:Linker
 
-        init(codelinks:CodelinkResolver<Scalar>, context:consuming Linker)
+        init(
+            codelinks:CodelinkResolver<Scalar>,
+            caseless:CodelinkResolver<Scalar>,
+            context:consuming Linker)
         {
             self.codelinks = codelinks
+            self.caseless = caseless
             self.context = context
         }
     }
@@ -110,33 +116,74 @@ extension Unidoc.Resolver
             }
 
         case    .unresolved(let unresolved):
+            let resolution:CodelinkResolver<Unidoc.Scalar>.Overload.Target?
+            let codelink:Codelink
+
+            resolution:
             if  case .web = unresolved.type
             {
                 let domain:Substring = unresolved.link.prefix { $0 != "/" }
-                //  We will follow links to Apple, GitHub, and reputable open-source indexes.
-                let safe:Bool = switch domain
+
+                if  let link:Codelink = .init(translating: unresolved.link, to: domain)
                 {
-                case "developer.apple.com":     true
-                case "www.freebsd.org":         true
-                case "github.com":              true
-                case "tools.ietf.org":          true
-                case "man7.org":                true
-                case "developer.mozilla.org":   true
-                case "docs.scala-lang.org":     true
-                case "swiftinit.org":           true
-                case "forums.swift.org":        true
-                case "swift.org":               true
-                case "en.wikipedia.org":        true
-                default:                        false
+                    codelink = link
                 }
+                else
+                {
+                    let root:Substring
+                    if  let j:String.Index = domain.lastIndex(of: "."),
+                        let i:String.Index = domain[..<j].lastIndex(of: ".")
+                    {
+                        root = domain[domain.index(after: i)...]
+                    }
+                    else
+                    {
+                        root = domain
+                    }
+                    //  We will follow links to GitHub and reputable open-source indexes.
+                    let safe:Bool = switch root
+                    {
+                    case "freebsd.org":     true
+                    case "github.com":      true
+                    case "ietf.org":        true
+                    case "man7.org":        true
+                    case "mozilla.org":     true
+                    case "scala-lang.org":  true
+                    case "swiftinit.org":   true
+                    case "swift.org":       true
+                    case "wikipedia.org":   true
+                    default:                false
+                    }
 
-                return .link(https: unresolved.link, safe: safe)
-            }
+                    return .link(https: unresolved.link, safe: safe)
+                }
+
+                //  Translation always lowercases the URL, so we need to use the collated table.
+                switch self.caseless.resolve(codelink)
+                {
+                case .some(let overloads):
+                    guard
+                    let overload:CodelinkResolver<Unidoc.Scalar>.Overload = overloads.first
+                    else
+                    {
+                        //  Not an error, this was only speculative.
+                        return .link(https: unresolved.link, safe: false)
+                    }
+
+                    resolution = overload.target
+
+                case .one(let overload):
+                    resolution = overload.target
+                }
 
-            guard
-            let (codelink, resolution):
-                (Codelink, CodelinkResolver<Unidoc.Scalar>.Overload.Target?) =
+                print("DEBUG: successful translation of '\(unresolved.link)'")
+            }
+            else if
+                let resolved:(Codelink, CodelinkResolver<Unidoc.Scalar>.Overload.Target?) =
                     self.resolve(unresolved)
+            {
+                (codelink, resolution) = resolved
+            }
             else
             {
                 return .text(unresolved.link)

Sources/UnidocLinker/Sema/WritableKeyPath (ext).swift

@@ -4,10 +4,13 @@ import UnidocRecords
 
 extension WritableKeyPath<Unidoc.Stats.Coverage, Int>
 {
+    //  TODO: This is a temporary solution. This statistic should not depend on the snapshot.
+    //  Ideally, we should reformulate the scoring system to assign a “expected” amount of
+    //  documentation for each declaration, which does not depend on other declarations.
     static
     func classify(_ decl:SymbolGraph.Decl,
-        from snapshot:Unidoc.Linker.Graph,
-        at local:Int32) -> WritableKeyPath<Unidoc.Stats.Coverage, Int>
+        _from snapshot:Unidoc.Linker.Graph,
+        _at local:Int32) -> WritableKeyPath<Unidoc.Stats.Coverage, Int>
     {
         if  case _? = decl.article
         {

Sources/UnidocLinker/Unidoc.Linker.Mesh.swift

@@ -1,3 +1,4 @@
+import BSON
 import JSON
 import SymbolGraphs
 import Symbols
@@ -86,15 +87,18 @@ extension Unidoc.Linker.Mesh
             {
                 if  let decl:SymbolGraph.Decl = linker.current.decls.nodes[d].decl
                 {
+                    let interface:BSON.Key = decl.signature.spis == nil ? "" : "__unknown__"
                     let coverage:WritableKeyPath<Unidoc.Stats.Coverage, Int> = .classify(decl,
-                        from: linker.current,
-                        at: d)
+                        _from: linker.current,
+                        _at: d)
 
                     let decl:WritableKeyPath<Unidoc.Stats.Decl, Int> = .classify(decl)
 
+                    cultures[c].census.interfaces[interface, default: 0] += 1
                     cultures[c].census.unweighted.coverage[keyPath: coverage] += 1
                     cultures[c].census.unweighted.decls[keyPath: decl] += 1
 
+                    snapshot.census.interfaces[interface, default: 0] += 1
                     snapshot.census.unweighted.coverage[keyPath: coverage] += 1
                     snapshot.census.unweighted.decls[keyPath: decl] += 1
                 }

Sources/UnidocLinker/Unidoc.Linker.swift

@@ -247,30 +247,31 @@ extension Unidoc.Linker
         {
             for (dependencies, cultures):([Symbol.Package: Set<String>], [Int]) in groups
             {
-                var shared:SymbolGraph.ModuleContext = .init()
-
-                if  let swift:Graph = self[dynamic: .swift]
-                {
-                    shared.add(snapshot: swift, context: self, filter: nil)
-                }
-                for (package, products):(Symbol.Package, Set<String>) in
-                    dependencies.sorted(by: { $0.key < $1.key })
+                let shared:SymbolGraph.ModuleContext = .init
                 {
-                    guard
-                    let snapshot:Graph = self[dynamic: package]
-                    else
+                    if  let swift:Graph = self[dynamic: .swift]
                     {
-                        continue
+                        $0.add(snapshot: swift, context: self, filter: nil)
                     }
-
-                    var filter:Set<Int> = []
-                    for product:SymbolGraph.Product in snapshot.metadata.products
-                        where products.contains(product.name)
+                    for (package, products):(Symbol.Package, Set<String>) in
+                        dependencies.sorted(by: { $0.key < $1.key })
                     {
-                        filter.formUnion(product.cultures)
-                    }
+                        guard
+                        let snapshot:Graph = self[dynamic: package]
+                        else
+                        {
+                            continue
+                        }
 
-                    shared.add(snapshot: snapshot, context: self, filter: filter)
+                        var filter:Set<Int> = []
+                        for product:SymbolGraph.Product in snapshot.metadata.products
+                            where products.contains(product.name)
+                        {
+                            filter.formUnion(product.cultures)
+                        }
+
+                        $0.add(snapshot: snapshot, context: self, filter: filter)
+                    }
                 }
 
                 for c:Int in cultures
@@ -430,6 +431,10 @@ extension Unidoc.Linker
                 namespace: namespace,
                 imports: module.imports,
                 path: scope)),
+            caseless: .init(table: module.caseless, scope: .init(
+                namespace: namespace,
+                imports: module.imports,
+                path: scope)),
             context: consume self)
 
         do