Client-side translation (#314)

* add a case-folded resolution table to the linker

* translate apple.developer.com links on the client side

* we seem to have accidentally fixed another issue

Author: tayloraswift

Sources/LinkResolution/UCF.ResolutionPath.swift

@@ -18,6 +18,10 @@ extension UCF
     }
 }
 extension UCF.ResolutionPath
+{
+    func lowercased() -> Self { .init(string: self.string.lowercased()) }
+}
+extension UCF.ResolutionPath
 {
     @inlinable
     init(_ namespace:Symbol.Module)

Sources/LinkResolution/UCF.ResolutionTable.swift

@@ -8,16 +8,16 @@ extension UCF
     @frozen public
     struct ResolutionTable<Overload> where Overload:ResolvableOverload
     {
-        public
-        var modules:[Symbol.Module]
         @usableFromInline
         var entries:[UCF.ResolutionPath: InlineArray<Overload>]
+        @usableFromInline
+        var modules:[UCF.ResolutionPath: Symbol.Module]
 
         @inlinable public
         init()
         {
-            self.modules = []
             self.entries = [:]
+            self.modules = [:]
         }
     }
 }
@@ -27,6 +27,46 @@ extension UCF.ResolutionTable:ExpressibleByDictionaryLiteral
     init(dictionaryLiteral:(Never, Never)...) { self.init() }
 }
 extension UCF.ResolutionTable
+{
+    /// Lowercases all paths in the table, merging overloads with the same case-folded path.
+    /// Some modules may become **unresolvable** if they have names that differ only in case.
+    public
+    func caseFolded() -> Self
+    {
+        var copy:Self = self
+
+        copy.entries.removeAll(keepingCapacity: true)
+        copy.modules.removeAll(keepingCapacity: true)
+
+        for (path, overloads):(UCF.ResolutionPath, InlineArray<Overload>) in self.entries
+        {
+            {
+                for overload:Overload in overloads
+                {
+                    $0.append(overload)
+                }
+            } (&copy.entries[path.lowercased(), default: .some([])])
+        }
+        //  We need to sort this one because it can suffer from path collisions, and the
+        //  “winning” module would otherwise be non-deterministic.
+        for (path, module):(UCF.ResolutionPath, Symbol.Module) in self.modules.sorted(
+            by: { $0.value < $1.value })
+        {
+            copy.modules[path.lowercased()] = module
+        }
+
+        return copy
+    }
+}
+extension UCF.ResolutionTable
+{
+    @inlinable public mutating
+    func register(_ module:Symbol.Module)
+    {
+        self.modules[.init(module)] = module
+    }
+}
+extension UCF.ResolutionTable
 {
     @inlinable public
     subscript(namespace:Symbol.Module,
@@ -58,6 +98,14 @@ extension UCF.ResolutionTable
 }
 extension UCF.ResolutionTable
 {
+    public
+    func resolve(qualified path:UCF.Selector.Path,
+        matching suffix:UCF.Selector.Suffix?) -> UCF.Resolution<Overload>
+    {
+        var search:Search = .init(matching: suffix)
+        return self.resolve(qualified: path, with: &search)
+    }
+
     func resolve(_ selector:UCF.Selector,
         in scope:UCF.ResolutionScope) -> UCF.Resolution<Overload>
     {
@@ -82,7 +130,7 @@ extension UCF.ResolutionTable
                 }
             }
 
-            for namespace:Symbol.Module in self.modules.reversed() where
+            for namespace:Symbol.Module in self.modules.values where
                 namespace != scope.namespace
             {
                 let path:UCF.ResolutionPath = .join(["\(namespace)"] + selector.path.components)
@@ -98,17 +146,24 @@ extension UCF.ResolutionTable
             }
         }
 
+        return self.resolve(qualified: selector.path, with: &search)
+    }
+
+    private
+    func resolve(qualified path:UCF.Selector.Path,
+        with search:inout Search) -> UCF.Resolution<Overload>
+    {
         //  If we got this far, assume the first path component is a module name.
-        if  selector.path.components.count == 1
+        if  path.components.count == 1
         {
-            let last:Symbol.Module = .init(selector.path.components[0])
-            if  self.modules.contains(last)
+            let path:UCF.ResolutionPath = .init(string: path.components[0])
+            if  let module:Symbol.Module = self.modules[path]
             {
-                return .module(last)
+                return .module(module)
             }
         }
 
-        let path:UCF.ResolutionPath = .join(selector.path.components)
+        let path:UCF.ResolutionPath = .join(path.components)
         if  let list:InlineArray<Overload> = self.entries[path]
         {
             search.add(list)

Sources/UnidocLinker/Resolver/UCF.Selector (ext).swift renamed to Sources/LinkResolution/UCF.Selector (ext).swift

@@ -1,9 +1,10 @@
 import FNV1
-import SymbolGraphs
 import UCF
 
+//  TODO: this does not belong in this module
 extension UCF.Selector
 {
+    public
     static func translate(domain:Substring, path:Substring) -> Self?
     {
         //  Does this look like a link to Swift documentation? If so, we probably already have a
@@ -16,33 +17,25 @@ extension UCF.Selector
             if  let c:Int = path.firstIndex(of: "documentation")
             {
                 let d:Int = path.index(after: c)
-                let suffix:Suffix?
-
                 if  let last:Int = path.indices.last
                 {
-                    suffix =
                     {
                         if  let hyphen:String.Index = $0.firstIndex(of: "-"),
-                            let hash:FNV24 = .init($0[..<hyphen], radix: 10)
+                            let _:Int = .init($0[..<hyphen], radix: 10)
                         {
+                            //  These numeric prefixes are not FNV-1 hashes, they seem to be
+                            //  ordinal numbers generated opaquely by Apple. We can strip them
+                            //  in the hopes that the bare path is resolvable, but we can’t use
+                            //  them to help disambiguate anything.
                             $0 = $0[$0.index(after: hyphen)...]
-                            return .hash(hash)
-                        }
-                        else
-                        {
-                            return nil
                         }
                     } (&path[last])
                 }
-                else
-                {
-                    suffix = nil
-                }
 
                 return .init(
                     base: .qualified,
                     path: .init(components: path[d...].map { $0.lowercased() }),
-                    suffix: suffix)
+                    suffix: nil)
             }
             else
             {

Sources/SymbolGraphCompiler/SSGC.ModuleIndex.swift

@@ -10,6 +10,8 @@ extension SSGC
         public
         let id:Symbol.Module
 
+        public
+        let resolvableModules:[Symbol.Module]
         public
         let resolvableLinks:UCF.ResolutionTable<UCF.CausalOverload>
         public
@@ -28,6 +30,7 @@ extension SSGC
 
 
         init(id:Symbol.Module,
+            resolvableModules:[Symbol.Module],
             resolvableLinks:UCF.ResolutionTable<UCF.CausalOverload>,
             declarations:[(id:Symbol.Module, decls:[Decl])],
             extensions:[SSGC.Extension],
@@ -37,6 +40,7 @@ extension SSGC
             resources:[any SSGC.ResourceFile] = [])
         {
             self.id = id
+            self.resolvableModules = resolvableModules
             self.resolvableLinks = resolvableLinks
             self.declarations = declarations
             self.extensions = extensions

Sources/SymbolGraphCompiler/SSGC.TypeChecker.swift

@@ -19,14 +19,18 @@ extension SSGC
 
         private
         var resolvableLinks:UCF.ResolutionTable<UCF.CausalOverload>
+        private
+        var resolvableModules:[Symbol.Module]
 
         public
         init(ignoreExportedInterfaces:Bool = true, threshold:Symbol.ACL = .public)
         {
             self.ignoreExportedInterfaces = ignoreExportedInterfaces
             self.declarations = .init(threshold: threshold)
             self.extensions = .init()
-            self.resolvableLinks = .init()
+
+            self.resolvableLinks = [:]
+            self.resolvableModules = []
         }
     }
 }
@@ -66,7 +70,8 @@ extension SSGC.TypeChecker
     private mutating
     func add(symbols culture:SSGC.SymbolCulture, from id:Symbol.Module) throws
     {
-        self.resolvableLinks.modules.append(id)
+        self.resolvableModules.append(id)
+        self.resolvableLinks.register(id)
 
         /// We use this to look up protocols by name instead of symbol. This is needed in order
         /// to work around some bizarre lib/SymbolGraphGen bugs.
@@ -550,6 +555,7 @@ extension SSGC.TypeChecker
         }
 
         return .init(id: culture,
+            resolvableModules: self.resolvableModules,
             resolvableLinks: self.resolvableLinks,
             declarations: self.declarations.load(culture: culture),
             extensions: extensions,

Sources/SymbolGraphLinker/Markdown.SourceURL (ext).swift

@@ -0,0 +1,30 @@
+import MarkdownAST
+import UCF
+
+extension Markdown.SourceURL
+{
+    var translatableSelector:UCF.Selector?
+    {
+        //  Skip the two slashes.
+        guard
+        let start:String.Index = self.suffix.string.index(self.suffix.string.startIndex,
+            offsetBy: 2,
+            limitedBy: self.suffix.string.endIndex),
+        case "//" = self.suffix.string[..<start]
+        else
+        {
+            return nil
+        }
+
+        guard
+        let slash:String.Index = self.suffix.string[start...].firstIndex(of: "/")
+        else
+        {
+            return nil
+        }
+
+        return .translate(
+            domain: self.suffix.string[start ..< slash],
+            path: self.suffix.string[slash...])
+    }
+}

Sources/SymbolGraphLinker/Resolution/SSGC.OutlineResolver.swift

@@ -135,6 +135,44 @@ extension SSGC.OutlineResolver
         }
     }
 
+    mutating
+    func translate(url:Markdown.SourceURL) -> SymbolGraph.Outline?
+    {
+        guard
+        let translatable:UCF.Selector = url.translatableSelector
+        else
+        {
+            return nil
+        }
+
+        switch self.scopes.causalURLs.resolve(
+            qualified: translatable.path,
+            matching: translatable.suffix)
+        {
+        case .module(let module):
+            //  Unidoc linker doesn’t currently support `symbol` outlines that are not
+            //  declarations, so for now we just synthesize a normal vertex outline.
+            let text:SymbolGraph.OutlineText = .init(path: "\(module)", fragment: nil)
+            return .vertex(self.tables.intern(module) * .module, text: text)
+
+        case .overload(let overload):
+            return .symbol(self.tables.intern(overload.id))
+
+        case .ambiguous(let overloads, rejected: let rejected):
+            if  overloads.isEmpty
+            {
+                return nil
+            }
+
+            self.diagnostics[url.suffix.source] = UCF.ResolutionError<SSGC.Symbolicator>.init(
+                overloads: overloads,
+                rejected: rejected,
+                selector: translatable)
+
+            return nil
+        }
+    }
+
     mutating
     func outline(_ codelink:UCF.Selector,
         at source:SourceReference<Markdown.Source>) -> SymbolGraph.Outline?

Sources/SymbolGraphLinker/Resolution/SSGC.OutlineResolverEnvironment.swift

@@ -10,6 +10,7 @@ extension SSGC
         let origin:Int32?
 
         let causalLinks:UCF.ResolutionTable<UCF.CausalOverload>
+        let causalURLs:UCF.ResolutionTable<UCF.CausalOverload>
         let resources:[String: SSGC.Resource]
         let codelink:UCF.ResolutionScope
         /// The scope to use when resolving `doc:` links. The namespace may be different from
@@ -21,12 +22,14 @@ extension SSGC
         private
         init(origin:Int32?,
             causalLinks:UCF.ResolutionTable<UCF.CausalOverload>,
+            causalURLs:UCF.ResolutionTable<UCF.CausalOverload>,
             resources:[String: SSGC.Resource],
             codelink:UCF.ResolutionScope,
             doclink:UCF.ArticleScope)
         {
             self.origin = origin
             self.causalLinks = causalLinks
+            self.causalURLs = causalURLs
             self.resources = resources
             self.codelink = codelink
             self.doclink = doclink
@@ -57,6 +60,7 @@ extension SSGC.OutlineResolverEnvironment
     {
         self.init(origin: origin,
             causalLinks: context.causalLinks,
+            causalURLs: context.causalURLs,
             resources: context.resources,
             codelink: .init(namespace: namespace ?? context.id,
                 imports: [],

Sources/SymbolGraphLinker/Resolution/SSGC.Outliner.swift

@@ -66,8 +66,7 @@ extension SSGC.Outliner
                 return self.outline(doc: url.suffix, as: url.provenance)
 
             case let scheme?:
-                return self.cache.add(outline: .url("\(scheme):\(url.suffix)",
-                    location: url.suffix.source.start))
+                return self.outline(url: url, scheme: scheme)
             }
 
         case .file(let link):
@@ -103,6 +102,24 @@ extension SSGC.Outliner
         return nil
     }
 
+    private mutating
+    func outline(url:Markdown.SourceURL, scheme:String) -> Int?
+    {
+        let translated:SymbolGraph.Outline?
+
+        switch scheme
+        {
+        case "http":    translated = self.resolver.translate(url: url)
+        case "https":   translated = self.resolver.translate(url: url)
+        default:        translated = nil
+        }
+
+        //  TODO: log translations?
+
+        return self.cache.add(outline: translated ?? .url("\(scheme):\(url.suffix)",
+            location: url.suffix.source.start))
+    }
+
     private mutating
     func outline(ucf link:Markdown.SourceString) -> Int?
     {