return to a symbolgraph-extract based model, and pass -fmodule-map (#408)

* return to a symbolgraph-extract based model, and pass -fmodule-map along with includes

* we shouldn’t actually need include paths if we have modulemaps

* i think we need --no-parallel here

* which swift?

* the runner uses a different runtime layout

* ?

* no double dash???

* centralize build planning logic in ModuleGraph

* fix failure to dump all stdlib modules

* create missing directories

* port SymbolGraphPartTests to Swift Testing, and update expectations to reflect bugs that have been fixed in the Swift 6.0 compiler

* add a test case for reexported symbols

Author: tayloraswift

.github/workflows/test.yml renamed to .github/workflows/Test.yml

@@ -57,7 +57,7 @@ jobs:
                 run: |
                     swift --version
                     swift build -c release \
-                        --build-tests \
+                        --product ssgc \
                         --explicit-target-dependency-import-check=error \
                         -Xcxx -I$SWIFT_INSTALLATION/lib/swift \
                         -Xcxx -I$SWIFT_INSTALLATION/lib/swift/Block
@@ -71,7 +71,7 @@ jobs:
             -   name: Validate packages
                 run: |
                     swift test -c release \
-                        --skip-build \
+                        --no-parallel \
                         --filter SymbolGraphValidationTests \
                         -Xcxx -I$SWIFT_INSTALLATION/lib/swift \
                         -Xcxx -I$SWIFT_INSTALLATION/lib/swift/Block
@@ -102,4 +102,5 @@ jobs:
             -   name: Validate packages
                 run: |
                     swift test -c release \
+                        --no-parallel \
                         --filter SymbolGraphValidationTests

Package.resolved

@@ -111,8 +111,6 @@ let package:Package = .init(
             from: "1.33.0")),
         .package(url: "https://github.com/apple/swift-markdown", .upToNextMinor(
             from: "0.4.0")),
-        .package(url: "https://github.com/apple/swift-system", .upToNextMinor(
-            from: "1.4.0")),
         .package(url: "https://github.com/apple/swift-syntax",
             from: "600.0.1"),
     ],
@@ -651,10 +649,9 @@ let package:Package = .init(
                 .target(name: "SymbolGraphLinker"),
             ]),
 
-        .executableTarget(name: "SymbolGraphPartTests",
+        .testTarget(name: "SymbolGraphPartTests",
             dependencies: [
                 .target(name: "SymbolGraphParts"),
-                .target(name: "Testing_"),
                 .product(name: "SystemIO", package: "swift-io"),
             ]),
 

Package.swift

@@ -2,7 +2,7 @@
 
 <strong><em><code>unidoc</code></em></strong>
 
-[![ci build status](https://github.com/tayloraswift/swift-unidoc/actions/workflows/test.yml/badge.svg)](https://github.com/tayloraswift/swift-unidoc/actions/workflows/test.yml)
+[![ci build status](https://github.com/tayloraswift/swift-unidoc/actions/workflows/Test.yml/badge.svg)](https://github.com/tayloraswift/swift-unidoc/actions/workflows/Test.yml)
 [![ci build status](https://github.com/tayloraswift/swift-unidoc/actions/workflows/docs.yml/badge.svg)](https://github.com/tayloraswift/swift-unidoc/actions/workflows/docs.yml)
 [![ci build status](https://github.com/tayloraswift/swift-unidoc/actions/workflows/Deploy.yml/badge.svg)](https://github.com/tayloraswift/swift-unidoc/actions/workflows/Deploy.yml)
 

README.md

@@ -3,42 +3,43 @@
 set -e
 
 swift --version
-swift build \
-    --package-path TestModules \
-    -Xswiftc -emit-symbol-graph \
-    -Xswiftc -emit-symbol-graph-dir -Xswiftc SymbolGraphs \
-    -Xswiftc -symbol-graph-minimum-access-level -Xswiftc internal \
-    -Xswiftc -emit-extension-block-symbols \
-    -Xswiftc -include-spi-symbols \
-    -Xswiftc -skip-inherited-docs \
-    -Xswiftc -pretty-print \
-    -Xswiftc -swift-version -Xswiftc 6
-
+swift build --package-path TestModules
 swift package --package-path TestModules dump-package > TestModules/Package.swift.json
 
-for TARGET in Swift _Concurrency
+for output in SymbolGraphs Determinism
 do
-    swift symbolgraph-extract \
-        -target x86_64-unknown-linux-gnu \
-        -minimum-access-level internal \
-        -output-dir TestModules/SymbolGraphs \
-        -skip-inherited-docs \
-        -emit-extension-block-symbols \
-        -include-spi-symbols \
-        -pretty-print \
-        -module-name $TARGET
+    mkdir -p TestModules/$output
+    for module in Swift _Concurrency _Differentiation
+    do
+        swift symbolgraph-extract \
+            -target x86_64-unknown-linux-gnu \
+            -minimum-access-level internal \
+            -output-dir TestModules/$output \
+            -skip-inherited-docs \
+            -emit-extension-block-symbols \
+            -include-spi-symbols \
+            -pretty-print \
+            -module-name $module
+    done
 done
 
-swift symbolgraph-extract \
-    -target x86_64-unknown-linux-gnu \
-    -minimum-access-level internal \
-    -output-dir TestModules/Determinism \
-    -skip-inherited-docs \
-    -emit-extension-block-symbols \
-    -include-spi-symbols \
-    -pretty-print \
-    -module-name Swift
-
+for include in TestModules/.build/debug/Modules
+do
+    for file in $include/*.swiftmodule
+    do
+        swift symbolgraph-extract \
+            -target x86_64-unknown-linux-gnu \
+            -minimum-access-level internal \
+            -output-dir TestModules/SymbolGraphs \
+            -skip-inherited-docs \
+            -emit-extension-block-symbols \
+            -include-spi-symbols \
+            -pretty-print \
+            -module-name $(basename $file .swiftmodule) \
+            -experimental-allowed-reexported-modules=ACL \
+            -I $include
+    done
+done
 
 .build/release/ssgc -u $SWIFT_INSTALLATION \
     -n swift \

Scripts/Linux/GenerateTestSymbolGraphs

@@ -7,8 +7,7 @@ swift --version
 swift build -c release \
     --explicit-target-dependency-import-check=error \
     -Xcxx -I$SWIFT_INSTALLATION/lib/swift \
-    -Xcxx -I$SWIFT_INSTALLATION/lib/swift/Block \
-    --build-tests
+    -Xcxx -I$SWIFT_INSTALLATION/lib/swift/Block
 
 Scripts/Linux/GenerateTestSymbolGraphs
 
@@ -17,7 +16,6 @@ swift test -c release \
     -Xcxx -I$SWIFT_INSTALLATION/lib/swift \
     -Xcxx -I$SWIFT_INSTALLATION/lib/swift/Block \
     --no-parallel \
-    --skip-build \
     --skip SymbolGraphValidationTests \
     --disable-testable-imports
 

Scripts/Linux/TestAll

@@ -0,0 +1,25 @@
+import SymbolGraphs
+import Symbols
+
+extension PackageNode
+{
+    @frozen public
+    struct Densified
+    {
+        public
+        let dependencies:[any Identifiable<Symbol.Package>]
+        public
+        let products:[SymbolGraph.Product]
+        public
+        let modules:[SymbolGraph.Module]
+
+        init(dependencies:[any Identifiable<Symbol.Package>],
+            products:[SymbolGraph.Product],
+            modules:[SymbolGraph.Module])
+        {
+            self.dependencies = dependencies
+            self.products = products
+            self.modules = modules
+        }
+    }
+}

Sources/PackageGraphs/Packages/PackageNode.Densified.swift

@@ -13,6 +13,9 @@ struct PackageNode:Identifiable
     public
     var dependencies:[any Identifiable<Symbol.Package>]
 
+    /// The name of the snippets directory.
+    public
+    var snippets:String
     public
     var products:[SymbolGraph.Product]
     public
@@ -26,13 +29,15 @@ struct PackageNode:Identifiable
     @inlinable public
     init(id:Symbol.Package,
         dependencies:[any Identifiable<Symbol.Package>],
+        snippets:String,
         products:[SymbolGraph.Product],
         modules:[SymbolGraph.Module],
         exclude:[[String]],
         root:Symbol.FileBase)
     {
         self.id = id
         self.dependencies = dependencies
+        self.snippets = snippets
         self.products = products
         self.modules = modules
         self.exclude = exclude
@@ -42,75 +47,72 @@ struct PackageNode:Identifiable
 extension PackageNode
 {
     public consuming
-    func flattened(dependencies:[PackageNode]) throws -> Self
-    {
-        try self.flatten(dependencies: dependencies)
-        return self
-    }
-
-    mutating
-    func flatten(dependencies:[PackageNode]) throws
+    func joined(with dependencies:[PackageNode]) throws -> ([Densified], Densified)
     {
-        var nodes:DigraphExplorer<ProductNode>.Nodes = .init()
-        for package:PackageNode in dependencies
+        let nodes:DigraphExplorer<ProductNode>.Nodes = try dependencies.reduce(into: .init())
         {
-            for product:SymbolGraph.Product in package.products
+            for product:SymbolGraph.Product in $1.products
             {
-                try nodes.index(.init(id: .init(name: product.name, package: package.id),
-                    predecessors: product.dependencies))
+                let id:Symbol.Product = .init(name: product.name, package: $1.id)
+                try $0.index(.init(id: id, predecessors: product.dependencies))
             }
         }
 
+        let densifiedUpstream:[Densified] = try dependencies.map
+        {
+            try $0.densified(products: nodes, packages: dependencies)
+        }
+        let densifiedSink:Densified = try self.densified(
+            products: nodes,
+            packages: dependencies)
+
+        return (densifiedUpstream, densifiedSink)
+    }
+
+    private consuming
+    func densified(
+        products:DigraphExplorer<ProductNode>.Nodes,
+        packages:[PackageNode]) throws -> Densified
+    {
         var cache:[Symbol.Product: [Symbol.Product]] = [:]
 
-        self.products = try self.products.map
+        for i:Int in self.products.indices
         {
-            .init(name: $0.name, type: $0.type,
-                dependencies: try nodes.included(by: $0.dependencies, cache: &cache),
-                cultures: $0.cultures)
+            try
+            {
+                $0 = try products.included(by: $0, cache: &cache)
+            } (&self.products[i].dependencies)
         }
-        self.modules = try self.modules.map
+        for i:Int in self.modules.indices
         {
-            .init(name: $0.name, type: $0.type, dependencies: .init(
-                    products: try nodes.included(by: $0.dependencies.products,
-                        cache: &cache),
-                    modules: $0.dependencies.modules),
-                location: $0.location)
+            try
+            {
+                $0 = try products.included(by: $0, cache: &cache)
+            } (&self.modules[i].dependencies.products)
         }
 
-        let declared:[Symbol.Package: any Identifiable<Symbol.Package>] =
-            self.dependencies.reduce(into: [:]) { $0[$1.id] = $1 }
-
-        let actuallyUsed:Set<Symbol.Package> = cache.values.reduce(into: [])
+        let dependenciesActuallyUsed:Set<Symbol.Package> = cache.values.reduce(into: [])
         {
             for product:Symbol.Product in $1
             {
                 $0.insert(product.package)
             }
         }
 
-        let directedEdges:[(Symbol.Package, Symbol.Package)] = dependencies.reduce(into: [])
+        let dependenciesDeclared:[Symbol.Package: any Identifiable<Symbol.Package>] =
+            self.dependencies.reduce(into: [:]) { $0[$1.id] = $1 }
+
+        let dependenciesBlamed:[any Identifiable<Symbol.Package>] = packages.reduce(
+            into: [])
         {
-            for dependency:any Identifiable<Symbol.Package> in $1.dependencies
+            if  dependenciesActuallyUsed.contains($1.id)
             {
-                $0.append((dependency.id, $1.id))
+                $0.append(dependenciesDeclared[$1.id] ?? TransitiveDependency.init(id: $1.id))
             }
         }
 
-        //  FIXME: in Swift 6, it will be legal to have cyclic package dependencies!
-        guard
-        let dependenciesOrdered:[PackageNode] = dependencies.sortedTopologically(
-            by: directedEdges)
-        else
-        {
-            throw DigraphCycleError<PackageNode>.init()
-        }
-
-        self.dependencies = dependenciesOrdered.compactMap
-        {
-            actuallyUsed.contains($0.id)
-                ? declared[$0.id] ?? TransitiveDependency.init(id: $0.id)
-                : nil
-        }
+        return .init(dependencies: dependenciesBlamed,
+            products: self.products,
+            modules: self.modules)
     }
 }