Extract mangled names from .swiftinterface rather than using nm

Makefile

@@ -37,6 +37,20 @@ BUILD_DIR := .build/$(ARCH_SUBDIR)-apple-macosx
 LIB_SUFFIX := dylib
 endif
 
+ifeq ($(UNAME), Darwin)
+ifeq ("${TOOLCHAINS}", "")
+	SWIFTC := "xcrun swiftc"
+else
+	SWIFTC := "xcrun ${TOOLCHAINS}/usr/bin/swiftc"
+endif
+else
+ifeq ("${TOOLCHAINS}", "")
+	SWIFTC := "swiftc"
+else
+	SWIFTC := "${TOOLCHAINS}/usr/bin/swiftc"
+endif
+endif
+
 SAMPLES_DIR := "Samples"
 
 all: generate-all
@@ -83,11 +97,13 @@ JEXTRACT_BUILD_DIR="$(BUILD_DIR)/jextract"
 define make_swiftinterface
     $(eval $@_MODULE = $(1))
     $(eval $@_FILENAME = $(2))
-	eval swiftc \
+	eval ${SWIFTC} \
 		-emit-module-interface-path ${JEXTRACT_BUILD_DIR}/${$@_MODULE}/${$@_FILENAME}.swiftinterface \
 		-emit-module-path ${JEXTRACT_BUILD_DIR}/${$@_MODULE}/${$@_FILENAME}.swiftmodule \
 		-enable-library-evolution \
+		-Xfrontend -abi-comments-in-module-interface \
 		-module-name ${$@_MODULE} \
+		-Xfrontend -abi-comments-in-module-interface \
 		Sources/${$@_MODULE}/${$@_FILENAME}.swift
 	echo "Generated: ${JEXTRACT_BUILD_DIR}/${$@_MODULE}/${$@_FILENAME}.swiftinterface"
 endef

Package.swift

@@ -122,7 +122,6 @@ let package = Package(
   dependencies: [
     .package(url: "https://github.com/swiftlang/swift-syntax.git", branch: "main"),
     .package(url: "https://github.com/apple/swift-argument-parser", from: "1.5.0"),
-    .package(url: "https://github.com/apple/swift-system", from: "1.0.0"), // TODO: remove, we should not need 'nm' or process callouts
     .package(url: "https://github.com/apple/swift-collections.git", .upToNextMinor(from: "1.1.0")),
   ],
   targets: [
@@ -251,31 +250,6 @@ let package = Package(
       ]
     ),
 
-    // FIXME: This is swift-foundation's proposed Subprocess; remove when available
-    //  https://github.com/apple/swift-foundation/pull/439
-    .target(
-      name: "_Subprocess",
-      dependencies: [
-        "_SubprocessCShims",
-        .product(name: "SystemPackage", package: "swift-system"),
-      ],
-      swiftSettings: [
-        .swiftLanguageMode(.v5)
-      ]
-    ),
-    .target(
-      name: "_SubprocessCShims",
-      cSettings: [
-        .define(
-          "_CRT_SECURE_NO_WARNINGS",
-          .when(platforms: [.windows])
-        )
-      ],
-      swiftSettings: [
-        .swiftLanguageMode(.v5)
-      ]
-    ),
-
     .target(
       name: "JExtractSwift",
       dependencies: [
@@ -284,7 +258,6 @@ let package = Package(
         .product(name: "SwiftSyntaxBuilder", package: "swift-syntax"),
         .product(name: "ArgumentParser", package: "swift-argument-parser"),
         .product(name: "Collections", package: "swift-collections"),
-        "_Subprocess",
         "JavaTypes",
       ],
       swiftSettings: [

README.md

@@ -16,6 +16,35 @@ The primary purpose of this repository is to create an environment for collabora
 
 ## Dependencies
 
+### Required Swift Development Toolchains
+
+To build and use this project, currently, you will need to download a custom toolchain which includes some improvements in Swift that this project relies on:
+
+**Required toolchain download:**
+
+- Go to https://www.swift.org/download/
+- Find the "latest" `Trunk Development (main)` toolchain for your OS
+
+If these are too old, you can resort to one of these fallback toolchains:
+
+Fallback development toolchain on **macOS**:
+
+- https://ci.swift.org/job/swift-PR-toolchain-macos/1539/artifact/branch-main/swift-PR-76905-1539-osx.tar.gz
+
+Fallback development toolchain on **Linux (Ubuntu 22.04)**:
+
+```
+URL=$(curl -s "https://ci.swift.org/job/oss-swift-package-ubuntu-22_04/lastSuccessfulBuild/consoleText" | grep 'Toolchain: ' | sed 's/Toolchain: //g')
+wget ${URL} 
+```
+
+or just use the provided docker image (explained below).
+
+https://www.swift.org/download/
+
+
+### Required JDK versions
+
 This project consists of different modules which have different Swift and Java runtime requirements.
 
 **JavaKit** – the Swift macros allowing the invocation of Java libraries from Swift

Samples/SwiftKitSampleApp/src/test/java/com/example/swift/generated/GeneratedJavaKitExampleModuleTest.java

@@ -34,15 +34,13 @@ static void beforeAll() {
     }
 
     @Test
-    @DisabledOnOs(OS.LINUX) // FIXME: enable on Linux when we get new compiler with mangled names in swift interfaces
     void call_helloWorld() {
         ExampleSwiftLibrary.helloWorld();
 
         assertNotNull(ExampleSwiftLibrary.helloWorld$address());
     }
 
     @Test
-    @DisabledOnOs(OS.LINUX) // FIXME: enable on Linux when we get new compiler with mangled names in swift interfaces
     void call_globalTakeInt() {
         ExampleSwiftLibrary.globalTakeInt(12);
 

Samples/SwiftKitSampleApp/src/test/java/com/example/swift/generated/MySwiftClassTest.java

@@ -35,22 +35,19 @@ static void beforeAll() {
     }
 
     @Test
-    @DisabledOnOs(OS.LINUX) // FIXME: enable on Linux when we get new compiler with mangled names in swift interfaces
     void test_MySwiftClass_voidMethod() {
         MySwiftClass o = new MySwiftClass(12, 42);
         o.voidMethod();
     }
 
     @Test
-    @DisabledOnOs(OS.LINUX) // FIXME: enable on Linux when we get new compiler with mangled names in swift interfaces
     void test_MySwiftClass_makeIntMethod() {
         MySwiftClass o = new MySwiftClass(12, 42);
         var got = o.makeIntMethod();
         assertEquals(12, got);
     }
 
     @Test
-    @DisabledOnOs(OS.LINUX) // FIXME: enable on Linux when we get new compiler with mangled names in swift interfaces
     void test_MySwiftClass_property_len() {
         MySwiftClass o = new MySwiftClass(12, 42);
         var got = o.makeIntMethod();

Samples/SwiftKitSampleApp/src/test/java/org/swift/swiftkit/MySwiftClassTest.java

@@ -36,7 +36,6 @@ static void beforeAll() {
     }
 
     @Test
-    @DisabledOnOs(OS.LINUX) // FIXME: enable on Linux when we get new compiler with mangled names in swift interfaces
     void call_retain_retainCount_release() {
         var obj = new MySwiftClass(1, 2);
 

Sources/JExtractSwift/Convenience/Collection+Extensions.swift

@@ -12,30 +12,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-extension Collection {
-
-  @_alwaysEmitIntoClient
-  public func _mapAsync<T, E>(
-    _ transform: (Element) async throws(E) -> T
-  ) async throws(E) -> [T] {
-    let initialCapacity = underestimatedCount
-    var result = Array<T>()
-    result.reserveCapacity(initialCapacity)
-
-    var iterator = self.makeIterator()
-
-    // Add elements up to the initial capacity without checking for regrowth.
-    for _ in 0..<initialCapacity {
-      result.append(try await transform(iterator.next()!))
-    }
-    // Add remaining elements, if any.
-    while let element = iterator.next() {
-      result.append(try await transform(element))
-    }
-    return Array(result)
-  }
-}
-
 public extension Dictionary {
   /// Same values, corresponding to mapped keys.
   func mapKeys<Transformed>(

Sources/JExtractSwift/Swift2Java.swift

@@ -18,7 +18,7 @@ import SwiftSyntax
 import SwiftSyntaxBuilder
 
 /// Command-line utility, similar to `jextract` to export Swift types to Java.
-public struct SwiftToJava: AsyncParsableCommand {
+public struct SwiftToJava: ParsableCommand {
   public init() {}
 
   public static var _commandName: String {
@@ -41,7 +41,7 @@ public struct SwiftToJava: AsyncParsableCommand {
   @Argument(help: "The Swift interface files to export to Java.")
   var swiftInterfaceFiles: [String]
 
-  public func run() async throws {
+  public func run() throws {
     let interfaceFiles = self.swiftInterfaceFiles.dropFirst()
     print("Interface files: \(interfaceFiles)")
 
@@ -56,7 +56,7 @@ public struct SwiftToJava: AsyncParsableCommand {
       print("[\(fileNo)/\(interfaceFiles.count)] Importing module '\(swiftModule)', interface file: \(interfaceFile)")
       defer { fileNo += 1 }
 
-      try await translator.analyze(swiftInterfacePath: interfaceFile)
+      try translator.analyze(swiftInterfacePath: interfaceFile)
       try translator.writeImportedTypesTo(outputDirectory: outputDirectory)
 
       print("[\(fileNo)/\(interfaceFiles.count)] Imported interface file: \(interfaceFile) " + "done.".green)

Sources/JExtractSwift/Swift2JavaTranslator.swift

@@ -70,7 +70,7 @@ extension Swift2JavaTranslator {
   public func analyze(
     swiftInterfacePath: String,
     text: String? = nil
-  ) async throws {
+  ) throws {
     if text == nil {
       precondition(
         swiftInterfacePath.hasSuffix(Self.SWIFT_INTERFACE_SUFFIX),
@@ -85,12 +85,12 @@ extension Swift2JavaTranslator {
     log.trace("Analyze: \(swiftInterfacePath)")
     let text = try text ?? String(contentsOfFile: swiftInterfacePath)
 
-    try await analyzeSwiftInterface(interfaceFilePath: swiftInterfacePath, text: text)
+    try analyzeSwiftInterface(interfaceFilePath: swiftInterfacePath, text: text)
 
     log.info("Done processing: \(swiftInterfacePath)")
   }
 
-  package func analyzeSwiftInterface(interfaceFilePath: String, text: String) async throws {
+  package func analyzeSwiftInterface(interfaceFilePath: String, text: String) throws {
     assert(interfaceFilePath.hasSuffix(Self.SWIFT_INTERFACE_SUFFIX))
 
     let sourceFileSyntax = Parser.parse(source: text)
@@ -105,60 +105,8 @@ extension Swift2JavaTranslator {
       translator: self
     )
     visitor.walk(sourceFileSyntax)
-
-    try await self.postProcessImportedDecls()
   }
 
-  public func postProcessImportedDecls() async throws {
-    log.debug(
-      "Post process imported decls...",
-      metadata: [
-        "types": "\(importedTypes.count)",
-        "global/funcs": "\(importedGlobalFuncs.count)",
-      ]
-    )
-
-    // FIXME: the use of dylibs to get symbols is a hack we need to remove and replace with interfaces containing mangled names
-    let dylibPath = ".build/arm64-apple-macosx/debug/lib\(swiftModuleName).dylib"
-    guard var dylib = SwiftDylib(path: dylibPath) else {
-      log.warning(
-        """
-        Unable to find mangled names for imported symbols. Dylib not found: \(dylibPath) This method of obtaining symbols is a workaround; it will be removed.
-        """
-      )
-      return
-    }
-
-    importedGlobalFuncs = try await importedGlobalFuncs._mapAsync { funcDecl in
-      let funcDecl = try await dylib.fillInMethodMangledName(funcDecl)
-      log.info("Mapped method '\(funcDecl.identifier)' -> '\(funcDecl.swiftMangledName)'")
-      return funcDecl
-    }
-
-    importedTypes = Dictionary(uniqueKeysWithValues: try await importedTypes._mapAsync { (tyName, tyDecl) in
-      var tyDecl = tyDecl
-      log.info("Mapping type: \(tyDecl.swiftTypeName)")
-
-      tyDecl = try await dylib.fillInTypeMangledName(tyDecl)
-
-      log.info("Mapping members of: \(tyDecl.swiftTypeName)")
-      tyDecl.initializers = try await tyDecl.initializers._mapAsync { initDecl in
-        dylib.log.logLevel = .info
-
-        let initDecl = try await dylib.fillInAllocatingInitMangledName(initDecl)
-        log.info("Mapped initializer '\(initDecl.identifier)' -> '\(initDecl.swiftMangledName)'")
-        return initDecl
-      }
-
-      tyDecl.methods = try await tyDecl.methods._mapAsync { funcDecl in
-        let funcDecl = try await dylib.fillInMethodMangledName(funcDecl)
-        log.info("Mapped method '\(funcDecl.identifier)' -> '\(funcDecl.swiftMangledName)'")
-        return funcDecl
-      }
-
-      return (tyName, tyDecl)
-    })
-  }
 }
 
 // ===== --------------------------------------------------------------------------------------------------------------