From aa84e1931e5ce35aa62df1e2c38abaf7893f7ef4 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 16 Jun 2025 15:53:24 +0900 Subject: [PATCH 01/37] Introduce resolve subcommand and further dis-entangle commands --- Package.swift | 11 +- .../JExtractSwiftCommandPlugin.swift | 2 +- .../SwiftJavaPluginProtocol.swift | 4 +- Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift | 62 ++-- Samples/JavaDependencySampleApp/% | 0 .../JavaDependencySampleApp/ci-validate.sh | 8 + .../Configuration.swift | 12 +- .../GradleDependencyParsing.swift | 48 +++ .../JavaTranslator+Configuration.swift | 10 +- .../Commands/ConfigureCommand.swift | 40 +-- .../Commands/ResolveCommand.swift | 82 +++-- ...teWrappers.swift => WrapJavaCommand.swift} | 96 ++++++ Sources/SwiftJavaTool/CommonOptions.swift | 75 ++++- Sources/SwiftJavaTool/SwiftJava.swift | 283 ++++++++---------- .../SwiftJavaBaseAsyncParsableCommand.swift | 55 ++-- .../GradleDependencyParsingTests.swift | 45 +++ 16 files changed, 553 insertions(+), 280 deletions(-) create mode 100644 Samples/JavaDependencySampleApp/% create mode 100644 Sources/JavaKitConfigurationShared/GradleDependencyParsing.swift rename Sources/SwiftJavaTool/Commands/{SwiftJava+GenerateWrappers.swift => WrapJavaCommand.swift} (57%) create mode 100644 Tests/JavaKitConfigurationSharedTests/GradleDependencyParsingTests.swift diff --git a/Package.swift b/Package.swift index 55102874..4cefd08e 100644 --- a/Package.swift +++ b/Package.swift @@ -85,7 +85,7 @@ let javaIncludePath = "\(javaHome)/include" let package = Package( name: "SwiftJava", platforms: [ - .macOS(.v10_15) + .macOS(.v13) ], products: [ // ==== JavaKit (i.e. calling Java directly Swift utilities) @@ -467,6 +467,15 @@ let package = Package( ] ), + .testTarget( + name: "JavaKitConfigurationSharedTests", + dependencies: ["JavaKitConfigurationShared"], + swiftSettings: [ + .swiftLanguageMode(.v5), + .unsafeFlags(["-I\(javaIncludePath)", "-I\(javaPlatformIncludePath)"]) + ] + ), + .testTarget( name: "JExtractSwiftTests", dependencies: [ diff --git a/Plugins/JExtractSwiftCommandPlugin/JExtractSwiftCommandPlugin.swift b/Plugins/JExtractSwiftCommandPlugin/JExtractSwiftCommandPlugin.swift index f97feab6..f2327dad 100644 --- a/Plugins/JExtractSwiftCommandPlugin/JExtractSwiftCommandPlugin.swift +++ b/Plugins/JExtractSwiftCommandPlugin/JExtractSwiftCommandPlugin.swift @@ -82,7 +82,7 @@ final class JExtractSwiftCommandPlugin: SwiftJavaPluginProtocol, BuildToolPlugin ] // arguments.append(sourceDir) // TODO: we could do this shape maybe? to have the dirs last? if let package = configuration?.javaPackage, !package.isEmpty { - ["--java-package", package] + arguments += ["--java-package", package] } return arguments diff --git a/Plugins/PluginsShared/SwiftJavaPluginProtocol.swift b/Plugins/PluginsShared/SwiftJavaPluginProtocol.swift index 68f8964e..57e38ef7 100644 --- a/Plugins/PluginsShared/SwiftJavaPluginProtocol.swift +++ b/Plugins/PluginsShared/SwiftJavaPluginProtocol.swift @@ -21,8 +21,6 @@ protocol SwiftJavaPluginProtocol { extension SwiftJavaPluginProtocol { func log(_ message: @autoclosure () -> String, terminator: String = "\n") { -// if self.verbose { - print("[\(pluginName)] \(message())", terminator: terminator) -// } + print("[\(pluginName)] \(message())", terminator: terminator) } } diff --git a/Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift b/Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift index effdcc53..3c989db1 100644 --- a/Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift +++ b/Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift @@ -88,24 +88,13 @@ struct SwiftJavaBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { } var arguments: [String] = [] - arguments += argumentsModuleName(sourceModule: sourceModule) + arguments += argumentsSwiftModuleDeprecated(sourceModule: sourceModule) arguments += argumentsOutputDirectory(context: context) - - arguments += dependentConfigFiles.flatMap { moduleAndConfigFile in - let (moduleName, configFile) = moduleAndConfigFile - return [ - "--depends-on", - "\(moduleName)=\(configFile.path(percentEncoded: false))" - ] - } + arguments += argumentsDependedOnConfigs(dependentConfigFiles) arguments.append(configFile.path(percentEncoded: false)) -// guard let classes = config.classes else { -// log("Config at \(configFile) did not have 'classes' configured, skipping java2swift step.") -// return [] -// } let classes = config.classes ?? [:] - print("Classes to wrap: \(classes.map(\.key))") + print("[swift-java-plugin] Classes to wrap (\(classes.count)): \(classes.map(\.key))") /// Determine the set of Swift files that will be emitted by the Java2Swift tool. // TODO: this is not precise and won't work with more advanced Java files, e.g. lambdas etc. @@ -165,12 +154,14 @@ struct SwiftJavaBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { .buildCommand( displayName: displayName, executable: executable, - arguments: [ - // FIXME: change to 'resolve' subcommand - "--fetch", configFile.path(percentEncoded: false), - "--swift-module", sourceModule.name, - "--output-directory", outputDirectory(context: context, generated: false).path(percentEncoded: false) - ], + arguments: ["resolve"] + + argumentsOutputDirectory(context: context, generated: false) + + argumentsSwiftModule(sourceModule: sourceModule) + // + [ + // // explicitly provide config path + // configFile.path(percentEncoded: false) + // ] + , environment: [:], inputFiles: [configFile], outputFiles: fetchDependenciesOutputFiles @@ -192,8 +183,10 @@ struct SwiftJavaBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { outputFiles: outputSwiftFiles ) ] - } else { - log("No Swift output files, skip wrapping") + } + + if commands.isEmpty { + log("No swift-java commands for module '\(sourceModule.name)'") } return commands @@ -201,19 +194,38 @@ struct SwiftJavaBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { } extension SwiftJavaBuildToolPlugin { - func argumentsModuleName(sourceModule: Target) -> [String] { + func argumentsSwiftModule(sourceModule: Target) -> [String] { return [ "--swift-module", sourceModule.name ] } - + + // FIXME: remove this and the deprecated property inside SwiftJava, this is a workaround + // since we cannot have the same option in common options and in the top level + // command from which we get into sub commands. The top command will NOT have this option. + func argumentsSwiftModuleDeprecated(sourceModule: Target) -> [String] { + return [ + "--swift-module-deprecated", sourceModule.name + ] + } + func argumentsOutputDirectory(context: PluginContext, generated: Bool = true) -> [String] { return [ "--output-directory", outputDirectory(context: context, generated: generated).path(percentEncoded: false) ] } - + + func argumentsDependedOnConfigs(_ dependentConfigFiles: [(String, URL)]) -> [String] { + dependentConfigFiles.flatMap { moduleAndConfigFile in + let (moduleName, configFile) = moduleAndConfigFile + return [ + "--depends-on", + "\(moduleName)=\(configFile.path(percentEncoded: false))" + ] + } + } + func outputDirectory(context: PluginContext, generated: Bool = true) -> URL { let dir = context.pluginWorkDirectoryURL if generated { diff --git a/Samples/JavaDependencySampleApp/% b/Samples/JavaDependencySampleApp/% new file mode 100644 index 00000000..e69de29b diff --git a/Samples/JavaDependencySampleApp/ci-validate.sh b/Samples/JavaDependencySampleApp/ci-validate.sh index 7012b841..2adce5a7 100755 --- a/Samples/JavaDependencySampleApp/ci-validate.sh +++ b/Samples/JavaDependencySampleApp/ci-validate.sh @@ -3,4 +3,12 @@ set -e set -x +# invoke resolve as part of a build run swift run --disable-sandbox + +# explicitly invoke resolve without explicit path or dependency +# the dependencies should be uses from the --swift-module +.build/plugins/tools/debug/SwiftJavaTool-tool resolve \ + Sources/JavaCommonsCSV/swift-java.config \ + --swift-module JavaCommonsCSV \ + --output-directory .build/plugins/outputs/javadependencysampleapp/JavaCommonsCSV/destination/SwiftJavaPlugin/ diff --git a/Sources/JavaKitConfigurationShared/Configuration.swift b/Sources/JavaKitConfigurationShared/Configuration.swift index 2298b9fa..1953167e 100644 --- a/Sources/JavaKitConfigurationShared/Configuration.swift +++ b/Sources/JavaKitConfigurationShared/Configuration.swift @@ -46,11 +46,7 @@ public struct Configuration: Codable { public var classpath: String? = nil public var classpathEntries: [String] { - guard let classpath else { - return [] - } - - return classpath.split(separator: ":").map(String.init) + return classpath?.split(separator: ":").map(String.init) ?? [] } /// The Java classes that should be translated to Swift. The keys are @@ -80,6 +76,12 @@ public struct JavaDependencyDescriptor: Hashable, Codable { public var artifactID: String public var version: String + public init(groupID: String, artifactID: String, version: String) { + self.groupID = groupID + self.artifactID = artifactID + self.version = version + } + public init(from decoder: any Decoder) throws { let container = try decoder.singleValueContainer() let string = try container.decode(String.self) diff --git a/Sources/JavaKitConfigurationShared/GradleDependencyParsing.swift b/Sources/JavaKitConfigurationShared/GradleDependencyParsing.swift new file mode 100644 index 00000000..bd4aada3 --- /dev/null +++ b/Sources/JavaKitConfigurationShared/GradleDependencyParsing.swift @@ -0,0 +1,48 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation + +// Regex is not sendable yet so we can't cache it in a let +fileprivate var GradleDependencyDescriptorRegex: Regex<(Substring, Substring, Substring, Substring)> { + try! Regex(#"^([^:]+):([^:]+):(\d[^:]+)$"#) // TODO: improve the regex to be more precise +} + +// note: can't use `package` access level since it would break in usage in plugins in `_PluginsShared`. +public func parseDependencyDescriptor(_ descriptor: String) -> JavaDependencyDescriptor? { + guard let match = try? GradleDependencyDescriptorRegex.firstMatch(in: descriptor) else { + return nil + } + + let groupID = String(match.1) + let artifactID = String(match.2) + let version = String(match.3) + + return JavaDependencyDescriptor(groupID: groupID, artifactID: artifactID, version: version) +} + +// note: can't use `package` access level since it would break in usage in plugins in `_PluginsShared`. +public func parseDependencyDescriptors(_ string: String) -> [JavaDependencyDescriptor] { + let descriptors = string.components(separatedBy: ",") + var parsedDependencies: [JavaDependencyDescriptor] = [] + parsedDependencies.reserveCapacity(descriptors.count) + + for descriptor in descriptors { + if let dependency = parseDependencyDescriptor(descriptor) { + parsedDependencies.append(dependency) + } + } + + return parsedDependencies +} \ No newline at end of file diff --git a/Sources/SwiftJavaLib/JavaTranslator+Configuration.swift b/Sources/SwiftJavaLib/JavaTranslator+Configuration.swift index e0f6d0cb..0f764633 100644 --- a/Sources/SwiftJavaLib/JavaTranslator+Configuration.swift +++ b/Sources/SwiftJavaLib/JavaTranslator+Configuration.swift @@ -16,11 +16,11 @@ import Foundation import JavaKitConfigurationShared extension JavaTranslator { - /// Read a configuration file from the given URL. - package static func readConfiguration(from url: URL) throws -> Configuration { - let contents = try Data(contentsOf: url) - return try JSONDecoder().decode(Configuration.self, from: contents) - } +// /// Read a configuration file from the given URL. +// package static func readConfiguration(from url: URL) throws -> Configuration { +// let contents = try Data(contentsOf: url) +// return try JSONDecoder().decode(Configuration.self, from: contents) +// } /// Load the configuration file with the given name to populate the known set of /// translated Java classes. diff --git a/Sources/SwiftJavaTool/Commands/ConfigureCommand.swift b/Sources/SwiftJavaTool/Commands/ConfigureCommand.swift index 8574888d..54859d3d 100644 --- a/Sources/SwiftJavaTool/Commands/ConfigureCommand.swift +++ b/Sources/SwiftJavaTool/Commands/ConfigureCommand.swift @@ -66,42 +66,11 @@ extension SwiftJava { extension SwiftJava.ConfigureCommand { mutating func runSwiftJavaCommand(config: inout Configuration) async throws { - // Form a class path from all of our input sources: - // * Command-line option --classpath - let classpathOptionEntries: [String] = self.commonJVMOptions.classpath.flatMap { $0.split(separator: ":").map(String.init) } - let classpathFromEnv = ProcessInfo.processInfo.environment["CLASSPATH"]?.split(separator: ":").map(String.init) ?? [] - let classpathFromConfig: [String] = config.classpath?.split(separator: ":").map(String.init) ?? [] - print("[debug][swift-java] Base classpath from config: \(classpathFromConfig)") - - var classpathEntries: [String] = classpathFromConfig - - let swiftJavaCachedModuleClasspath = findSwiftJavaClasspaths(in: - // self.effectiveCacheDirectory ?? - FileManager.default.currentDirectoryPath) - print("[debug][swift-java] Classpath from *.swift-java.classpath files: \(swiftJavaCachedModuleClasspath)") - classpathEntries += swiftJavaCachedModuleClasspath - - if !classpathOptionEntries.isEmpty { - print("[debug][swift-java] Classpath from options: \(classpathOptionEntries)") - classpathEntries += classpathOptionEntries - } else { - // * Base classpath from CLASSPATH env variable - print("[debug][swift-java] Classpath from environment: \(classpathFromEnv)") - classpathEntries += classpathFromEnv - } - - let extraClasspath = input ?? "" // FIXME: just use the -cp as usual - let extraClasspathEntries = extraClasspath.split(separator: ":").map(String.init) - print("[debug][swift-java] Extra classpath: \(extraClasspathEntries)") - classpathEntries += extraClasspathEntries + let classpathEntries = + self.configureCommandJVMClasspath(effectiveSwiftModuleURL: self.effectiveSwiftModuleURL, config: config) - // Bring up the Java VM when necessary - - if logLevel >= .debug { - let classpathString = classpathEntries.joined(separator: ":") - print("[debug][swift-java] Initialize JVM with classpath: \(classpathString)") - } - let jvm = try JavaVirtualMachine.shared(classpath: classpathEntries) + let jvm = + try self.makeJVM(classpathEntries: classpathEntries) try emitConfiguration(classpath: self.commonJVMOptions.classpath, environment: jvm.environment()) } @@ -178,6 +147,7 @@ extension SwiftJava.ConfigureCommand { // Write the file. try writeContents( contents, + outputDirectory: self.actualOutputDirectory, to: "swift-java.config", description: "swift-java configuration file" ) diff --git a/Sources/SwiftJavaTool/Commands/ResolveCommand.swift b/Sources/SwiftJavaTool/Commands/ResolveCommand.swift index eb96e490..dafa2520 100644 --- a/Sources/SwiftJavaTool/Commands/ResolveCommand.swift +++ b/Sources/SwiftJavaTool/Commands/ResolveCommand.swift @@ -24,12 +24,13 @@ import JavaKitShared import _Subprocess extension SwiftJava { - struct ResolveCommand: SwiftJavaBaseAsyncParsableCommand { + struct ResolveCommand: SwiftJavaBaseAsyncParsableCommand, HasCommonOptions, HasCommonJVMOptions { static let configuration = CommandConfiguration( commandName: "resolve", abstract: "Resolve dependencies and write the resulting swift-java.classpath file") @OptionGroup var commonOptions: SwiftJava.CommonOptions + @OptionGroup var commonJVMOptions: SwiftJava.CommonJVMOptions @Option(help: "The name of the Swift module into which the resulting Swift types will be generated.") var swiftModule: String @@ -38,31 +39,79 @@ extension SwiftJava { swiftModule } + @Argument( + help: """ + Additional configuration paths (swift-java.config) files, with defined 'dependencies', \ + or dependency descriptors formatted as 'groupID:artifactID:version' separated by ','. \ + May be empty, in which case the target Swift module's configuration's 'dependencies' will be used. + """ + ) + var input: String? } } extension SwiftJava.ResolveCommand { + var SwiftJavaClasspathPrefix: String { "SWIFT_JAVA_CLASSPATH:" } + var printRuntimeClasspathTaskName: String { "printRuntimeClasspath" } + mutating func runSwiftJavaCommand(config: inout Configuration) async throws { - fatalError("NOT IMPLEMENTED: resolve") - } -} + // Form a class path from all of our input sources: + let classpathOptionEntries: [String] = self.classpathEntries + let classpathFromEnv = self.classpathEnvEntries + let classpathFromConfig: [String] = config.classpath?.split(separator: ":").map(String.init) ?? [] + print("[debug][swift-java] Base classpath from config: \(classpathFromConfig)") + var classpathEntries: [String] = classpathFromConfig + let classPathFilesSearchDirectory = self.effectiveSwiftModuleURL.absoluteString + print("[debug][swift-java] Search *.swift-java.classpath in: \(classPathFilesSearchDirectory)") + let swiftJavaCachedModuleClasspath = findSwiftJavaClasspaths(in: classPathFilesSearchDirectory) -extension SwiftJava { - var SwiftJavaClasspathPrefix: String { "SWIFT_JAVA_CLASSPATH:" } + print("[debug][swift-java] Classpath from *.swift-java.classpath files: \(swiftJavaCachedModuleClasspath)") + classpathEntries += swiftJavaCachedModuleClasspath - var printRuntimeClasspathTaskName: String { "printRuntimeClasspath" } + if logLevel >= .debug { + let classpathString = classpathEntries.joined(separator: ":") + print("[debug][swift-java] Initialize JVM with classpath: \(classpathString)") + } + let jvm = try JavaVirtualMachine.shared(classpath: classpathEntries) - func fetchDependencies(swiftModule: String, - dependencies: [JavaDependencyDescriptor]) async throws -> ResolvedDependencyClasspath { + var dependenciesToResolve: [JavaDependencyDescriptor] = [] + if let input, let inputDependencies = parseDependencyDescriptor(input) { + dependenciesToResolve.append(inputDependencies) + } + if let dependencies = config.dependencies { + dependenciesToResolve += dependencies + } + + if dependenciesToResolve.isEmpty { + print("[warn][swift-java] Attempted to 'resolve' dependencies but no dependencies specified in swift-java.config or command input!") + return + } + + let dependenciesClasspath = + try await resolveDependencies(swiftModule: swiftModule, dependencies: dependenciesToResolve) + + // FIXME: disentangle the output directory from SwiftJava and then make it a required option in this Command + guard let outputDirectory = self.commonOptions.outputDirectory else { + fatalError("error: Must specify --output-directory in 'resolve' mode! This option will become explicitly required") + } + + try writeSwiftJavaClasspathFile( + swiftModule: swiftModule, + outputDirectory: outputDirectory, + resolvedClasspath: dependenciesClasspath) + } + + func resolveDependencies( + swiftModule: String, dependencies: [JavaDependencyDescriptor] + ) async throws -> ResolvedDependencyClasspath { let deps = dependencies.map { $0.descriptionGradleStyle } print("[debug][swift-java] Resolve and fetch dependencies for: \(deps)") let dependenciesClasspath = await resolveDependencies(dependencies: dependencies) let classpathEntries = dependenciesClasspath.split(separator: ":") - print("[info][swift-java] Resolved classpath for \(deps.count) dependencies of '\(swiftModule)', classpath entries: \(classpathEntries.count), ", terminator: "") print("done.".green) @@ -153,9 +202,9 @@ extension SwiftJava { try settingsGradleText.write(to: settingsGradle, atomically: true, encoding: .utf8) } - mutating func writeFetchedDependenciesClasspath( + mutating func writeSwiftJavaClasspathFile( swiftModule: String, - cacheDir: String, + outputDirectory: String, resolvedClasspath: ResolvedDependencyClasspath) throws { // Convert the artifact name to a module name // e.g. reactive-streams -> ReactiveStreams @@ -163,13 +212,14 @@ extension SwiftJava { // The file contents are just plain let contents = resolvedClasspath.classpath - print("[debug][swift-java] Resolved dependency: \(commonJVMOptions.classpath)") + let filename = "\(swiftModule).swift-java.classpath" + print("[debug][swift-java] Write resolved dependencies to: \(outputDirectory)/\(filename)") // Write the file try writeContents( contents, - outputDirectoryOverride: URL(fileURLWithPath: cacheDir), - to: "\(swiftModule).swift-java.classpath", + outputDirectory: URL(fileURLWithPath: outputDirectory), + to: filename, description: "swift-java.classpath file for module \(swiftModule)" ) } @@ -184,8 +234,6 @@ extension SwiftJava { var searchDir = URL(fileURLWithPath: FileManager.default.currentDirectoryPath) while searchDir.pathComponents.count > 1 { - print("[COPY] Search dir: \(searchDir)") - let gradlewFile = searchDir.appendingPathComponent("gradlew") let gradlewExists = FileManager.default.fileExists(atPath: gradlewFile.path) guard gradlewExists else { diff --git a/Sources/SwiftJavaTool/Commands/SwiftJava+GenerateWrappers.swift b/Sources/SwiftJavaTool/Commands/WrapJavaCommand.swift similarity index 57% rename from Sources/SwiftJavaTool/Commands/SwiftJava+GenerateWrappers.swift rename to Sources/SwiftJavaTool/Commands/WrapJavaCommand.swift index 676b278d..7fdedcbf 100644 --- a/Sources/SwiftJavaTool/Commands/SwiftJava+GenerateWrappers.swift +++ b/Sources/SwiftJavaTool/Commands/WrapJavaCommand.swift @@ -21,6 +21,101 @@ import SwiftJavaLib import JavaKitConfigurationShared extension SwiftJava { + + struct WrapJavaCommand: SwiftJavaBaseAsyncParsableCommand, HasCommonOptions, HasCommonJVMOptions { + static let configuration = CommandConfiguration( + commandName: "wrap-java", + abstract: "Wrap Java classes with corresponding Swift bindings.") + + @OptionGroup var commonOptions: SwiftJava.CommonOptions + @OptionGroup var commonJVMOptions: SwiftJava.CommonJVMOptions + + @Option(help: "The name of the Swift module into which the resulting Swift types will be generated.") + var swiftModule: String + + var effectiveSwiftModule: String { + swiftModule + } + + @Option( + help: """ + A swift-java configuration file for a given Swift module name on which this module depends, + e.g., JavaKitJar=Sources/JavaKitJar/Java2Swift.config. There should be one of these options + for each Swift module that this module depends on (transitively) that contains wrapped Java sources. + """ + ) + var dependsOn: [String] = [] + + @Option(help: "The Java package the generated Java code should be emitted into.") + var javaPackage: String? = nil + + @Option(help: "The names of Java classes whose declared native methods will be implemented in Swift.") + var swiftNativeImplementation: [String] = [] + + @Argument(help: "Path to .jar file whose Java classes should be wrapped using Swift bindings") + var input: String + } +} + +extension SwiftJava.WrapJavaCommand { + + mutating func runSwiftJavaCommand(config: inout Configuration) async throws { + if let javaPackage { + config.javaPackage = javaPackage + } + + // Load all of the dependent configurations and associate them with Swift + // modules. + let dependentConfigs = try loadDependentConfigs() + + // Configure our own classpath based on config + var classpathEntries = + self.configureCommandJVMClasspath(effectiveSwiftModuleURL: self.effectiveSwiftModuleURL, config: config) + + // Include classpath entries which libs we depend on require... + for (fromModule, config) in dependentConfigs { + // TODO: may need to resolve the dependent configs rather than just get their configs + // TODO: We should cache the resolved classpaths as well so we don't do it many times + config.classpath.map { entry in + print("[trace][swift-java] Add dependent config (\(fromModule)) classpath element: \(entry)") + classpathEntries.append(entry) + } + } + + let completeClasspath = classpathEntries.joined(separator: ":") + let jvm = try self.makeJVM(classpathEntries: classpathEntries) + + try self.generateWrappers( + config: config, + classpath: completeClasspath, + dependentConfigs: dependentConfigs, + environment: jvm.environment() + ) + } +} + +extension SwiftJava.WrapJavaCommand { + + /// Load all dependent configs configured with `--depends-on` and return a list of + /// `(SwiftModuleName, Configuration)` tuples. + func loadDependentConfigs() throws -> [(String, Configuration)] { + try dependsOn.map { dependentConfig in + guard let equalLoc = dependentConfig.firstIndex(of: "=") else { + throw JavaToSwiftError.badConfigOption(dependentConfig) + } + + let afterEqual = dependentConfig.index(after: equalLoc) + let swiftModuleName = String(dependentConfig[.. [String] { + // Form a class path from all of our input sources: + // * Command-line option --classpath + let classpathOptionEntries: [String] = self.classpathEntries + let classpathFromEnv = ProcessInfo.processInfo.environment["CLASSPATH"]?.split(separator: ":").map(String.init) ?? [] + let classpathFromConfig: [String] = config.classpath?.split(separator: ":").map(String.init) ?? [] + print("[debug][swift-java] Base classpath from config: \(classpathFromConfig)") + + var classpathEntries: [String] = classpathFromConfig + + let classPathFilesSearchDirectory = effectiveSwiftModuleURL.absoluteString + print("[debug][swift-java] Search *.swift-java.classpath in: \(classPathFilesSearchDirectory)") + let swiftJavaCachedModuleClasspath = findSwiftJavaClasspaths(in: classPathFilesSearchDirectory) + + print("[debug][swift-java] Classpath from *.swift-java.classpath files: \(swiftJavaCachedModuleClasspath)") + classpathEntries += swiftJavaCachedModuleClasspath + + if !classpathOptionEntries.isEmpty { + print("[debug][swift-java] Classpath from options: \(classpathOptionEntries)") + classpathEntries += classpathOptionEntries + } else { + // * Base classpath from CLASSPATH env variable + print("[debug][swift-java] Classpath from environment: \(classpathFromEnv)") + classpathEntries += classpathFromEnv + } + + let extraClasspath = self.commonJVMOptions.classpath + let extraClasspathEntries = extraClasspath.split(separator: ":").map(String.init) + print("[debug][swift-java] Extra classpath: \(extraClasspathEntries)") + classpathEntries += extraClasspathEntries + + // Bring up the Java VM when necessary + + // if logLevel >= .debug { + let classpathString = classpathEntries.joined(separator: ":") + print("[debug][swift-java] Initialize JVM with classpath: \(classpathString)") + // } + + return classpathEntries + } + + func makeJVM(classpathEntries: [String]) throws -> JavaVirtualMachine { + try JavaVirtualMachine.shared(classpath: classpathEntries) + } } \ No newline at end of file diff --git a/Sources/SwiftJavaTool/SwiftJava.swift b/Sources/SwiftJavaTool/SwiftJava.swift index 0e6e64ae..b54a7b58 100644 --- a/Sources/SwiftJavaTool/SwiftJava.swift +++ b/Sources/SwiftJavaTool/SwiftJava.swift @@ -27,7 +27,7 @@ import JavaKitShared /// Command-line utility to drive the export of Java classes into Swift types. @main -struct SwiftJava: SwiftJavaBaseAsyncParsableCommand { // FIXME: this is just a normal async command, no parsing happening here +struct SwiftJava: SwiftJavaBaseAsyncParsableCommand, HasCommonJVMOptions { // FIXME: this is just a normal async command, no parsing happening here static var _commandName: String { "swift-java" } static let configuration = CommandConfiguration( @@ -37,26 +37,32 @@ struct SwiftJava: SwiftJavaBaseAsyncParsableCommand { // FIXME: this is just a n ResolveCommand.self, ]) - @Option(help: "The name of the Swift module into which the resulting Swift types will be generated.") - var swiftModule: String? + // FIXME: this must be removed when we move things out from the SwiftJava main file! + @Option( + name: .long, + help: "The name of the Swift module into which the resulting Swift types will be generated.") + var swiftModuleDeprecated: String? var effectiveSwiftModule: String { - swiftModule ?? "UnknownSwiftModule" + if let module = swiftModuleDeprecated { + module + } else if let module = self.outputSwift?.split(separator: "/").last { + String(module) + } else { + "UnknownSwiftModule" + } } - - @Option( - help: - "A Java2Swift configuration file for a given Swift module name on which this module depends, e.g., JavaKitJar=Sources/JavaKitJar/Java2Swift.config. There should be one of these options for each Swift module that this module depends on (transitively) that contains wrapped Java sources." - ) - var dependsOn: [String] = [] - - @Flag(help: "Fetch dependencies from given target (containing swift-java configuration) or dependency string") - var fetch: Bool = false - - @Option( - help: "The names of Java classes whose declared native methods will be implemented in Swift." - ) - var swiftNativeImplementation: [String] = [] +// +// @Option( +// help: +// "A Java2Swift configuration file for a given Swift module name on which this module depends, e.g., JavaKitJar=Sources/JavaKitJar/Java2Swift.config. There should be one of these options for each Swift module that this module depends on (transitively) that contains wrapped Java sources." +// ) +// var dependsOn: [String] = [] +// +// @Option( +// help: "The names of Java classes whose declared native methods will be implemented in Swift." +// ) +// var swiftNativeImplementation: [String] = [] @Option(help: "The directory where generated Swift files should be written. Generally used with jextract mode.") var outputSwift: String? = nil @@ -64,16 +70,12 @@ struct SwiftJava: SwiftJavaBaseAsyncParsableCommand { // FIXME: this is just a n @Option(help: "The directory where generated Java files should be written. Generally used with jextract mode.") var outputJava: String? = nil - @Option(help: "The Java package the generated Java code should be emitted into.") - var javaPackage: String? = nil +// @Option(help: "The Java package the generated Java code should be emitted into.") +// var javaPackage: String? = nil @Option(help: "The mode of generation to use for the output files. Used with jextract mode.") var mode: GenerationMode = .ffm -// // TODO: clarify this vs outputSwift (history: outputSwift is jextract, and this was java2swift) -// @Option(name: .shortAndLong, help: "The directory in which to output the generated Swift files or the SwiftJava configuration file.") -// var outputDirectory: String? = nil - @Option(name: .shortAndLong, help: "Directory where to write cached values (e.g. swift-java.classpath files)") var cacheDirectory: String? = nil @@ -83,48 +85,41 @@ struct SwiftJava: SwiftJavaBaseAsyncParsableCommand { // FIXME: this is just a n var effectiveCacheDirectory: String? { if let cacheDirectory { return cacheDirectory - } else if let outputDirectory = commonOptions.outputDirectory { - return outputDirectory } else { return nil } } - @Argument( - help: "The input file, which is either a Java2Swift configuration file or (if '-jar' was specified) a Jar file." - ) +// @Argument( +// help: "The input file, which is either a Java2Swift configuration file or (if '-jar' was specified) a Jar file." +// ) var input: String? // FIXME: top level command cannot have input argument like this // FIXME: this is subcommands /// Describes what kind of generation action is being performed by swift-java. enum ToolMode { - // /// Generate a configuration file given a Jar file. - // case configuration(extraClasspath: String) // FIXME: this is more like "extract" configuration from classpath - - /// Generate Swift wrappers for Java classes based on the given - /// configuration. - case classWrappers - - /// Fetch dependencies for a module - case fetchDependencies +// /// Generate Swift wrappers for Java classes based on the given +// /// configuration. +// case classWrappers /// Extract Java bindings from provided Swift sources. case jextract // TODO: carry jextract specific config here? } mutating func runSwiftJavaCommand(config: inout Configuration) async throws { - guard CommandLine.arguments.count > 2 else { + guard CommandLine.arguments.count >= 2 else { // there's no "default" command, print USAGE when no arguments/parameters are passed. print("error: Must specify mode subcommand (e.g. configure, resolve, jextract, ...).\n\n\(Self.helpMessage())") return } - if let javaPackage { - config.javaPackage = javaPackage - } +// if let javaPackage { +// config.javaPackage = javaPackage +// } // Determine the mode in which we'll execute. - let toolMode: ToolMode + let toolMode: ToolMode = .jextract + // TODO: some options are exclusive to each other so we should detect that if let inputSwift = commonOptions.inputSwift { guard let inputSwift = commonOptions.inputSwift else { @@ -139,63 +134,60 @@ struct SwiftJava: SwiftJavaBaseAsyncParsableCommand { // FIXME: this is just a n print("[swift-java] --output-java enabled 'jextract' mode, however no --output-java directory was provided!\n\(Self.helpMessage())") return } - config.swiftModule = self.swiftModule ?? "UnknownModule" + config.swiftModule = self.effectiveSwiftModule config.inputSwiftDirectory = inputSwift config.outputSwiftDirectory = outputSwift config.outputJavaDirectory = outputJava - toolMode = .jextract -// } else if jar { -// guard let input else { -// fatalError("Mode -jar requires path\n\(Self.helpMessage())") -// } -// toolMode = .configuration(extraClasspath: input) - } else if fetch { - guard let input else { - fatalError("Mode 'fetch' requires path\n\(Self.helpMessage())") - } - config = try JavaTranslator.readConfiguration(from: URL(fileURLWithPath: input)) - guard let dependencies = config.dependencies else { - print("[swift-java] Running in 'fetch dependencies' mode but dependencies list was empty!") - print("[swift-java] Nothing to do: done.") - return - } - toolMode = .fetchDependencies - } else { - guard let input else { - fatalError("Mode -jar requires path\n\(Self.helpMessage())") - } - config = try JavaTranslator.readConfiguration(from: URL(fileURLWithPath: input)) - toolMode = .classWrappers +// toolMode = .jextract +// } else if fetch { +// guard let input else { +// fatalError("Mode 'fetch' requires path\n\(Self.helpMessage())") +// } +// config = try JavaTranslator.readConfiguration(from: URL(fileURLWithPath: input)) +// guard let dependencies = config.dependencies else { +// print("[swift-java] Running in 'fetch dependencies' mode but dependencies list was empty!") +// print("[swift-java] Nothing to do: done.") +// return +// } +// toolMode = .fetchDependencies } +// else { +// guard let input else { +// fatalError("Mode -jar requires path\n\(Self.helpMessage())") +// } +// config = try JavaTranslator.readConfiguration(from: URL(fileURLWithPath: input)) +// toolMode = .classWrappers +// } print("[debug][swift-java] Running swift-java in mode: " + "\(toolMode.prettyName)".bold) let swiftModule: String = - self.swiftModule ?? - self.effectiveSwiftModule.split(separator: "/").dropLast().last.map(String.init) ?? "__UnknownModule" - - // Load all of the dependent configurations and associate them with Swift - // modules. - let dependentConfigs = try dependsOn.map { dependentConfig in - guard let equalLoc = dependentConfig.firstIndex(of: "=") else { - throw JavaToSwiftError.badConfigOption(dependentConfig) - } - - let afterEqual = dependentConfig.index(after: equalLoc) - let swiftModuleName = String(dependentConfig[.. (javaClassName: String, swiftName: String) { @@ -327,8 +297,7 @@ extension JavaToSwiftError: CustomStringConvertible { extension SwiftJava.ToolMode { var prettyName: String { switch self { - case .fetchDependencies: "Fetch dependencies" - case .classWrappers: "Wrap Java classes" +// case .classWrappers: "Wrap Java classes" case .jextract: "JExtract Swift for Java" } } diff --git a/Sources/SwiftJavaTool/SwiftJavaBaseAsyncParsableCommand.swift b/Sources/SwiftJavaTool/SwiftJavaBaseAsyncParsableCommand.swift index afe4f80c..92b3f84c 100644 --- a/Sources/SwiftJavaTool/SwiftJavaBaseAsyncParsableCommand.swift +++ b/Sources/SwiftJavaTool/SwiftJavaBaseAsyncParsableCommand.swift @@ -28,6 +28,7 @@ import JavaKitShared protocol SwiftJavaBaseAsyncParsableCommand: AsyncParsableCommand { var logLevel: Logger.Level { get set } + /// Must be implemented with an `@OptionGroup` in Command implementations var commonOptions: SwiftJava.CommonOptions { get set } var effectiveSwiftModule: String { get } @@ -36,9 +37,15 @@ protocol SwiftJavaBaseAsyncParsableCommand: AsyncParsableCommand { } +extension SwiftJavaBaseAsyncParsableCommand { + var outputDirectory: String? { + self.commonOptions.outputDirectory + } +} + extension SwiftJavaBaseAsyncParsableCommand { public mutating func run() async { - print("[info][swift-java] Run: \(CommandLine.arguments.joined(separator: " "))") + print("[info][swift-java] Run \(Self.self): \(CommandLine.arguments.joined(separator: " "))") print("[info][swift-java] Current work directory: \(URL(fileURLWithPath: "."))") do { @@ -57,22 +64,22 @@ extension SwiftJavaBaseAsyncParsableCommand { } extension SwiftJavaBaseAsyncParsableCommand { - mutating func writeContents( - _ contents: String, - to filename: String, description: String) throws { - try writeContents( - contents, - outputDirectoryOverride: self.actualOutputDirectory, - to: filename, - description: description) - } +// mutating func writeContents( +// _ contents: String, +// to filename: String, description: String) throws { +// try writeContents( +// contents, +// outputDirectoryOverride: self.actualOutputDirectory, +// to: filename, +// description: description) +// } mutating func writeContents( _ contents: String, - outputDirectoryOverride: Foundation.URL?, + outputDirectory: Foundation.URL?, to filename: String, description: String) throws { - guard let outputDir = (outputDirectoryOverride ?? actualOutputDirectory) else { + guard let outputDir = outputDirectory else { print("// \(filename) - \(description)") print(contents) return @@ -90,7 +97,7 @@ extension SwiftJavaBaseAsyncParsableCommand { // Write the file: let file = outputDir.appendingPathComponent(filename) - print("[debug][swift-java] Writing \(description) to '\(file.path)'... ", terminator: "") + print("[trace][swift-java] Writing \(description) to '\(file.path)'... ", terminator: "") try contents.write(to: file, atomically: true, encoding: .utf8) print("done.".green) } @@ -106,22 +113,22 @@ extension SwiftJavaBaseAsyncParsableCommand { self.commonOptions.logLevel = newValue } } + + var effectiveSwiftModuleURL: Foundation.URL { + let fm = FileManager.default + return URL(fileURLWithPath: fm.currentDirectoryPath + "/Sources/\(self.effectiveSwiftModule)") + } } extension SwiftJavaBaseAsyncParsableCommand { var moduleBaseDir: Foundation.URL? { -// if let outputDirectory = commonOptions.outputDirectory { -// if outputDirectory == "-" { -// return nil -// } -// + if let outputDirectory = commonOptions.outputDirectory { + if outputDirectory == "-" { + return nil + } // print("[debug][swift-java] Module base directory based on outputDirectory!") // return URL(fileURLWithPath: outputDirectory) -// } - -// guard let swiftModule else { -// return nil -// } + } // Put the result into Sources/\(swiftModule). let baseDir = URL(fileURLWithPath: ".") @@ -137,7 +144,7 @@ extension SwiftJavaBaseAsyncParsableCommand { /// /// Returns `nil` only when we should emit the files to standard output. var actualOutputDirectory: Foundation.URL? { - if let outputDirectory = commonOptions.outputDirectory { + if let outputDirectory = self.commonOptions.outputDirectory { if outputDirectory == "-" { return nil } diff --git a/Tests/JavaKitConfigurationSharedTests/GradleDependencyParsingTests.swift b/Tests/JavaKitConfigurationSharedTests/GradleDependencyParsingTests.swift new file mode 100644 index 00000000..dbd05f66 --- /dev/null +++ b/Tests/JavaKitConfigurationSharedTests/GradleDependencyParsingTests.swift @@ -0,0 +1,45 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024-2025 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import JavaKitConfigurationShared +import Testing + +@Suite +struct GradleDependencyParsingTests { + + @Test + func parseSingleDependency() throws { + let inputString = "com.example:thing:12.2" + let parsed: JavaDependencyDescriptor = parseDependencyDescriptor(inputString)! + + #expect(parsed.groupID == "com.example") + #expect(parsed.artifactID == "thing") + #expect(parsed.version == "12.2") + } + + @Test + func parseMultiple() throws { + let inputString = "com.example:thing:12.2,com.example:another:1.2.3-beta.1," + let parsed: [JavaDependencyDescriptor] = parseDependencyDescriptors(inputString) + + #expect(parsed.count == 2) + #expect(parsed[0].groupID == "com.example") + #expect(parsed[0].artifactID == "thing") + #expect(parsed[0].version == "12.2") + #expect(parsed[1].groupID == "com.example") + #expect(parsed[1].artifactID == "another") + #expect(parsed[1].version == "1.2.3-beta.1") + } +} + From 8278923986f714c894993f0597cfaac8b1373c36 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Tue, 17 Jun 2025 22:35:42 +0900 Subject: [PATCH 02/37] _Subprocess: Update to a more recent version of vendored lib. We have to keep a vendored lib _for now_ because we need to support 6.0 and upstream library has dropped that. We may eventually do so as well, but currently we cannot require 6.1 yet. --- Package.swift | 9 +- Samples/JavaDependencySampleApp/% | 0 .../Commands/ResolveCommand.swift | 61 +- Sources/_CShims/process_shims.c | 340 ------ Sources/_Subprocess/API.swift | 370 ++++++ Sources/_Subprocess/AsyncBufferSequence.swift | 99 ++ Sources/_Subprocess/Buffer.swift | 104 ++ Sources/_Subprocess/Configuration.swift | 851 +++++++++++++ Sources/_Subprocess/Error.swift | 141 +++ Sources/_Subprocess/Execution.swift | 192 +++ Sources/_Subprocess/IO/Input.swift | 315 +++++ Sources/_Subprocess/IO/Output.swift | 298 +++++ Sources/_Subprocess/LockedState.swift | 160 --- .../Platforms/Subprocess+Darwin.swift | 625 +++++----- .../Platforms/Subprocess+Linux.swift | 398 +++--- .../Platforms/Subprocess+Unix.swift | 522 ++++---- .../Platforms/Subprocess+Windows.swift | 1088 +++++++++-------- Sources/_Subprocess/Result.swift | 123 ++ Sources/_Subprocess/Subprocess+API.swift | 465 ------- .../Subprocess+AsyncDataSequence.swift | 86 -- .../Subprocess+Configuration.swift | 769 ------------ Sources/_Subprocess/Subprocess+IO.swift | 437 ------- Sources/_Subprocess/Subprocess+Teardown.swift | 125 -- Sources/_Subprocess/Subprocess.swift | 315 ----- .../Input+Foundation.swift | 179 +++ .../Output+Foundation.swift | 53 + .../Span+SubprocessFoundation.swift | 76 ++ Sources/_Subprocess/Teardown.swift | 217 ++++ Sources/_Subprocess/_nio_locks.swift | 526 -------- .../include/process_shims.h | 30 +- .../include/target_conditionals.h} | 10 +- Sources/_SubprocessCShims/process_shims.c | 682 +++++++++++ 32 files changed, 5172 insertions(+), 4494 deletions(-) delete mode 100644 Samples/JavaDependencySampleApp/% delete mode 100644 Sources/_CShims/process_shims.c create mode 100644 Sources/_Subprocess/API.swift create mode 100644 Sources/_Subprocess/AsyncBufferSequence.swift create mode 100644 Sources/_Subprocess/Buffer.swift create mode 100644 Sources/_Subprocess/Configuration.swift create mode 100644 Sources/_Subprocess/Error.swift create mode 100644 Sources/_Subprocess/Execution.swift create mode 100644 Sources/_Subprocess/IO/Input.swift create mode 100644 Sources/_Subprocess/IO/Output.swift delete mode 100644 Sources/_Subprocess/LockedState.swift create mode 100644 Sources/_Subprocess/Result.swift delete mode 100644 Sources/_Subprocess/Subprocess+API.swift delete mode 100644 Sources/_Subprocess/Subprocess+AsyncDataSequence.swift delete mode 100644 Sources/_Subprocess/Subprocess+Configuration.swift delete mode 100644 Sources/_Subprocess/Subprocess+IO.swift delete mode 100644 Sources/_Subprocess/Subprocess+Teardown.swift delete mode 100644 Sources/_Subprocess/Subprocess.swift create mode 100644 Sources/_Subprocess/SubprocessFoundation/Input+Foundation.swift create mode 100644 Sources/_Subprocess/SubprocessFoundation/Output+Foundation.swift create mode 100644 Sources/_Subprocess/SubprocessFoundation/Span+SubprocessFoundation.swift create mode 100644 Sources/_Subprocess/Teardown.swift delete mode 100644 Sources/_Subprocess/_nio_locks.swift rename Sources/{_CShims => _SubprocessCShims}/include/process_shims.h (74%) rename Sources/{_CShims/include/_CShimsTargetConditionals.h => _SubprocessCShims/include/target_conditionals.h} (76%) create mode 100644 Sources/_SubprocessCShims/process_shims.c diff --git a/Package.swift b/Package.swift index 4cefd08e..80879ca3 100644 --- a/Package.swift +++ b/Package.swift @@ -195,6 +195,10 @@ let package = Package( .package(url: "https://github.com/apple/swift-argument-parser", from: "1.5.0"), .package(url: "https://github.com/apple/swift-system", from: "1.4.0"), +// // FIXME: swift-subprocess stopped supporting 6.0 when it moved into a package; +// // we'll need to drop 6.0 as well, but currently blocked on doing so by swiftpm plugin pending design questions +// .package(url: "https://github.com/swiftlang/swift-subprocess.git", revision: "de15b67f7871c8a039ef7f4813eb39a8878f61a6"), + // Benchmarking .package(url: "https://github.com/ordo-one/package-benchmark", .upToNextMajor(from: "1.4.0")), ], @@ -363,6 +367,7 @@ let package = Package( "JavaTypes", "JavaKitShared", "JavaKitConfigurationShared", + // .product(name: "Subprocess", package: "swift-subprocess") "_Subprocess", // using process spawning ], swiftSettings: [ @@ -489,7 +494,7 @@ let package = Package( // Experimental Foundation Subprocess Copy .target( - name: "_CShims", + name: "_SubprocessCShims", swiftSettings: [ .swiftLanguageMode(.v5) ] @@ -497,7 +502,7 @@ let package = Package( .target( name: "_Subprocess", dependencies: [ - "_CShims", + "_SubprocessCShims", .product(name: "SystemPackage", package: "swift-system"), ], swiftSettings: [ diff --git a/Samples/JavaDependencySampleApp/% b/Samples/JavaDependencySampleApp/% deleted file mode 100644 index e69de29b..00000000 diff --git a/Sources/SwiftJavaTool/Commands/ResolveCommand.swift b/Sources/SwiftJavaTool/Commands/ResolveCommand.swift index dafa2520..66f0eec4 100644 --- a/Sources/SwiftJavaTool/Commands/ResolveCommand.swift +++ b/Sources/SwiftJavaTool/Commands/ResolveCommand.swift @@ -22,6 +22,9 @@ import SwiftJavaLib import JavaKitConfigurationShared import JavaKitShared import _Subprocess +import System + +typealias Configuration = JavaKitConfigurationShared.Configuration extension SwiftJava { struct ResolveCommand: SwiftJavaBaseAsyncParsableCommand, HasCommonOptions, HasCommonJVMOptions { @@ -137,37 +140,39 @@ extension SwiftJava.ResolveCommand { try! printGradleProject(directory: resolverDir, dependencies: dependencies) - let process = try! await Subprocess.run( - .at(.init(resolverDir.appendingPathComponent("gradlew").path)), - arguments: [ - "--no-daemon", - "--rerun-tasks", - "\(printRuntimeClasspathTaskName)", - ], - workingDirectory: .init(platformString: resolverDir.path) - ) - - let outString = String( - data: process.standardOutput, - encoding: .utf8 - ) - let errString = String( - data: process.standardError, - encoding: .utf8 - ) + if #available(macOS 15, *) { + let process = try! await _Subprocess.run( + .path(FilePath(resolverDir.appendingPathComponent("gradlew").path)), + arguments: [ + "--no-daemon", + "--rerun-tasks", + "\(printRuntimeClasspathTaskName)", + ], + workingDirectory: FilePath(resolverDir.path), + // TODO: we could move to stream processing the outputs + output: .string(limit: Int.max, encoding: UTF8.self), // Don't limit output, we know it will be reasonable size + error: .string(limit: Int.max, encoding: UTF8.self) // Don't limit output, we know it will be reasonable size + ) + + let outString = process.standardOutput ?? "" + let errString = process.standardError ?? "" + + let classpathOutput: String + if let found = outString.split(separator: "\n").first(where: { $0.hasPrefix(self.SwiftJavaClasspathPrefix) }) { + classpathOutput = String(found) + } else if let found = errString.split(separator: "\n").first(where: { $0.hasPrefix(self.SwiftJavaClasspathPrefix) }) { + classpathOutput = String(found) + } else { + let suggestDisablingSandbox = "It may be that the Sandbox has prevented dependency fetching, please re-run with '--disable-sandbox'." + fatalError("Gradle output had no SWIFT_JAVA_CLASSPATH! \(suggestDisablingSandbox). \n" + + "Output was:<<<\(outString ?? "")>>>; Err was:<<<\(errString ?? "")>>>") + } - let classpathOutput: String - if let found = outString?.split(separator: "\n").first(where: { $0.hasPrefix(self.SwiftJavaClasspathPrefix) }) { - classpathOutput = String(found) - } else if let found = errString?.split(separator: "\n").first(where: { $0.hasPrefix(self.SwiftJavaClasspathPrefix) }) { - classpathOutput = String(found) + return String(classpathOutput.dropFirst(SwiftJavaClasspathPrefix.count)) } else { - let suggestDisablingSandbox = "It may be that the Sandbox has prevented dependency fetching, please re-run with '--disable-sandbox'." - fatalError("Gradle output had no SWIFT_JAVA_CLASSPATH! \(suggestDisablingSandbox). \n" + - "Output was:<<<\(outString ?? "")>>>; Err was:<<<\(errString ?? "")>>>") + // Subprocess is unavailable + fatalError("Subprocess is unavailable yet required to execute `gradlew` subprocess. Please update to macOS 15+") } - - return String(classpathOutput.dropFirst(SwiftJavaClasspathPrefix.count)) } func printGradleProject(directory: URL, dependencies: [JavaDependencyDescriptor]) throws { diff --git a/Sources/_CShims/process_shims.c b/Sources/_CShims/process_shims.c deleted file mode 100644 index fe96c675..00000000 --- a/Sources/_CShims/process_shims.c +++ /dev/null @@ -1,340 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift.org project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -#include "include/_CShimsTargetConditionals.h" -#include "include/process_shims.h" - -#if !TARGET_OS_WINDOWS -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -int _was_process_exited(int status) { - return WIFEXITED(status); -} - -int _get_exit_code(int status) { - return WEXITSTATUS(status); -} - -int _was_process_signaled(int status) { - return WIFSIGNALED(status); -} - -int _get_signal_code(int status) { - return WTERMSIG(status); -} - -int _was_process_suspended(int status) { - return WIFSTOPPED(status); -} - -#if TARGET_OS_LINUX -#include - -int _shims_snprintf( - char * _Nonnull str, - int len, - const char * _Nonnull format, - char * _Nonnull str1, - char * _Nonnull str2 -) { - return snprintf(str, len, format, str1, str2); -} -#endif - -// MARK: - Darwin (posix_spawn) -#if TARGET_OS_MAC - -int _subprocess_spawn( - pid_t * _Nonnull pid, - const char * _Nonnull exec_path, - const posix_spawn_file_actions_t _Nullable * _Nonnull file_actions, - const posix_spawnattr_t _Nullable * _Nonnull spawn_attrs, - char * _Nullable const args[_Nonnull], - char * _Nullable const env[_Nullable], - uid_t * _Nullable uid, - gid_t * _Nullable gid, - int number_of_sgroups, const gid_t * _Nullable sgroups, - int create_session -) { - int require_pre_fork = uid != NULL || - gid != NULL || - number_of_sgroups > 0 || - create_session > 0; - - if (require_pre_fork != 0) { -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated" - pid_t childPid = vfork(); -#pragma GCC diagnostic pop - if (childPid != 0) { - *pid = childPid; - return childPid < 0 ? errno : 0; - } - - if (number_of_sgroups > 0 && sgroups != NULL) { - if (setgroups(number_of_sgroups, sgroups) != 0) { - return errno; - } - } - - if (uid != NULL) { - if (setuid(*uid) != 0) { - return errno; - } - } - - if (gid != NULL) { - if (setgid(*gid) != 0) { - return errno; - } - } - - if (create_session != 0) { - (void)setsid(); - } - } - - // Set POSIX_SPAWN_SETEXEC if we already forked - if (require_pre_fork) { - short flags = 0; - int rc = posix_spawnattr_getflags(spawn_attrs, &flags); - if (rc != 0) { - return rc; - } - - rc = posix_spawnattr_setflags( - (posix_spawnattr_t *)spawn_attrs, flags | POSIX_SPAWN_SETEXEC - ); - if (rc != 0) { - return rc; - } - } - - // Spawn - return posix_spawn(pid, exec_path, file_actions, spawn_attrs, args, env); -} - -#endif // TARGET_OS_MAC - -// MARK: - Linux (fork/exec + posix_spawn fallback) - -#if _POSIX_SPAWN -static int _subprocess_posix_spawn_fallback( - pid_t * _Nonnull pid, - const char * _Nonnull exec_path, - const char * _Nullable working_directory, - const int file_descriptors[_Nonnull], - char * _Nullable const args[_Nonnull], - char * _Nullable const env[_Nullable], - gid_t * _Nullable process_group_id -) { - // Setup stdin, stdout, and stderr - posix_spawn_file_actions_t file_actions; - - int rc = posix_spawn_file_actions_init(&file_actions); - if (rc != 0) { return rc; } - if (file_descriptors[0] >= 0) { - rc = posix_spawn_file_actions_adddup2( - &file_actions, file_descriptors[0], STDIN_FILENO - ); - if (rc != 0) { return rc; } - } - if (file_descriptors[2] >= 0) { - rc = posix_spawn_file_actions_adddup2( - &file_actions, file_descriptors[2], STDOUT_FILENO - ); - if (rc != 0) { return rc; } - } - if (file_descriptors[4] >= 0) { - rc = posix_spawn_file_actions_adddup2( - &file_actions, file_descriptors[4], STDERR_FILENO - ); - if (rc != 0) { return rc; } - } - - // Close parent side - if (file_descriptors[1] >= 0) { - rc = posix_spawn_file_actions_addclose(&file_actions, file_descriptors[1]); - if (rc != 0) { return rc; } - } - if (file_descriptors[3] >= 0) { - rc = posix_spawn_file_actions_addclose(&file_actions, file_descriptors[3]); - if (rc != 0) { return rc; } - } - if (file_descriptors[5] >= 0) { - rc = posix_spawn_file_actions_addclose(&file_actions, file_descriptors[5]); - if (rc != 0) { return rc; } - } - - // Setup spawnattr - posix_spawnattr_t spawn_attr; - rc = posix_spawnattr_init(&spawn_attr); - if (rc != 0) { return rc; } - // Masks - sigset_t no_signals; - sigset_t all_signals; - sigemptyset(&no_signals); - sigfillset(&all_signals); - rc = posix_spawnattr_setsigmask(&spawn_attr, &no_signals); - if (rc != 0) { return rc; } - rc = posix_spawnattr_setsigdefault(&spawn_attr, &all_signals); - if (rc != 0) { return rc; } - // Flags - short flags = POSIX_SPAWN_SETSIGMASK | POSIX_SPAWN_SETSIGDEF; - if (process_group_id != NULL) { - flags |= POSIX_SPAWN_SETPGROUP; - rc = posix_spawnattr_setpgroup(&spawn_attr, *process_group_id); - if (rc != 0) { return rc; } - } - rc = posix_spawnattr_setflags(&spawn_attr, flags); - - // Spawn! - rc = posix_spawn( - pid, exec_path, - &file_actions, &spawn_attr, - args, env - ); - posix_spawn_file_actions_destroy(&file_actions); - posix_spawnattr_destroy(&spawn_attr); - return rc; -} -#endif // _POSIX_SPAWN - -int _subprocess_fork_exec( - pid_t * _Nonnull pid, - const char * _Nonnull exec_path, - const char * _Nullable working_directory, - const int file_descriptors[_Nonnull], - char * _Nullable const args[_Nonnull], - char * _Nullable const env[_Nullable], - uid_t * _Nullable uid, - gid_t * _Nullable gid, - gid_t * _Nullable process_group_id, - int number_of_sgroups, const gid_t * _Nullable sgroups, - int create_session, - void (* _Nullable configurator)(void) -) { - int require_pre_fork = working_directory != NULL || - uid != NULL || - gid != NULL || - process_group_id != NULL || - (number_of_sgroups > 0 && sgroups != NULL) || - create_session || - configurator != NULL; - -#if _POSIX_SPAWN - // If posix_spawn is available on this platform and - // we do not require prefork, use posix_spawn if possible. - // - // (Glibc's posix_spawn does not support - // `POSIX_SPAWN_SETEXEC` therefore we have to keep - // using fork/exec if `require_pre_fork` is true. - if (require_pre_fork == 0) { - return _subprocess_posix_spawn_fallback( - pid, exec_path, - working_directory, - file_descriptors, - args, env, - process_group_id - ); - } -#endif - - pid_t child_pid = fork(); - if (child_pid != 0) { - *pid = child_pid; - return child_pid < 0 ? errno : 0; - } - - if (working_directory != NULL) { - if (chdir(working_directory) != 0) { - return errno; - } - } - - - if (uid != NULL) { - if (setuid(*uid) != 0) { - return errno; - } - } - - if (gid != NULL) { - if (setgid(*gid) != 0) { - return errno; - } - } - - if (number_of_sgroups > 0 && sgroups != NULL) { - if (setgroups(number_of_sgroups, sgroups) != 0) { - return errno; - } - } - - if (create_session != 0) { - (void)setsid(); - } - - if (process_group_id != NULL) { - (void)setpgid(0, *process_group_id); - } - - // Bind stdin, stdout, and stderr - int rc = 0; - if (file_descriptors[0] >= 0) { - rc = dup2(file_descriptors[0], STDIN_FILENO); - if (rc < 0) { return errno; } - } - if (file_descriptors[2] >= 0) { - rc = dup2(file_descriptors[2], STDOUT_FILENO); - if (rc < 0) { return errno; } - } - if (file_descriptors[4] >= 0) { - rc = dup2(file_descriptors[4], STDERR_FILENO); - if (rc < 0) { return errno; } - } - // Close parent side - if (file_descriptors[1] >= 0) { - rc = close(file_descriptors[1]); - } - if (file_descriptors[3] >= 0) { - rc = close(file_descriptors[3]); - } - if (file_descriptors[4] >= 0) { - rc = close(file_descriptors[5]); - } - if (rc != 0) { - return errno; - } - // Run custom configuratior - if (configurator != NULL) { - configurator(); - } - // Finally, exec - execve(exec_path, args, env); - // If we got here, something went wrong - return errno; -} - -#endif // !TARGET_OS_WINDOWS - diff --git a/Sources/_Subprocess/API.swift b/Sources/_Subprocess/API.swift new file mode 100644 index 00000000..9673d3e1 --- /dev/null +++ b/Sources/_Subprocess/API.swift @@ -0,0 +1,370 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +#if canImport(System) +import System +#else +@preconcurrency import SystemPackage +#endif + +// MARK: - Collected Result + +/// Run a executable with given parameters asynchrously and returns +/// a `CollectedResult` containing the output of the child process. +/// - Parameters: +/// - executable: The executable to run. +/// - arguments: The arguments to pass to the executable. +/// - environment: The environment in which to run the executable. +/// - workingDirectory: The working directory in which to run the executable. +/// - platformOptions: The platform specific options to use +/// when running the executable. +/// - input: The input to send to the executable. +/// - output: The method to use for redirecting the standard output. +/// - error: The method to use for redirecting the standard error. +/// - Returns a CollectedResult containing the result of the run. +@available(macOS 15.0, *) // FIXME: manually added availability +public func run< + Input: InputProtocol, + Output: OutputProtocol, + Error: OutputProtocol +>( + _ executable: Executable, + arguments: Arguments = [], + environment: Environment = .inherit, + workingDirectory: FilePath? = nil, + platformOptions: PlatformOptions = PlatformOptions(), + input: Input = .none, + output: Output = .string, + error: Error = .discarded +) async throws -> CollectedResult { + let configuration = Configuration( + executable: executable, + arguments: arguments, + environment: environment, + workingDirectory: workingDirectory, + platformOptions: platformOptions + ) + return try await run( + configuration, + input: input, + output: output, + error: error + ) +} + +// MARK: - Custom Execution Body + +/// Run a executable with given parameters and a custom closure +/// to manage the running subprocess' lifetime and its IOs. +/// - Parameters: +/// - executable: The executable to run. +/// - arguments: The arguments to pass to the executable. +/// - environment: The environment in which to run the executable. +/// - workingDirectory: The working directory in which to run the executable. +/// - platformOptions: The platform specific options to use +/// when running the executable. +/// - input: The input to send to the executable. +/// - output: How to manage the executable standard ouput. +/// - error: How to manager executable standard error. +/// - isolation: the isolation context to run the body closure. +/// - body: The custom execution body to manually control the running process +/// - Returns a ExecutableResult type containing the return value +/// of the closure. +@available(macOS 15.0, *) // FIXME: manually added availability +public func run( + _ executable: Executable, + arguments: Arguments = [], + environment: Environment = .inherit, + workingDirectory: FilePath? = nil, + platformOptions: PlatformOptions = PlatformOptions(), + input: Input = .none, + output: Output, + error: Error, + isolation: isolated (any Actor)? = #isolation, + body: ((Execution) async throws -> Result) +) async throws -> ExecutionResult where Output.OutputType == Void, Error.OutputType == Void { + return try await Configuration( + executable: executable, + arguments: arguments, + environment: environment, + workingDirectory: workingDirectory, + platformOptions: platformOptions + ) + .run(input: input, output: output, error: error, body) +} + +/// Run a executable with given parameters and a custom closure +/// to manage the running subprocess' lifetime and write to its +/// standard input via `StandardInputWriter` +/// - Parameters: +/// - executable: The executable to run. +/// - arguments: The arguments to pass to the executable. +/// - environment: The environment in which to run the executable. +/// - workingDirectory: The working directory in which to run the executable. +/// - platformOptions: The platform specific options to use +/// when running the executable. +/// - output:How to handle executable's standard output +/// - error: How to handle executable's standard error +/// - isolation: the isolation context to run the body closure. +/// - body: The custom execution body to manually control the running process +/// - Returns a ExecutableResult type containing the return value +/// of the closure. +@available(macOS 15.0, *) // FIXME: manually added availability +public func run( + _ executable: Executable, + arguments: Arguments = [], + environment: Environment = .inherit, + workingDirectory: FilePath? = nil, + platformOptions: PlatformOptions = PlatformOptions(), + output: Output, + error: Error, + isolation: isolated (any Actor)? = #isolation, + body: ((Execution, StandardInputWriter) async throws -> Result) +) async throws -> ExecutionResult where Output.OutputType == Void, Error.OutputType == Void { + return try await Configuration( + executable: executable, + arguments: arguments, + environment: environment, + workingDirectory: workingDirectory, + platformOptions: platformOptions + ) + .run(output: output, error: error, body) +} + +// MARK: - Configuration Based + +/// Run a `Configuration` asynchrously and returns +/// a `CollectedResult` containing the output of the child process. +/// - Parameters: +/// - configuration: The `Subprocess` configuration to run. +/// - input: The input to send to the executable. +/// - output: The method to use for redirecting the standard output. +/// - error: The method to use for redirecting the standard error. +/// - Returns a CollectedResult containing the result of the run. +@available(macOS 15.0, *) // FIXME: manually added availability +public func run< + Input: InputProtocol, + Output: OutputProtocol, + Error: OutputProtocol +>( + _ configuration: Configuration, + input: Input = .none, + output: Output = .string, + error: Error = .discarded +) async throws -> CollectedResult { + let result = try await configuration.run( + input: input, + output: output, + error: error + ) { execution in + let ( + standardOutput, + standardError + ) = try await execution.captureIOs() + return ( + processIdentifier: execution.processIdentifier, + standardOutput: standardOutput, + standardError: standardError + ) + } + return CollectedResult( + processIdentifier: result.value.processIdentifier, + terminationStatus: result.terminationStatus, + standardOutput: result.value.standardOutput, + standardError: result.value.standardError + ) +} + +/// Run a executable with given parameters specified by a `Configuration` +/// - Parameters: +/// - configuration: The `Subprocess` configuration to run. +/// - output: The method to use for redirecting the standard output. +/// - error: The method to use for redirecting the standard error. +/// - isolation: the isolation context to run the body closure. +/// - body: The custom configuration body to manually control +/// the running process and write to its standard input. +/// - Returns a ExecutableResult type containing the return value +/// of the closure. +@available(macOS 15.0, *) // FIXME: manually added availability +public func run( + _ configuration: Configuration, + output: Output, + error: Error, + isolation: isolated (any Actor)? = #isolation, + body: ((Execution, StandardInputWriter) async throws -> Result) +) async throws -> ExecutionResult where Output.OutputType == Void, Error.OutputType == Void { + return try await configuration.run(output: output, error: error, body) +} + +// MARK: - Detached + +/// Run a executable with given parameters and return its process +/// identifier immediately without monitoring the state of the +/// subprocess nor waiting until it exits. +/// +/// This method is useful for launching subprocesses that outlive their +/// parents (for example, daemons and trampolines). +/// +/// - Parameters: +/// - executable: The executable to run. +/// - arguments: The arguments to pass to the executable. +/// - environment: The environment to use for the process. +/// - workingDirectory: The working directory for the process. +/// - platformOptions: The platform specific options to use for the process. +/// - input: A file descriptor to bind to the subprocess' standard input. +/// - output: A file descriptor to bind to the subprocess' standard output. +/// - error: A file descriptor to bind to the subprocess' standard error. +/// - Returns: the process identifier for the subprocess. +@available(macOS 15.0, *) // FIXME: manually added availability +public func runDetached( + _ executable: Executable, + arguments: Arguments = [], + environment: Environment = .inherit, + workingDirectory: FilePath? = nil, + platformOptions: PlatformOptions = PlatformOptions(), + input: FileDescriptor? = nil, + output: FileDescriptor? = nil, + error: FileDescriptor? = nil +) throws -> ProcessIdentifier { + let config: Configuration = Configuration( + executable: executable, + arguments: arguments, + environment: environment, + workingDirectory: workingDirectory, + platformOptions: platformOptions + ) + return try runDetached(config, input: input, output: output, error: error) +} + +/// Run a executable with given configuration and return its process +/// identifier immediately without monitoring the state of the +/// subprocess nor waiting until it exits. +/// +/// This method is useful for launching subprocesses that outlive their +/// parents (for example, daemons and trampolines). +/// +/// - Parameters: +/// - configuration: The `Subprocess` configuration to run. +/// - input: A file descriptor to bind to the subprocess' standard input. +/// - output: A file descriptor to bind to the subprocess' standard output. +/// - error: A file descriptor to bind to the subprocess' standard error. +/// - Returns: the process identifier for the subprocess. +@available(macOS 15.0, *) // FIXME: manually added availability +public func runDetached( + _ configuration: Configuration, + input: FileDescriptor? = nil, + output: FileDescriptor? = nil, + error: FileDescriptor? = nil +) throws -> ProcessIdentifier { + switch (input, output, error) { + case (.none, .none, .none): + let processOutput = DiscardedOutput() + let processError = DiscardedOutput() + return try configuration.spawn( + withInput: NoInput().createPipe(), + output: processOutput, + outputPipe: try processOutput.createPipe(), + error: processError, + errorPipe: try processError.createPipe() + ).processIdentifier + case (.none, .none, .some(let errorFd)): + let processOutput = DiscardedOutput() + let processError = FileDescriptorOutput(fileDescriptor: errorFd, closeAfterSpawningProcess: false) + return try configuration.spawn( + withInput: NoInput().createPipe(), + output: processOutput, + outputPipe: try processOutput.createPipe(), + error: processError, + errorPipe: try processError.createPipe() + ).processIdentifier + case (.none, .some(let outputFd), .none): + let processOutput = FileDescriptorOutput(fileDescriptor: outputFd, closeAfterSpawningProcess: false) + let processError = DiscardedOutput() + return try configuration.spawn( + withInput: NoInput().createPipe(), + output: processOutput, + outputPipe: try processOutput.createPipe(), + error: processError, + errorPipe: try processError.createPipe() + ).processIdentifier + case (.none, .some(let outputFd), .some(let errorFd)): + let processOutput = FileDescriptorOutput( + fileDescriptor: outputFd, + closeAfterSpawningProcess: false + ) + let processError = FileDescriptorOutput( + fileDescriptor: errorFd, + closeAfterSpawningProcess: false + ) + return try configuration.spawn( + withInput: NoInput().createPipe(), + output: processOutput, + outputPipe: try processOutput.createPipe(), + error: processError, + errorPipe: try processError.createPipe() + ).processIdentifier + case (.some(let inputFd), .none, .none): + let processOutput = DiscardedOutput() + let processError = DiscardedOutput() + return try configuration.spawn( + withInput: FileDescriptorInput( + fileDescriptor: inputFd, + closeAfterSpawningProcess: false + ).createPipe(), + output: processOutput, + outputPipe: try processOutput.createPipe(), + error: processError, + errorPipe: try processError.createPipe() + ).processIdentifier + case (.some(let inputFd), .none, .some(let errorFd)): + let processOutput = DiscardedOutput() + let processError = FileDescriptorOutput( + fileDescriptor: errorFd, + closeAfterSpawningProcess: false + ) + return try configuration.spawn( + withInput: FileDescriptorInput(fileDescriptor: inputFd, closeAfterSpawningProcess: false).createPipe(), + output: processOutput, + outputPipe: try processOutput.createPipe(), + error: processError, + errorPipe: try processError.createPipe() + ).processIdentifier + case (.some(let inputFd), .some(let outputFd), .none): + let processOutput = FileDescriptorOutput( + fileDescriptor: outputFd, + closeAfterSpawningProcess: false + ) + let processError = DiscardedOutput() + return try configuration.spawn( + withInput: FileDescriptorInput(fileDescriptor: inputFd, closeAfterSpawningProcess: false).createPipe(), + output: processOutput, + outputPipe: try processOutput.createPipe(), + error: processError, + errorPipe: try processError.createPipe() + ).processIdentifier + case (.some(let inputFd), .some(let outputFd), .some(let errorFd)): + let processOutput = FileDescriptorOutput( + fileDescriptor: outputFd, + closeAfterSpawningProcess: false + ) + let processError = FileDescriptorOutput( + fileDescriptor: errorFd, + closeAfterSpawningProcess: false + ) + return try configuration.spawn( + withInput: FileDescriptorInput(fileDescriptor: inputFd, closeAfterSpawningProcess: false).createPipe(), + output: processOutput, + outputPipe: try processOutput.createPipe(), + error: processError, + errorPipe: try processError.createPipe() + ).processIdentifier + } +} diff --git a/Sources/_Subprocess/AsyncBufferSequence.swift b/Sources/_Subprocess/AsyncBufferSequence.swift new file mode 100644 index 00000000..4316f7eb --- /dev/null +++ b/Sources/_Subprocess/AsyncBufferSequence.swift @@ -0,0 +1,99 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +#if canImport(System) +import System +#else +@preconcurrency import SystemPackage +#endif + +internal struct AsyncBufferSequence: AsyncSequence, Sendable { + internal typealias Failure = any Swift.Error + + internal typealias Element = SequenceOutput.Buffer + + @_nonSendable + internal struct Iterator: AsyncIteratorProtocol { + internal typealias Element = SequenceOutput.Buffer + + private let fileDescriptor: TrackedFileDescriptor + private var buffer: [UInt8] + private var currentPosition: Int + private var finished: Bool + + internal init(fileDescriptor: TrackedFileDescriptor) { + self.fileDescriptor = fileDescriptor + self.buffer = [] + self.currentPosition = 0 + self.finished = false + } + + internal mutating func next() async throws -> SequenceOutput.Buffer? { + let data = try await self.fileDescriptor.wrapped.readChunk( + upToLength: readBufferSize + ) + if data == nil { + // We finished reading. Close the file descriptor now + try self.fileDescriptor.safelyClose() + return nil + } + return data + } + } + + private let fileDescriptor: TrackedFileDescriptor + + init(fileDescriptor: TrackedFileDescriptor) { + self.fileDescriptor = fileDescriptor + } + + internal func makeAsyncIterator() -> Iterator { + return Iterator(fileDescriptor: self.fileDescriptor) + } +} + +// MARK: - Page Size +import _SubprocessCShims + +#if canImport(Darwin) +import Darwin +internal import MachO.dyld + +private let _pageSize: Int = { + Int(_subprocess_vm_size()) +}() +#elseif canImport(WinSDK) +import WinSDK +private let _pageSize: Int = { + var sysInfo: SYSTEM_INFO = SYSTEM_INFO() + GetSystemInfo(&sysInfo) + return Int(sysInfo.dwPageSize) +}() +#elseif os(WASI) +// WebAssembly defines a fixed page size +private let _pageSize: Int = 65_536 +#elseif canImport(Android) +@preconcurrency import Android +private let _pageSize: Int = Int(getpagesize()) +#elseif canImport(Glibc) +@preconcurrency import Glibc +private let _pageSize: Int = Int(getpagesize()) +#elseif canImport(Musl) +@preconcurrency import Musl +private let _pageSize: Int = Int(getpagesize()) +#elseif canImport(C) +private let _pageSize: Int = Int(getpagesize()) +#endif // canImport(Darwin) + +@inline(__always) +internal var readBufferSize: Int { + return _pageSize +} diff --git a/Sources/_Subprocess/Buffer.swift b/Sources/_Subprocess/Buffer.swift new file mode 100644 index 00000000..3ce73d7e --- /dev/null +++ b/Sources/_Subprocess/Buffer.swift @@ -0,0 +1,104 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +@preconcurrency internal import Dispatch + +extension SequenceOutput { + /// A immutable collection of bytes + public struct Buffer: Sendable { + #if os(Windows) + private var data: [UInt8] + + internal init(data: [UInt8]) { + self.data = data + } + #else + private var data: DispatchData + + internal init(data: DispatchData) { + self.data = data + } + #endif + } +} + +// MARK: - Properties +extension SequenceOutput.Buffer { + /// Number of bytes stored in the buffer + public var count: Int { + return self.data.count + } + + /// A Boolean value indicating whether the collection is empty. + public var isEmpty: Bool { + return self.data.isEmpty + } +} + +// MARK: - Accessors +extension SequenceOutput.Buffer { + #if !SubprocessSpan + /// Access the raw bytes stored in this buffer + /// - Parameter body: A closure with an `UnsafeRawBufferPointer` parameter that + /// points to the contiguous storage for the type. If no such storage exists, + /// the method creates it. If body has a return value, this method also returns + /// that value. The argument is valid only for the duration of the + /// closure’s SequenceOutput. + /// - Returns: The return value, if any, of the body closure parameter. + public func withUnsafeBytes( + _ body: (UnsafeRawBufferPointer) throws -> ResultType + ) rethrows -> ResultType { + return try self._withUnsafeBytes(body) + } + #endif // !SubprocessSpan + + internal func _withUnsafeBytes( + _ body: (UnsafeRawBufferPointer) throws -> ResultType + ) rethrows -> ResultType { + #if os(Windows) + return try self.data.withUnsafeBytes(body) + #else + // Although DispatchData was designed to be uncontiguous, in practice + // we found that almost all DispatchData are contiguous. Therefore + // we can access this body in O(1) most of the time. + return try self.data.withUnsafeBytes { ptr in + let bytes = UnsafeRawBufferPointer(start: ptr, count: self.data.count) + return try body(bytes) + } + #endif + } + + private enum SpanBacking { + case pointer(UnsafeBufferPointer) + case array([UInt8]) + } +} + +// MARK: - Hashable, Equatable +extension SequenceOutput.Buffer: Equatable, Hashable { + #if os(Windows) + // Compiler generated conformances + #else + public static func == (lhs: SequenceOutput.Buffer, rhs: SequenceOutput.Buffer) -> Bool { + return lhs.data.elementsEqual(rhs.data) + } + + public func hash(into hasher: inout Hasher) { + self.data.withUnsafeBytes { ptr in + let bytes = UnsafeRawBufferPointer( + start: ptr, + count: self.data.count + ) + hasher.combine(bytes: bytes) + } + } + #endif +} diff --git a/Sources/_Subprocess/Configuration.swift b/Sources/_Subprocess/Configuration.swift new file mode 100644 index 00000000..4dad1a47 --- /dev/null +++ b/Sources/_Subprocess/Configuration.swift @@ -0,0 +1,851 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +#if canImport(System) +import System +#else +@preconcurrency import SystemPackage +#endif + +#if canImport(Darwin) +import Darwin +#elseif canImport(Bionic) +import Bionic +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#elseif canImport(WinSDK) +import WinSDK +#endif + +internal import Dispatch + +/// A collection of configurations parameters to use when +/// spawning a subprocess. +public struct Configuration: Sendable { + /// The executable to run. + public var executable: Executable + /// The arguments to pass to the executable. + public var arguments: Arguments + /// The environment to use when running the executable. + public var environment: Environment + /// The working directory to use when running the executable. + public var workingDirectory: FilePath + /// The platform specifc options to use when + /// running the subprocess. + public var platformOptions: PlatformOptions + + public init( + executable: Executable, + arguments: Arguments = [], + environment: Environment = .inherit, + workingDirectory: FilePath? = nil, + platformOptions: PlatformOptions = PlatformOptions() + ) { + self.executable = executable + self.arguments = arguments + self.environment = environment + self.workingDirectory = workingDirectory ?? .currentWorkingDirectory + self.platformOptions = platformOptions + } + + @available(macOS 15.0, *) // FIXME: manually added availability + internal func run< + Result, + Output: OutputProtocol, + Error: OutputProtocol + >( + output: Output, + error: Error, + isolation: isolated (any Actor)? = #isolation, + _ body: ( + Execution, + StandardInputWriter + ) async throws -> Result + ) async throws -> ExecutionResult { + let input = CustomWriteInput() + + let inputPipe = try input.createPipe() + let outputPipe = try output.createPipe() + let errorPipe = try error.createPipe() + + let execution = try self.spawn( + withInput: inputPipe, + output: output, + outputPipe: outputPipe, + error: error, + errorPipe: errorPipe + ) + // After spawn, cleanup child side fds + try await self.cleanup( + execution: execution, + inputPipe: inputPipe, + outputPipe: outputPipe, + errorPipe: errorPipe, + childSide: true, + parentSide: false, + attemptToTerminateSubProcess: false + ) + return try await withAsyncTaskCleanupHandler { + async let waitingStatus = try await monitorProcessTermination( + forProcessWithIdentifier: execution.processIdentifier + ) + // Body runs in the same isolation + let result = try await body( + execution, + .init(fileDescriptor: inputPipe.writeFileDescriptor!) + ) + return ExecutionResult( + terminationStatus: try await waitingStatus, + value: result + ) + } onCleanup: { + // Attempt to terminate the child process + // Since the task has already been cancelled, + // this is the best we can do + try? await self.cleanup( + execution: execution, + inputPipe: inputPipe, + outputPipe: outputPipe, + errorPipe: errorPipe, + childSide: false, + parentSide: true, + attemptToTerminateSubProcess: true + ) + } + } + + @available(macOS 15.0, *) // FIXME: manually added availability + internal func run< + Result, + Input: InputProtocol, + Output: OutputProtocol, + Error: OutputProtocol + >( + input: Input, + output: Output, + error: Error, + isolation: isolated (any Actor)? = #isolation, + _ body: ((Execution) async throws -> Result) + ) async throws -> ExecutionResult { + + let inputPipe = try input.createPipe() + let outputPipe = try output.createPipe() + let errorPipe = try error.createPipe() + + let execution = try self.spawn( + withInput: inputPipe, + output: output, + outputPipe: outputPipe, + error: error, + errorPipe: errorPipe + ) + // After spawn, clean up child side + try await self.cleanup( + execution: execution, + inputPipe: inputPipe, + outputPipe: outputPipe, + errorPipe: errorPipe, + childSide: true, + parentSide: false, + attemptToTerminateSubProcess: false + ) + + return try await withAsyncTaskCleanupHandler { + return try await withThrowingTaskGroup( + of: TerminationStatus?.self, + returning: ExecutionResult.self + ) { group in + group.addTask { + if let writeFd = inputPipe.writeFileDescriptor { + let writer = StandardInputWriter(fileDescriptor: writeFd) + try await input.write(with: writer) + try await writer.finish() + } + return nil + } + group.addTask { + return try await monitorProcessTermination( + forProcessWithIdentifier: execution.processIdentifier + ) + } + + // Body runs in the same isolation + let result = try await body(execution) + var status: TerminationStatus? = nil + while let monitorResult = try await group.next() { + if let monitorResult = monitorResult { + status = monitorResult + } + } + return ExecutionResult(terminationStatus: status!, value: result) + } + } onCleanup: { + // Attempt to terminate the child process + // Since the task has already been cancelled, + // this is the best we can do + try? await self.cleanup( + execution: execution, + inputPipe: inputPipe, + outputPipe: outputPipe, + errorPipe: errorPipe, + childSide: false, + parentSide: true, + attemptToTerminateSubProcess: true + ) + } + } +} + +extension Configuration: CustomStringConvertible, CustomDebugStringConvertible { + public var description: String { + return """ + Configuration( + executable: \(self.executable.description), + arguments: \(self.arguments.description), + environment: \(self.environment.description), + workingDirectory: \(self.workingDirectory), + platformOptions: \(self.platformOptions.description(withIndent: 1)) + ) + """ + } + + public var debugDescription: String { + return """ + Configuration( + executable: \(self.executable.debugDescription), + arguments: \(self.arguments.debugDescription), + environment: \(self.environment.debugDescription), + workingDirectory: \(self.workingDirectory), + platformOptions: \(self.platformOptions.description(withIndent: 1)) + ) + """ + } +} + +// MARK: - Cleanup +extension Configuration { + /// Close each input individually, and throw the first error if there's multiple errors thrown + @Sendable + @available(macOS 15.0, *) // FIXME: manually added availability + private func cleanup< + Output: OutputProtocol, + Error: OutputProtocol + >( + execution: Execution, + inputPipe: CreatedPipe, + outputPipe: CreatedPipe, + errorPipe: CreatedPipe, + childSide: Bool, + parentSide: Bool, + attemptToTerminateSubProcess: Bool + ) async throws { + func captureError(_ work: () throws -> Void) -> Swift.Error? { + do { + try work() + return nil + } catch { + // Ignore badFileDescriptor for double close + return error + } + } + + guard childSide || parentSide || attemptToTerminateSubProcess else { + return + } + + // Attempt to teardown the subprocess + if attemptToTerminateSubProcess { + await execution.teardown( + using: self.platformOptions.teardownSequence + ) + } + + var inputError: Swift.Error? + var outputError: Swift.Error? + var errorError: Swift.Error? // lol + + if childSide { + inputError = captureError { + try inputPipe.readFileDescriptor?.safelyClose() + } + outputError = captureError { + try outputPipe.writeFileDescriptor?.safelyClose() + } + errorError = captureError { + try errorPipe.writeFileDescriptor?.safelyClose() + } + } + + if parentSide { + inputError = captureError { + try inputPipe.writeFileDescriptor?.safelyClose() + } + outputError = captureError { + try outputPipe.readFileDescriptor?.safelyClose() + } + errorError = captureError { + try errorPipe.readFileDescriptor?.safelyClose() + } + } + + if let inputError = inputError { + throw inputError + } + + if let outputError = outputError { + throw outputError + } + + if let errorError = errorError { + throw errorError + } + } + + /// Close each input individually, and throw the first error if there's multiple errors thrown + @Sendable + internal func cleanupPreSpawn( + input: CreatedPipe, + output: CreatedPipe, + error: CreatedPipe + ) throws { + var inputError: Swift.Error? + var outputError: Swift.Error? + var errorError: Swift.Error? + + do { + try input.readFileDescriptor?.safelyClose() + try input.writeFileDescriptor?.safelyClose() + } catch { + inputError = error + } + + do { + try output.readFileDescriptor?.safelyClose() + try output.writeFileDescriptor?.safelyClose() + } catch { + outputError = error + } + + do { + try error.readFileDescriptor?.safelyClose() + try error.writeFileDescriptor?.safelyClose() + } catch { + errorError = error + } + + if let inputError = inputError { + throw inputError + } + if let outputError = outputError { + throw outputError + } + if let errorError = errorError { + throw errorError + } + } +} + +// MARK: - Executable + +/// `Executable` defines how the executable should +/// be looked up for execution. +public struct Executable: Sendable, Hashable { + internal enum Storage: Sendable, Hashable { + case executable(String) + case path(FilePath) + } + + internal let storage: Storage + + private init(_config: Storage) { + self.storage = _config + } + + /// Locate the executable by its name. + /// `Subprocess` will use `PATH` value to + /// determine the full path to the executable. + public static func name(_ executableName: String) -> Self { + return .init(_config: .executable(executableName)) + } + /// Locate the executable by its full path. + /// `Subprocess` will use this path directly. + public static func path(_ filePath: FilePath) -> Self { + return .init(_config: .path(filePath)) + } + /// Returns the full executable path given the environment value. + public func resolveExecutablePath(in environment: Environment) throws -> FilePath { + let path = try self.resolveExecutablePath(withPathValue: environment.pathValue()) + return FilePath(path) + } +} + +extension Executable: CustomStringConvertible, CustomDebugStringConvertible { + public var description: String { + switch storage { + case .executable(let executableName): + return executableName + case .path(let filePath): + return filePath.string + } + } + + public var debugDescription: String { + switch storage { + case .executable(let string): + return "executable(\(string))" + case .path(let filePath): + return "path(\(filePath.string))" + } + } +} + +extension Executable { + internal func possibleExecutablePaths( + withPathValue pathValue: String? + ) -> Set { + switch self.storage { + case .executable(let executableName): + #if os(Windows) + // Windows CreateProcessW accepts executable name directly + return Set([executableName]) + #else + var results: Set = [] + // executableName could be a full path + results.insert(executableName) + // Get $PATH from environment + let searchPaths: Set + if let pathValue = pathValue { + let localSearchPaths = pathValue.split(separator: ":").map { String($0) } + searchPaths = Set(localSearchPaths).union(Self.defaultSearchPaths) + } else { + searchPaths = Self.defaultSearchPaths + } + for path in searchPaths { + results.insert( + FilePath(path).appending(executableName).string + ) + } + return results + #endif + case .path(let executablePath): + return Set([executablePath.string]) + } + } +} + +// MARK: - Arguments + +/// A collection of arguments to pass to the subprocess. +public struct Arguments: Sendable, ExpressibleByArrayLiteral, Hashable { + public typealias ArrayLiteralElement = String + + internal let storage: [StringOrRawBytes] + internal let executablePathOverride: StringOrRawBytes? + + /// Create an Arguments object using the given literal values + public init(arrayLiteral elements: String...) { + self.storage = elements.map { .string($0) } + self.executablePathOverride = nil + } + /// Create an Arguments object using the given array + public init(_ array: [String]) { + self.storage = array.map { .string($0) } + self.executablePathOverride = nil + } + + #if !os(Windows) // Windows does NOT support arg0 override + /// Create an `Argument` object using the given values, but + /// override the first Argument value to `executablePathOverride`. + /// If `executablePathOverride` is nil, + /// `Arguments` will automatically use the executable path + /// as the first argument. + /// - Parameters: + /// - executablePathOverride: the value to override the first argument. + /// - remainingValues: the rest of the argument value + public init(executablePathOverride: String?, remainingValues: [String]) { + self.storage = remainingValues.map { .string($0) } + if let executablePathOverride = executablePathOverride { + self.executablePathOverride = .string(executablePathOverride) + } else { + self.executablePathOverride = nil + } + } + + /// Create an `Argument` object using the given values, but + /// override the first Argument value to `executablePathOverride`. + /// If `executablePathOverride` is nil, + /// `Arguments` will automatically use the executable path + /// as the first argument. + /// - Parameters: + /// - executablePathOverride: the value to override the first argument. + /// - remainingValues: the rest of the argument value + public init(executablePathOverride: [UInt8]?, remainingValues: [[UInt8]]) { + self.storage = remainingValues.map { .rawBytes($0) } + if let override = executablePathOverride { + self.executablePathOverride = .rawBytes(override) + } else { + self.executablePathOverride = nil + } + } + + public init(_ array: [[UInt8]]) { + self.storage = array.map { .rawBytes($0) } + self.executablePathOverride = nil + } + #endif +} + +extension Arguments: CustomStringConvertible, CustomDebugStringConvertible { + public var description: String { + var result: [String] = self.storage.map(\.description) + + if let override = self.executablePathOverride { + result.insert("override\(override.description)", at: 0) + } + return result.description + } + + public var debugDescription: String { return self.description } +} + +// MARK: - Environment + +/// A set of environment variables to use when executing the subprocess. +public struct Environment: Sendable, Hashable { + internal enum Configuration: Sendable, Hashable { + case inherit([String: String]) + case custom([String: String]) + #if !os(Windows) + case rawBytes([[UInt8]]) + #endif + } + + internal let config: Configuration + + init(config: Configuration) { + self.config = config + } + /// Child process should inherit the same environment + /// values from its parent process. + public static var inherit: Self { + return .init(config: .inherit([:])) + } + /// Override the provided `newValue` in the existing `Environment` + public func updating(_ newValue: [String: String]) -> Self { + return .init(config: .inherit(newValue)) + } + /// Use custom environment variables + public static func custom(_ newValue: [String: String]) -> Self { + return .init(config: .custom(newValue)) + } + + #if !os(Windows) + /// Use custom environment variables of raw bytes + public static func custom(_ newValue: [[UInt8]]) -> Self { + return .init(config: .rawBytes(newValue)) + } + #endif +} + +extension Environment: CustomStringConvertible, CustomDebugStringConvertible { + public var description: String { + switch self.config { + case .custom(let customDictionary): + return """ + Custom environment: + \(customDictionary) + """ + case .inherit(let updateValue): + return """ + Inherting current environment with updates: + \(updateValue) + """ + #if !os(Windows) + case .rawBytes(let rawBytes): + return """ + Raw bytes: + \(rawBytes) + """ + #endif + } + } + + public var debugDescription: String { + return self.description + } + + internal static func currentEnvironmentValues() -> [String: String] { + return self.withCopiedEnv { environments in + var results: [String: String] = [:] + for env in environments { + let environmentString = String(cString: env) + + #if os(Windows) + // Windows GetEnvironmentStringsW API can return + // magic environment variables set by the cmd shell + // that starts with `=` + // We should exclude these values + if environmentString.utf8.first == Character("=").utf8.first { + continue + } + #endif // os(Windows) + + guard let delimiter = environmentString.firstIndex(of: "=") else { + continue + } + + let key = String(environmentString[environmentString.startIndex.. UnsafeMutablePointer { + switch self { + case .string(let string): + return strdup(string) + case .rawBytes(let rawBytes): + return strdup(rawBytes) + } + } + + var stringValue: String? { + switch self { + case .string(let string): + return string + case .rawBytes(let rawBytes): + return String(decoding: rawBytes, as: UTF8.self) + } + } + + var description: String { + switch self { + case .string(let string): + return string + case .rawBytes(let bytes): + return bytes.description + } + } + + var count: Int { + switch self { + case .string(let string): + return string.count + case .rawBytes(let rawBytes): + return strnlen(rawBytes, Int.max) + } + } + + func hash(into hasher: inout Hasher) { + // If Raw bytes is valid UTF8, hash it as so + switch self { + case .string(let string): + hasher.combine(string) + case .rawBytes(let bytes): + if let stringValue = self.stringValue { + hasher.combine(stringValue) + } else { + hasher.combine(bytes) + } + } + } +} + +/// A simple wrapper on `FileDescriptor` plus a flag indicating +/// whether it should be closed automactially when done. +internal struct TrackedFileDescriptor: Hashable { + internal let closeWhenDone: Bool + internal let wrapped: FileDescriptor + + internal init( + _ wrapped: FileDescriptor, + closeWhenDone: Bool + ) { + self.wrapped = wrapped + self.closeWhenDone = closeWhenDone + } + + internal func safelyClose() throws { + guard self.closeWhenDone else { + return + } + + do { + try self.wrapped.close() + } catch { + guard let errno: Errno = error as? Errno else { + throw error + } + if errno != .badFileDescriptor { + throw errno + } + } + } + + internal var platformDescriptor: PlatformFileDescriptor { + return self.wrapped.platformDescriptor + } +} + +internal struct CreatedPipe { + internal let readFileDescriptor: TrackedFileDescriptor? + internal let writeFileDescriptor: TrackedFileDescriptor? + + internal init( + readFileDescriptor: TrackedFileDescriptor?, + writeFileDescriptor: TrackedFileDescriptor? + ) { + self.readFileDescriptor = readFileDescriptor + self.writeFileDescriptor = writeFileDescriptor + } + + internal init(closeWhenDone: Bool) throws { + let pipe = try FileDescriptor.pipe() + + self.readFileDescriptor = .init( + pipe.readEnd, + closeWhenDone: closeWhenDone + ) + self.writeFileDescriptor = .init( + pipe.writeEnd, + closeWhenDone: closeWhenDone + ) + } +} + +extension FilePath { + static var currentWorkingDirectory: Self { + let path = getcwd(nil, 0)! + defer { free(path) } + return .init(String(cString: path)) + } +} + +extension Optional where Wrapped: Collection { + func withOptionalUnsafeBufferPointer( + _ body: ((UnsafeBufferPointer)?) throws -> Result + ) rethrows -> Result { + switch self { + case .some(let wrapped): + guard let array: [Wrapped.Element] = wrapped as? Array else { + return try body(nil) + } + return try array.withUnsafeBufferPointer { ptr in + return try body(ptr) + } + case .none: + return try body(nil) + } + } +} + +extension Optional where Wrapped == String { + func withOptionalCString( + _ body: ((UnsafePointer)?) throws -> Result + ) rethrows -> Result { + switch self { + case .none: + return try body(nil) + case .some(let wrapped): + return try wrapped.withCString { + return try body($0) + } + } + } + + var stringValue: String { + return self ?? "nil" + } +} + +internal func withAsyncTaskCleanupHandler( + _ body: () async throws -> Result, + onCleanup handler: @Sendable @escaping () async -> Void, + isolation: isolated (any Actor)? = #isolation +) async rethrows -> Result { + return try await withThrowingTaskGroup( + of: Void.self, + returning: Result.self + ) { group in + group.addTask { + // Keep this task sleep indefinitely until the parent task is cancelled. + // `Task.sleep` throws `CancellationError` when the task is canceled + // before the time ends. We then run the cancel handler. + do { while true { try await Task.sleep(nanoseconds: 1_000_000_000) } } catch {} + // Run task cancel handler + await handler() + } + + do { + let result = try await body() + group.cancelAll() + return result + } catch { + await handler() + throw error + } + } +} diff --git a/Sources/_Subprocess/Error.swift b/Sources/_Subprocess/Error.swift new file mode 100644 index 00000000..bf1c9114 --- /dev/null +++ b/Sources/_Subprocess/Error.swift @@ -0,0 +1,141 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +#if canImport(Darwin) +import Darwin +#elseif canImport(Bionic) +import Bionic +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#elseif canImport(WinSDK) +import WinSDK +#endif + +/// Error thrown from Subprocess +public struct SubprocessError: Swift.Error, Hashable, Sendable { + /// The error code of this error + public let code: SubprocessError.Code + /// The underlying error that caused this error, if any + public let underlyingError: UnderlyingError? +} + +// MARK: - Error Codes +extension SubprocessError { + /// A SubprocessError Code + public struct Code: Hashable, Sendable { + internal enum Storage: Hashable, Sendable { + case spawnFailed + case executableNotFound(String) + case failedToChangeWorkingDirectory(String) + case failedToReadFromSubprocess + case failedToWriteToSubprocess + case failedToMonitorProcess + // Signal + case failedToSendSignal(Int32) + // Windows Only + case failedToTerminate + case failedToSuspend + case failedToResume + case failedToCreatePipe + case invalidWindowsPath(String) + } + + public var value: Int { + switch self.storage { + case .spawnFailed: + return 0 + case .executableNotFound(_): + return 1 + case .failedToChangeWorkingDirectory(_): + return 2 + case .failedToReadFromSubprocess: + return 3 + case .failedToWriteToSubprocess: + return 4 + case .failedToMonitorProcess: + return 5 + case .failedToSendSignal(_): + return 6 + case .failedToTerminate: + return 7 + case .failedToSuspend: + return 8 + case .failedToResume: + return 9 + case .failedToCreatePipe: + return 10 + case .invalidWindowsPath(_): + return 11 + } + } + + internal let storage: Storage + + internal init(_ storage: Storage) { + self.storage = storage + } + } +} + +// MARK: - Description +extension SubprocessError: CustomStringConvertible, CustomDebugStringConvertible { + public var description: String { + switch self.code.storage { + case .spawnFailed: + return "Failed to spawn the new process." + case .executableNotFound(let executableName): + return "Executable \"\(executableName)\" is not found or cannot be executed." + case .failedToChangeWorkingDirectory(let workingDirectory): + return "Failed to set working directory to \"\(workingDirectory)\"." + case .failedToReadFromSubprocess: + return "Failed to read bytes from the child process with underlying error: \(self.underlyingError!)" + case .failedToWriteToSubprocess: + return "Failed to write bytes to the child process." + case .failedToMonitorProcess: + return "Failed to monitor the state of child process with underlying error: \(self.underlyingError!)" + case .failedToSendSignal(let signal): + return "Failed to send signal \(signal) to the child process." + case .failedToTerminate: + return "Failed to terminate the child process." + case .failedToSuspend: + return "Failed to suspend the child process." + case .failedToResume: + return "Failed to resume the child process." + case .failedToCreatePipe: + return "Failed to create a pipe to communicate to child process." + case .invalidWindowsPath(let badPath): + return "\"\(badPath)\" is not a valid Windows path." + } + } + + public var debugDescription: String { self.description } +} + +extension SubprocessError { + /// The underlying error that caused this SubprocessError. + /// - On Unix-like systems, `UnderlyingError` wraps `errno` from libc; + /// - On Windows, `UnderlyingError` wraps Windows Error code + public struct UnderlyingError: Swift.Error, RawRepresentable, Hashable, Sendable { + #if os(Windows) + public typealias RawValue = DWORD + #else + public typealias RawValue = Int32 + #endif + + public let rawValue: RawValue + + public init(rawValue: RawValue) { + self.rawValue = rawValue + } + } +} diff --git a/Sources/_Subprocess/Execution.swift b/Sources/_Subprocess/Execution.swift new file mode 100644 index 00000000..8da9b492 --- /dev/null +++ b/Sources/_Subprocess/Execution.swift @@ -0,0 +1,192 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +#if canImport(System) +import System +#else +@preconcurrency import SystemPackage +#endif + +#if canImport(Darwin) +import Darwin +#elseif canImport(Bionic) +import Bionic +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#elseif canImport(WinSDK) +import WinSDK +#endif + +import Synchronization + +/// An object that repersents a subprocess that has been +/// executed. You can use this object to send signals to the +/// child process as well as stream its output and error. +@available(macOS 15.0, *) // FIXME: manually added availability +public final class Execution< + Output: OutputProtocol, + Error: OutputProtocol +>: Sendable { + /// The process identifier of the current execution + public let processIdentifier: ProcessIdentifier + + internal let output: Output + internal let error: Error + internal let outputPipe: CreatedPipe + internal let errorPipe: CreatedPipe + internal let outputConsumptionState: Atomic + #if os(Windows) + internal let consoleBehavior: PlatformOptions.ConsoleBehavior + + init( + processIdentifier: ProcessIdentifier, + output: Output, + error: Error, + outputPipe: CreatedPipe, + errorPipe: CreatedPipe, + consoleBehavior: PlatformOptions.ConsoleBehavior + ) { + self.processIdentifier = processIdentifier + self.output = output + self.error = error + self.outputPipe = outputPipe + self.errorPipe = errorPipe + self.outputConsumptionState = Atomic(0) + self.consoleBehavior = consoleBehavior + } + #else + init( + processIdentifier: ProcessIdentifier, + output: Output, + error: Error, + outputPipe: CreatedPipe, + errorPipe: CreatedPipe + ) { + self.processIdentifier = processIdentifier + self.output = output + self.error = error + self.outputPipe = outputPipe + self.errorPipe = errorPipe + self.outputConsumptionState = Atomic(0) + } + #endif // os(Windows) +} + +@available(macOS 15.0, *) // FIXME: manually added availability +extension Execution where Output == SequenceOutput { + /// The standard output of the subprocess. + /// + /// Accessing this property will **fatalError** if this property was + /// accessed multiple times. Subprocess communicates with parent process + /// via pipe under the hood and each pipe can only be consumed once. + @available(macOS 15.0, *) // FIXME: manually added availability + public var standardOutput: some AsyncSequence { + let consumptionState = self.outputConsumptionState.bitwiseXor( + OutputConsumptionState.standardOutputConsumed.rawValue, + ordering: .relaxed + ).newValue + + guard OutputConsumptionState(rawValue: consumptionState).contains(.standardOutputConsumed), + let fd = self.outputPipe.readFileDescriptor + else { + fatalError("The standard output has already been consumed") + } + return AsyncBufferSequence(fileDescriptor: fd) + } +} + +@available(macOS 15.0, *) // FIXME: manually added availability +extension Execution where Error == SequenceOutput { + /// The standard error of the subprocess. + /// + /// Accessing this property will **fatalError** if this property was + /// accessed multiple times. Subprocess communicates with parent process + /// via pipe under the hood and each pipe can only be consumed once. + @available(macOS 15.0, *) // FIXME: manually added availability + public var standardError: some AsyncSequence { + let consumptionState = self.outputConsumptionState.bitwiseXor( + OutputConsumptionState.standardErrorConsumed.rawValue, + ordering: .relaxed + ).newValue + + guard OutputConsumptionState(rawValue: consumptionState).contains(.standardErrorConsumed), + let fd = self.errorPipe.readFileDescriptor + else { + fatalError("The standard output has already been consumed") + } + return AsyncBufferSequence(fileDescriptor: fd) + } +} + +// MARK: - Output Capture +internal enum OutputCapturingState: Sendable { + case standardOutputCaptured(Output) + case standardErrorCaptured(Error) +} + +internal struct OutputConsumptionState: OptionSet { + typealias RawValue = UInt8 + + internal let rawValue: UInt8 + + internal init(rawValue: UInt8) { + self.rawValue = rawValue + } + + static let standardOutputConsumed: Self = .init(rawValue: 0b0001) + static let standardErrorConsumed: Self = .init(rawValue: 0b0010) +} + +internal typealias CapturedIOs< + Output: Sendable, + Error: Sendable +> = (standardOutput: Output, standardError: Error) + +@available(macOS 15.0, *) // FIXME: manually added availability +extension Execution { + internal func captureIOs() async throws -> CapturedIOs< + Output.OutputType, Error.OutputType + > { + return try await withThrowingTaskGroup( + of: OutputCapturingState.self + ) { group in + group.addTask { + let stdout = try await self.output.captureOutput( + from: self.outputPipe.readFileDescriptor + ) + return .standardOutputCaptured(stdout) + } + group.addTask { + let stderr = try await self.error.captureOutput( + from: self.errorPipe.readFileDescriptor + ) + return .standardErrorCaptured(stderr) + } + + var stdout: Output.OutputType! + var stderror: Error.OutputType! + while let state = try await group.next() { + switch state { + case .standardOutputCaptured(let output): + stdout = output + case .standardErrorCaptured(let error): + stderror = error + } + } + return ( + standardOutput: stdout, + standardError: stderror + ) + } + } +} diff --git a/Sources/_Subprocess/IO/Input.swift b/Sources/_Subprocess/IO/Input.swift new file mode 100644 index 00000000..5aad5d94 --- /dev/null +++ b/Sources/_Subprocess/IO/Input.swift @@ -0,0 +1,315 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +#if canImport(System) +import System +#else +@preconcurrency import SystemPackage +#endif + +#if SubprocessFoundation + +#if canImport(Darwin) +// On Darwin always prefer system Foundation +import Foundation +#else +// On other platforms prefer FoundationEssentials +import FoundationEssentials +#endif + +#endif // SubprocessFoundation + +// MARK: - Input + +/// `InputProtocol` defines the `write(with:)` method that a type +/// must implement to serve as the input source for a subprocess. +public protocol InputProtocol: Sendable, ~Copyable { + /// Asynchronously write the input to the subprocess using the + /// write file descriptor + func write(with writer: StandardInputWriter) async throws +} + +/// A concrete `Input` type for subprocesses that indicates +/// the absence of input to the subprocess. On Unix-like systems, +/// `NoInput` redirects the standard input of the subprocess +/// to `/dev/null`, while on Windows, it does not bind any +/// file handle to the subprocess standard input handle. +public struct NoInput: InputProtocol { + internal func createPipe() throws -> CreatedPipe { + #if os(Windows) + // On Windows, instead of binding to dev null, + // we don't set the input handle in the `STARTUPINFOW` + // to signal no input + return CreatedPipe( + readFileDescriptor: nil, + writeFileDescriptor: nil + ) + #else + let devnull: FileDescriptor = try .openDevNull(withAcessMode: .readOnly) + return CreatedPipe( + readFileDescriptor: .init(devnull, closeWhenDone: true), + writeFileDescriptor: nil + ) + #endif + } + + public func write(with writer: StandardInputWriter) async throws { + // noop + } + + internal init() {} +} + +/// A concrete `Input` type for subprocesses that +/// reads input from a specified `FileDescriptor`. +/// Developers have the option to instruct the `Subprocess` to +/// automatically close the provided `FileDescriptor` +/// after the subprocess is spawned. +public struct FileDescriptorInput: InputProtocol { + private let fileDescriptor: FileDescriptor + private let closeAfterSpawningProcess: Bool + + internal func createPipe() throws -> CreatedPipe { + return CreatedPipe( + readFileDescriptor: .init( + self.fileDescriptor, + closeWhenDone: self.closeAfterSpawningProcess + ), + writeFileDescriptor: nil + ) + } + + public func write(with writer: StandardInputWriter) async throws { + // noop + } + + internal init( + fileDescriptor: FileDescriptor, + closeAfterSpawningProcess: Bool + ) { + self.fileDescriptor = fileDescriptor + self.closeAfterSpawningProcess = closeAfterSpawningProcess + } +} + +/// A concrete `Input` type for subprocesses that reads input +/// from a given type conforming to `StringProtocol`. +/// Developers can specify the string encoding to use when +/// encoding the string to data, which defaults to UTF-8. +public struct StringInput< + InputString: StringProtocol & Sendable, + Encoding: Unicode.Encoding +>: InputProtocol { + private let string: InputString + + public func write(with writer: StandardInputWriter) async throws { + guard let array = self.string.byteArray(using: Encoding.self) else { + return + } + _ = try await writer.write(array) + } + + internal init(string: InputString, encoding: Encoding.Type) { + self.string = string + } +} + +/// A concrete `Input` type for subprocesses that reads input +/// from a given `UInt8` Array. +public struct ArrayInput: InputProtocol { + private let array: [UInt8] + + public func write(with writer: StandardInputWriter) async throws { + _ = try await writer.write(self.array) + } + + internal init(array: [UInt8]) { + self.array = array + } +} + +/// A concrete `Input` type for subprocess that indicates that +/// the Subprocess should read its input from `StandardInputWriter`. +public struct CustomWriteInput: InputProtocol { + public func write(with writer: StandardInputWriter) async throws { + // noop + } + + internal init() {} +} + +extension InputProtocol where Self == NoInput { + /// Create a Subprocess input that specfies there is no input + public static var none: Self { .init() } +} + +extension InputProtocol where Self == FileDescriptorInput { + /// Create a Subprocess input from a `FileDescriptor` and + /// specify whether the `FileDescriptor` should be closed + /// after the process is spawned. + public static func fileDescriptor( + _ fd: FileDescriptor, + closeAfterSpawningProcess: Bool + ) -> Self { + return .init( + fileDescriptor: fd, + closeAfterSpawningProcess: closeAfterSpawningProcess + ) + } +} + +extension InputProtocol { + /// Create a Subprocess input from a `Array` of `UInt8`. + public static func array( + _ array: [UInt8] + ) -> Self where Self == ArrayInput { + return ArrayInput(array: array) + } + + /// Create a Subprocess input from a type that conforms to `StringProtocol` + public static func string< + InputString: StringProtocol & Sendable + >( + _ string: InputString + ) -> Self where Self == StringInput { + return .init(string: string, encoding: UTF8.self) + } + + /// Create a Subprocess input from a type that conforms to `StringProtocol` + public static func string< + InputString: StringProtocol & Sendable, + Encoding: Unicode.Encoding + >( + _ string: InputString, + using encoding: Encoding.Type + ) -> Self where Self == StringInput { + return .init(string: string, encoding: encoding) + } +} + +extension InputProtocol { + internal func createPipe() throws -> CreatedPipe { + if let noInput = self as? NoInput { + return try noInput.createPipe() + } else if let fdInput = self as? FileDescriptorInput { + return try fdInput.createPipe() + } + // Base implementation + return try CreatedPipe(closeWhenDone: true) + } +} + +// MARK: - StandardInputWriter + +/// A writer that writes to the standard input of the subprocess. +public final actor StandardInputWriter: Sendable { + + internal let fileDescriptor: TrackedFileDescriptor + + init(fileDescriptor: TrackedFileDescriptor) { + self.fileDescriptor = fileDescriptor + } + + /// Write an array of UInt8 to the standard input of the subprocess. + /// - Parameter array: The sequence of bytes to write. + /// - Returns number of bytes written. + public func write( + _ array: [UInt8] + ) async throws -> Int { + return try await self.fileDescriptor.wrapped.write(array) + } + + /// Write a StringProtocol to the standard input of the subprocess. + /// - Parameters: + /// - string: The string to write. + /// - encoding: The encoding to use when converting string to bytes + /// - Returns number of bytes written. + public func write( + _ string: some StringProtocol, + using encoding: Encoding.Type = UTF8.self + ) async throws -> Int { + if let array = string.byteArray(using: encoding) { + return try await self.write(array) + } + return 0 + } + + /// Signal all writes are finished + public func finish() async throws { + try self.fileDescriptor.safelyClose() + } +} + +extension StringProtocol { + #if SubprocessFoundation + private func convertEncoding( + _ encoding: Encoding.Type + ) -> String.Encoding? { + switch encoding { + case is UTF8.Type: + return .utf8 + case is UTF16.Type: + return .utf16 + case is UTF32.Type: + return .utf32 + default: + return nil + } + } + #endif + package func byteArray(using encoding: Encoding.Type) -> [UInt8]? { + if Encoding.self == Unicode.ASCII.self { + let isASCII = self.utf8.allSatisfy { + return Character(Unicode.Scalar($0)).isASCII + } + + guard isASCII else { + return nil + } + return Array(self.utf8) + } + if Encoding.self == UTF8.self { + return Array(self.utf8) + } + if Encoding.self == UTF16.self { + return Array(self.utf16).flatMap { input in + var uint16: UInt16 = input + return withUnsafeBytes(of: &uint16) { ptr in + Array(ptr) + } + } + } + #if SubprocessFoundation + if let stringEncoding = self.convertEncoding(encoding), + let encoded = self.data(using: stringEncoding) + { + return Array(encoded) + } + return nil + #else + return nil + #endif + } +} + +extension String { + package init( + decodingBytes bytes: [T], + as encoding: Encoding.Type + ) { + self = bytes.withUnsafeBytes { raw in + String( + decoding: raw.bindMemory(to: Encoding.CodeUnit.self).lazy.map { $0 }, + as: encoding + ) + } + } +} diff --git a/Sources/_Subprocess/IO/Output.swift b/Sources/_Subprocess/IO/Output.swift new file mode 100644 index 00000000..be186dd6 --- /dev/null +++ b/Sources/_Subprocess/IO/Output.swift @@ -0,0 +1,298 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +#if canImport(System) +import System +#else +@preconcurrency import SystemPackage +#endif +internal import Dispatch + +// MARK: - Output + +/// `OutputProtocol` specifies the set of methods that a type +/// must implement to serve as the output target for a subprocess. +/// Instead of developing custom implementations of `OutputProtocol`, +/// it is recommended to utilize the default implementations provided +/// by the `Subprocess` library to specify the output handling requirements. +public protocol OutputProtocol: Sendable, ~Copyable { + associatedtype OutputType: Sendable + + /// Convert the output from buffer to expected output type + func output(from buffer: some Sequence) throws -> OutputType + + /// The max amount of data to collect for this output. + var maxSize: Int { get } +} + +extension OutputProtocol { + /// The max amount of data to collect for this output. + public var maxSize: Int { 128 * 1024 } +} + +/// A concrete `Output` type for subprocesses that indicates that +/// the `Subprocess` should not collect or redirect output +/// from the child process. On Unix-like systems, `DiscardedOutput` +/// redirects the standard output of the subprocess to `/dev/null`, +/// while on Windows, it does not bind any file handle to the +/// subprocess standard output handle. +public struct DiscardedOutput: OutputProtocol { + public typealias OutputType = Void + + internal func createPipe() throws -> CreatedPipe { + #if os(Windows) + // On Windows, instead of binding to dev null, + // we don't set the input handle in the `STARTUPINFOW` + // to signal no output + return CreatedPipe( + readFileDescriptor: nil, + writeFileDescriptor: nil + ) + #else + let devnull: FileDescriptor = try .openDevNull(withAcessMode: .readOnly) + return CreatedPipe( + readFileDescriptor: .init(devnull, closeWhenDone: true), + writeFileDescriptor: nil + ) + #endif + } + + internal init() {} +} + +/// A concrete `Output` type for subprocesses that +/// writes output to a specified `FileDescriptor`. +/// Developers have the option to instruct the `Subprocess` to +/// automatically close the provided `FileDescriptor` +/// after the subprocess is spawned. +public struct FileDescriptorOutput: OutputProtocol { + public typealias OutputType = Void + + private let closeAfterSpawningProcess: Bool + private let fileDescriptor: FileDescriptor + + internal func createPipe() throws -> CreatedPipe { + return CreatedPipe( + readFileDescriptor: nil, + writeFileDescriptor: .init( + self.fileDescriptor, + closeWhenDone: self.closeAfterSpawningProcess + ) + ) + } + + internal init( + fileDescriptor: FileDescriptor, + closeAfterSpawningProcess: Bool + ) { + self.fileDescriptor = fileDescriptor + self.closeAfterSpawningProcess = closeAfterSpawningProcess + } +} + +/// A concrete `Output` type for subprocesses that collects output +/// from the subprocess as `String` with the given encoding. +/// This option must be used with he `run()` method that +/// returns a `CollectedResult`. +public struct StringOutput: OutputProtocol { + public typealias OutputType = String? + public let maxSize: Int + private let encoding: Encoding.Type + + public func output(from buffer: some Sequence) throws -> String? { + // FIXME: Span to String + let array = Array(buffer) + return String(decodingBytes: array, as: Encoding.self) + } + + internal init(limit: Int, encoding: Encoding.Type) { + self.maxSize = limit + self.encoding = encoding + } +} + +/// A concrete `Output` type for subprocesses that collects output +/// from the subprocess as `[UInt8]`. This option must be used with +/// the `run()` method that returns a `CollectedResult` +public struct BytesOutput: OutputProtocol { + public typealias OutputType = [UInt8] + public let maxSize: Int + + internal func captureOutput(from fileDescriptor: TrackedFileDescriptor?) async throws -> [UInt8] { + return try await withCheckedThrowingContinuation { continuation in + guard let fileDescriptor = fileDescriptor else { + // Show not happen due to type system constraints + fatalError("Trying to capture output without file descriptor") + } + fileDescriptor.wrapped.readUntilEOF(upToLength: self.maxSize) { result in + switch result { + case .success(let data): + // FIXME: remove workaround for + // rdar://143992296 + // https://github.com/swiftlang/swift-subprocess/issues/3 + #if os(Windows) + continuation.resume(returning: data) + #else + continuation.resume(returning: data.array()) + #endif + case .failure(let error): + continuation.resume(throwing: error) + } + } + } + } + + public func output(from buffer: some Sequence) throws -> [UInt8] { + fatalError("Not implemented") + } + + internal init(limit: Int) { + self.maxSize = limit + } +} + +/// A concrete `Output` type for subprocesses that redirects +/// the child output to the `.standardOutput` (a sequence) or `.standardError` +/// property of `Execution`. This output type is +/// only applicable to the `run()` family that takes a custom closure. +public struct SequenceOutput: OutputProtocol { + public typealias OutputType = Void + + internal init() {} +} + +extension OutputProtocol where Self == DiscardedOutput { + /// Create a Subprocess output that discards the output + public static var discarded: Self { .init() } +} + +extension OutputProtocol where Self == FileDescriptorOutput { + /// Create a Subprocess output that writes output to a `FileDescriptor` + /// and optionally close the `FileDescriptor` once process spawned. + public static func fileDescriptor( + _ fd: FileDescriptor, + closeAfterSpawningProcess: Bool + ) -> Self { + return .init(fileDescriptor: fd, closeAfterSpawningProcess: closeAfterSpawningProcess) + } +} + +extension OutputProtocol where Self == StringOutput { + /// Create a `Subprocess` output that collects output as + /// UTF8 String with 128kb limit. + public static var string: Self { + .init(limit: 128 * 1024, encoding: UTF8.self) + } +} + +extension OutputProtocol { + /// Create a `Subprocess` output that collects output as + /// `String` using the given encoding up to limit it bytes. + public static func string( + limit: Int, + encoding: Encoding.Type + ) -> Self where Self == StringOutput { + return .init(limit: limit, encoding: encoding) + } +} + +extension OutputProtocol where Self == BytesOutput { + /// Create a `Subprocess` output that collects output as + /// `Buffer` with 128kb limit. + public static var bytes: Self { .init(limit: 128 * 1024) } + + /// Create a `Subprocess` output that collects output as + /// `Buffer` up to limit it bytes. + public static func bytes(limit: Int) -> Self { + return .init(limit: limit) + } +} + +extension OutputProtocol where Self == SequenceOutput { + /// Create a `Subprocess` output that redirects the output + /// to the `.standardOutput` (or `.standardError`) property + /// of `Execution` as `AsyncSequence`. + public static var sequence: Self { .init() } +} + +// MARK: - Default Implementations +extension OutputProtocol { + @_disfavoredOverload + internal func createPipe() throws -> CreatedPipe { + if let discard = self as? DiscardedOutput { + return try discard.createPipe() + } else if let fdOutput = self as? FileDescriptorOutput { + return try fdOutput.createPipe() + } + // Base pipe based implementation for everything else + return try CreatedPipe(closeWhenDone: true) + } + + /// Capture the output from the subprocess up to maxSize + @_disfavoredOverload + internal func captureOutput( + from fileDescriptor: TrackedFileDescriptor? + ) async throws -> OutputType { + if let bytesOutput = self as? BytesOutput { + return try await bytesOutput.captureOutput(from: fileDescriptor) as! Self.OutputType + } + return try await withCheckedThrowingContinuation { continuation in + if OutputType.self == Void.self { + continuation.resume(returning: () as! OutputType) + return + } + guard let fileDescriptor = fileDescriptor else { + // Show not happen due to type system constraints + fatalError("Trying to capture output without file descriptor") + } + + fileDescriptor.wrapped.readUntilEOF(upToLength: self.maxSize) { result in + do { + switch result { + case .success(let data): + // FIXME: remove workaround for + // rdar://143992296 + // https://github.com/swiftlang/swift-subprocess/issues/3 + let output = try self.output(from: data) + continuation.resume(returning: output) + case .failure(let error): + continuation.resume(throwing: error) + } + } catch { + continuation.resume(throwing: error) + } + } + } + } +} + +extension OutputProtocol where OutputType == Void { + internal func captureOutput(from fileDescriptor: TrackedFileDescriptor?) async throws {} + + public func output(from buffer: some Sequence) throws { + // noop + } +} + +extension DispatchData { + internal func array() -> [UInt8] { + var result: [UInt8]? + self.enumerateBytes { buffer, byteIndex, stop in + let currentChunk = Array(UnsafeRawBufferPointer(buffer)) + if result == nil { + result = currentChunk + } else { + result?.append(contentsOf: currentChunk) + } + } + return result ?? [] + } +} diff --git a/Sources/_Subprocess/LockedState.swift b/Sources/_Subprocess/LockedState.swift deleted file mode 100644 index e095668c..00000000 --- a/Sources/_Subprocess/LockedState.swift +++ /dev/null @@ -1,160 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift.org project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -#if canImport(os) -internal import os -#if FOUNDATION_FRAMEWORK && canImport(C.os.lock) -internal import C.os.lock -#endif -#elseif canImport(Bionic) -import Bionic -#elseif canImport(Glibc) -import Glibc -#elseif canImport(Musl) -import Musl -#elseif canImport(WinSDK) -import WinSDK -#endif - -package struct LockedState { - - // Internal implementation for a cheap lock to aid sharing code across platforms - private struct _Lock { -#if canImport(os) - typealias Primitive = os_unfair_lock -#elseif canImport(Bionic) || canImport(Glibc) || canImport(Musl) - typealias Primitive = pthread_mutex_t -#elseif canImport(WinSDK) - typealias Primitive = SRWLOCK -#elseif os(WASI) - // WASI is single-threaded, so we don't need a lock. - typealias Primitive = Void -#endif - - typealias PlatformLock = UnsafeMutablePointer - var _platformLock: PlatformLock - - fileprivate static func initialize(_ platformLock: PlatformLock) { -#if canImport(os) - platformLock.initialize(to: os_unfair_lock()) -#elseif canImport(Bionic) || canImport(Glibc) - pthread_mutex_init(platformLock, nil) -#elseif canImport(WinSDK) - InitializeSRWLock(platformLock) -#elseif os(WASI) - // no-op -#endif - } - - fileprivate static func deinitialize(_ platformLock: PlatformLock) { -#if canImport(Bionic) || canImport(Glibc) - pthread_mutex_destroy(platformLock) -#endif - platformLock.deinitialize(count: 1) - } - - static fileprivate func lock(_ platformLock: PlatformLock) { -#if canImport(os) - os_unfair_lock_lock(platformLock) -#elseif canImport(Bionic) || canImport(Glibc) - pthread_mutex_lock(platformLock) -#elseif canImport(WinSDK) - AcquireSRWLockExclusive(platformLock) -#elseif os(WASI) - // no-op -#endif - } - - static fileprivate func unlock(_ platformLock: PlatformLock) { -#if canImport(os) - os_unfair_lock_unlock(platformLock) -#elseif canImport(Bionic) || canImport(Glibc) - pthread_mutex_unlock(platformLock) -#elseif canImport(WinSDK) - ReleaseSRWLockExclusive(platformLock) -#elseif os(WASI) - // no-op -#endif - } - } - - private class _Buffer: ManagedBuffer { - deinit { - withUnsafeMutablePointerToElements { - _Lock.deinitialize($0) - } - } - } - - private let _buffer: ManagedBuffer - - package init(initialState: State) { - _buffer = _Buffer.create(minimumCapacity: 1, makingHeaderWith: { buf in - buf.withUnsafeMutablePointerToElements { - _Lock.initialize($0) - } - return initialState - }) - } - - package func withLock(_ body: @Sendable (inout State) throws -> T) rethrows -> T { - try withLockUnchecked(body) - } - - package func withLockUnchecked(_ body: (inout State) throws -> T) rethrows -> T { - try _buffer.withUnsafeMutablePointers { state, lock in - _Lock.lock(lock) - defer { _Lock.unlock(lock) } - return try body(&state.pointee) - } - } - - // Ensures the managed state outlives the locked scope. - package func withLockExtendingLifetimeOfState(_ body: @Sendable (inout State) throws -> T) rethrows -> T { - try _buffer.withUnsafeMutablePointers { state, lock in - _Lock.lock(lock) - return try withExtendedLifetime(state.pointee) { - defer { _Lock.unlock(lock) } - return try body(&state.pointee) - } - } - } -} - -extension LockedState where State == Void { - package init() { - self.init(initialState: ()) - } - - package func withLock(_ body: @Sendable () throws -> R) rethrows -> R { - return try withLock { _ in - try body() - } - } - - package func lock() { - _buffer.withUnsafeMutablePointerToElements { lock in - _Lock.lock(lock) - } - } - - package func unlock() { - _buffer.withUnsafeMutablePointerToElements { lock in - _Lock.unlock(lock) - } - } -} - -extension LockedState: @unchecked Sendable where State: Sendable {} - diff --git a/Sources/_Subprocess/Platforms/Subprocess+Darwin.swift b/Sources/_Subprocess/Platforms/Subprocess+Darwin.swift index 4ac2276e..cd5c310a 100644 --- a/Sources/_Subprocess/Platforms/Subprocess+Darwin.swift +++ b/Sources/_Subprocess/Platforms/Subprocess+Darwin.swift @@ -2,306 +2,144 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors -// Licensed under Apache License v2.0 +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception // -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift.org project authors -// -// SPDX-License-Identifier: Apache-2.0 +// See https://swift.org/LICENSE.txt for license information // //===----------------------------------------------------------------------===// #if canImport(Darwin) -#if canImport(FoundationEssentials) -import FoundationEssentials -#elseif canImport(Foundation) -import Foundation -#endif - import Darwin -import Dispatch -import SystemPackage - -#if FOUNDATION_FRAMEWORK -@_implementationOnly import _FoundationCShims +internal import Dispatch +#if canImport(System) +import System #else -import _CShims +@preconcurrency import SystemPackage #endif -// Darwin specific implementation -extension Subprocess.Configuration { - internal typealias StringOrRawBytes = Subprocess.StringOrRawBytes +import _SubprocessCShims - internal func spawn( - withInput input: Subprocess.ExecutionInput, - output: Subprocess.ExecutionOutput, - error: Subprocess.ExecutionOutput - ) throws -> Subprocess { - let (executablePath, - env, argv, - intendedWorkingDir, - uidPtr, gidPtr, supplementaryGroups - ) = try self.preSpawn() - defer { - for ptr in env { ptr?.deallocate() } - for ptr in argv { ptr?.deallocate() } - uidPtr?.deallocate() - gidPtr?.deallocate() - } +#if SubprocessFoundation - // Setup file actions and spawn attributes - var fileActions: posix_spawn_file_actions_t? = nil - var spawnAttributes: posix_spawnattr_t? = nil - // Setup stdin, stdout, and stderr - posix_spawn_file_actions_init(&fileActions) - defer { - posix_spawn_file_actions_destroy(&fileActions) - } - // Input - var result: Int32 = -1 - if let inputRead = input.getReadFileDescriptor() { - result = posix_spawn_file_actions_adddup2(&fileActions, inputRead.rawValue, 0) - guard result == 0 else { - try self.cleanupAll(input: input, output: output, error: error) - throw POSIXError(.init(rawValue: result) ?? .ENODEV) - } - } - if let inputWrite = input.getWriteFileDescriptor() { - // Close parent side - result = posix_spawn_file_actions_addclose(&fileActions, inputWrite.rawValue) - guard result == 0 else { - try self.cleanupAll(input: input, output: output, error: error) - throw POSIXError(.init(rawValue: result) ?? .ENODEV) - } - } - // Output - if let outputWrite = output.getWriteFileDescriptor() { - result = posix_spawn_file_actions_adddup2(&fileActions, outputWrite.rawValue, 1) - guard result == 0 else { - try self.cleanupAll(input: input, output: output, error: error) - throw POSIXError(.init(rawValue: result) ?? .ENODEV) - } - } - if let outputRead = output.getReadFileDescriptor() { - // Close parent side - result = posix_spawn_file_actions_addclose(&fileActions, outputRead.rawValue) - guard result == 0 else { - try self.cleanupAll(input: input, output: output, error: error) - throw POSIXError(.init(rawValue: result) ?? .ENODEV) - } - } - // Error - if let errorWrite = error.getWriteFileDescriptor() { - result = posix_spawn_file_actions_adddup2(&fileActions, errorWrite.rawValue, 2) - guard result == 0 else { - try self.cleanupAll(input: input, output: output, error: error) - throw POSIXError(.init(rawValue: result) ?? .ENODEV) - } - } - if let errorRead = error.getReadFileDescriptor() { - // Close parent side - result = posix_spawn_file_actions_addclose(&fileActions, errorRead.rawValue) - guard result == 0 else { - try self.cleanupAll(input: input, output: output, error: error) - throw POSIXError(.init(rawValue: result) ?? .ENODEV) - } - } - // Setup spawnAttributes - posix_spawnattr_init(&spawnAttributes) - defer { - posix_spawnattr_destroy(&spawnAttributes) - } - var noSignals = sigset_t() - var allSignals = sigset_t() - sigemptyset(&noSignals) - sigfillset(&allSignals) - posix_spawnattr_setsigmask(&spawnAttributes, &noSignals) - posix_spawnattr_setsigdefault(&spawnAttributes, &allSignals) - // Configure spawnattr - var spawnAttributeError: Int32 = 0 - var flags: Int32 = POSIX_SPAWN_CLOEXEC_DEFAULT | - POSIX_SPAWN_SETSIGMASK | POSIX_SPAWN_SETSIGDEF - if let pgid = self.platformOptions.processGroupID { - flags |= POSIX_SPAWN_SETPGROUP - spawnAttributeError = posix_spawnattr_setpgroup(&spawnAttributes, pid_t(pgid)) - } - spawnAttributeError = posix_spawnattr_setflags(&spawnAttributes, Int16(flags)) - // Set QualityOfService - // spanattr_qos seems to only accept `QOS_CLASS_UTILITY` or `QOS_CLASS_BACKGROUND` - // and returns an error of `EINVAL` if anything else is provided - if spawnAttributeError == 0 && self.platformOptions.qualityOfService == .utility{ - spawnAttributeError = posix_spawnattr_set_qos_class_np(&spawnAttributes, QOS_CLASS_UTILITY) - } else if spawnAttributeError == 0 && self.platformOptions.qualityOfService == .background { - spawnAttributeError = posix_spawnattr_set_qos_class_np(&spawnAttributes, QOS_CLASS_BACKGROUND) - } - - // Setup cwd - var chdirError: Int32 = 0 - if intendedWorkingDir != .currentWorkingDirectory { - chdirError = intendedWorkingDir.withPlatformString { workDir in - return posix_spawn_file_actions_addchdir_np(&fileActions, workDir) - } - } - - // Error handling - if chdirError != 0 || spawnAttributeError != 0 { - try self.cleanupAll(input: input, output: output, error: error) - if spawnAttributeError != 0 { - throw POSIXError(.init(rawValue: result) ?? .ENODEV) - } +#if canImport(Darwin) +// On Darwin always prefer system Foundation +import Foundation +#else +// On other platforms prefer FoundationEssentials +import FoundationEssentials +#endif - if chdirError != 0 { - throw CocoaError(.fileNoSuchFile, userInfo: [ - .debugDescriptionErrorKey: "Cannot failed to change the working directory to \(intendedWorkingDir) with errno \(chdirError)" - ]) - } - } - // Run additional config - if let spawnConfig = self.platformOptions.preSpawnProcessConfigurator { - try spawnConfig(&spawnAttributes, &fileActions) - } - // Spawn - var pid: pid_t = 0 - let spawnError: CInt = executablePath.withCString { exePath in - return supplementaryGroups.withOptionalUnsafeBufferPointer { sgroups in - return _subprocess_spawn( - &pid, exePath, - &fileActions, &spawnAttributes, - argv, env, - uidPtr, gidPtr, - Int32(supplementaryGroups?.count ?? 0), sgroups?.baseAddress, - self.platformOptions.createSession ? 1 : 0 - ) - } - } - // Spawn error - if spawnError != 0 { - try self.cleanupAll(input: input, output: output, error: error) - throw POSIXError(.init(rawValue: spawnError) ?? .ENODEV) - } - return Subprocess( - processIdentifier: .init(value: pid), - executionInput: input, - executionOutput: output, - executionError: error - ) - } -} +#endif // SubprocessFoundation -// Special keys used in Error's user dictionary -extension String { - static let debugDescriptionErrorKey = "NSDebugDescription" -} +// MARK: - PlatformOptions -// MARK: - Platform Specific Options -extension Subprocess { - /// The collection of platform-specific settings - /// to configure the subprocess when running - public struct PlatformOptions: Sendable { - public var qualityOfService: QualityOfService = .default - /// Set user ID for the subprocess - public var userID: uid_t? = nil - /// Set the real and effective group ID and the saved - /// set-group-ID of the subprocess, equivalent to calling - /// `setgid()` on the child process. - /// Group ID is used to control permissions, particularly - /// for file access. - public var groupID: gid_t? = nil - /// Set list of supplementary group IDs for the subprocess - public var supplementaryGroups: [gid_t]? = nil - /// Set the process group for the subprocess, equivalent to - /// calling `setpgid()` on the child process. - /// Process group ID is used to group related processes for - /// controlling signals. - public var processGroupID: pid_t? = nil - /// Creates a session and sets the process group ID - /// i.e. Detach from the terminal. - public var createSession: Bool = false - /// A lightweight code requirement that you use to - /// evaluate the executable for a launching process. - public var launchRequirementData: Data? = nil - /// An ordered list of steps in order to tear down the child - /// process in case the parent task is cancelled before - /// the child proces terminates. - /// Always ends in sending a `.kill` signal at the end. - public var teardownSequence: [TeardownStep] = [] - /// A closure to configure platform-specific - /// spawning constructs. This closure enables direct - /// configuration or override of underlying platform-specific - /// spawn settings that `Subprocess` utilizes internally, - /// in cases where Subprocess does not provide higher-level - /// APIs for such modifications. - /// - /// On Darwin, Subprocess uses `posix_spawn()` as the - /// underlying spawning mechanism. This closure allows - /// modification of the `posix_spawnattr_t` spawn attribute - /// and file actions `posix_spawn_file_actions_t` before - /// they are sent to `posix_spawn()`. - public var preSpawnProcessConfigurator: ( +/// The collection of platform-specific settings +/// to configure the subprocess when running +public struct PlatformOptions: Sendable { + public var qualityOfService: QualityOfService = .default + /// Set user ID for the subprocess + public var userID: uid_t? = nil + /// Set the real and effective group ID and the saved + /// set-group-ID of the subprocess, equivalent to calling + /// `setgid()` on the child process. + /// Group ID is used to control permissions, particularly + /// for file access. + public var groupID: gid_t? = nil + /// Set list of supplementary group IDs for the subprocess + public var supplementaryGroups: [gid_t]? = nil + /// Set the process group for the subprocess, equivalent to + /// calling `setpgid()` on the child process. + /// Process group ID is used to group related processes for + /// controlling signals. + public var processGroupID: pid_t? = nil + /// Creates a session and sets the process group ID + /// i.e. Detach from the terminal. + public var createSession: Bool = false + /// An ordered list of steps in order to tear down the child + /// process in case the parent task is cancelled before + /// the child proces terminates. + /// Always ends in sending a `.kill` signal at the end. + public var teardownSequence: [TeardownStep] = [] + /// A closure to configure platform-specific + /// spawning constructs. This closure enables direct + /// configuration or override of underlying platform-specific + /// spawn settings that `Subprocess` utilizes internally, + /// in cases where Subprocess does not provide higher-level + /// APIs for such modifications. + /// + /// On Darwin, Subprocess uses `posix_spawn()` as the + /// underlying spawning mechanism. This closure allows + /// modification of the `posix_spawnattr_t` spawn attribute + /// and file actions `posix_spawn_file_actions_t` before + /// they are sent to `posix_spawn()`. + public var preSpawnProcessConfigurator: + ( @Sendable ( inout posix_spawnattr_t?, inout posix_spawn_file_actions_t? ) throws -> Void )? = nil - public init() {} - } + public init() {} } -extension Subprocess.PlatformOptions: Hashable { - public static func == (lhs: Subprocess.PlatformOptions, rhs: Subprocess.PlatformOptions) -> Bool { - // Since we can't compare closure equality, - // as long as preSpawnProcessConfigurator is set - // always returns false so that `PlatformOptions` - // with it set will never equal to each other - if lhs.preSpawnProcessConfigurator != nil || - rhs.preSpawnProcessConfigurator != nil { - return false - } - return lhs.qualityOfService == rhs.qualityOfService && - lhs.userID == rhs.userID && - lhs.groupID == rhs.groupID && - lhs.supplementaryGroups == rhs.supplementaryGroups && - lhs.processGroupID == rhs.processGroupID && - lhs.createSession == rhs.createSession && - lhs.launchRequirementData == rhs.launchRequirementData - } - - public func hash(into hasher: inout Hasher) { - hasher.combine(self.qualityOfService) - hasher.combine(self.userID) - hasher.combine(self.groupID) - hasher.combine(self.supplementaryGroups) - hasher.combine(self.processGroupID) - hasher.combine(self.createSession) - hasher.combine(self.launchRequirementData) - // Since we can't really hash closures, - // use an UUID such that as long as - // `preSpawnProcessConfigurator` is set, it will - // never equal to other PlatformOptions - if self.preSpawnProcessConfigurator != nil { - hasher.combine(UUID()) - } +extension PlatformOptions { + #if SubprocessFoundation + public typealias QualityOfService = Foundation.QualityOfService + #else + /// Constants that indicate the nature and importance of work to the system. + /// + /// Work with higher quality of service classes receive more resources + /// than work with lower quality of service classes whenever + /// there’s resource contention. + public enum QualityOfService: Int, Sendable { + /// Used for work directly involved in providing an + /// interactive UI. For example, processing control + /// events or drawing to the screen. + case userInteractive = 0x21 + /// Used for performing work that has been explicitly requested + /// by the user, and for which results must be immediately + /// presented in order to allow for further user interaction. + /// For example, loading an email after a user has selected + /// it in a message list. + case userInitiated = 0x19 + /// Used for performing work which the user is unlikely to be + /// immediately waiting for the results. This work may have been + /// requested by the user or initiated automatically, and often + /// operates at user-visible timescales using a non-modal + /// progress indicator. For example, periodic content updates + /// or bulk file operations, such as media import. + case utility = 0x11 + /// Used for work that is not user initiated or visible. + /// In general, a user is unaware that this work is even happening. + /// For example, pre-fetching content, search indexing, backups, + /// or syncing of data with external systems. + case background = 0x09 + /// Indicates no explicit quality of service information. + /// Whenever possible, an appropriate quality of service is determined + /// from available sources. Otherwise, some quality of service level + /// between `.userInteractive` and `.utility` is used. + case `default` = -1 } + #endif } -extension Subprocess.PlatformOptions : CustomStringConvertible, CustomDebugStringConvertible { +extension PlatformOptions: CustomStringConvertible, CustomDebugStringConvertible { internal func description(withIndent indent: Int) -> String { let indent = String(repeating: " ", count: indent * 4) return """ -PlatformOptions( -\(indent) qualityOfService: \(self.qualityOfService), -\(indent) userID: \(String(describing: userID)), -\(indent) groupID: \(String(describing: groupID)), -\(indent) supplementaryGroups: \(String(describing: supplementaryGroups)), -\(indent) processGroupID: \(String(describing: processGroupID)), -\(indent) createSession: \(createSession), -\(indent) launchRequirementData: \(String(describing: launchRequirementData)), -\(indent) preSpawnProcessConfigurator: \(self.preSpawnProcessConfigurator == nil ? "not set" : "set") -\(indent)) -""" + PlatformOptions( + \(indent) qualityOfService: \(self.qualityOfService), + \(indent) userID: \(String(describing: userID)), + \(indent) groupID: \(String(describing: groupID)), + \(indent) supplementaryGroups: \(String(describing: supplementaryGroups)), + \(indent) processGroupID: \(String(describing: processGroupID)), + \(indent) createSession: \(createSession), + \(indent) preSpawnProcessConfigurator: \(self.preSpawnProcessConfigurator == nil ? "not set" : "set") + \(indent)) + """ } public var description: String { @@ -313,11 +151,243 @@ PlatformOptions( } } +// MARK: - Spawn +extension Configuration { + @available(macOS 15.0, *) // FIXME: manually added availability + internal func spawn< + Output: OutputProtocol, + Error: OutputProtocol + >( + withInput inputPipe: CreatedPipe, + output: Output, + outputPipe: CreatedPipe, + error: Error, + errorPipe: CreatedPipe + ) throws -> Execution { + // Instead of checking if every possible executable path + // is valid, spawn each directly and catch ENOENT + let possiblePaths = self.executable.possibleExecutablePaths( + withPathValue: self.environment.pathValue() + ) + return try self.preSpawn { args throws -> Execution in + let (env, uidPtr, gidPtr, supplementaryGroups) = args + for possibleExecutablePath in possiblePaths { + var pid: pid_t = 0 + + // Setup Arguments + let argv: [UnsafeMutablePointer?] = self.arguments.createArgs( + withExecutablePath: possibleExecutablePath + ) + defer { + for ptr in argv { ptr?.deallocate() } + } + + // Setup file actions and spawn attributes + var fileActions: posix_spawn_file_actions_t? = nil + var spawnAttributes: posix_spawnattr_t? = nil + // Setup stdin, stdout, and stderr + posix_spawn_file_actions_init(&fileActions) + defer { + posix_spawn_file_actions_destroy(&fileActions) + } + // Input + var result: Int32 = -1 + if let inputRead = inputPipe.readFileDescriptor { + result = posix_spawn_file_actions_adddup2(&fileActions, inputRead.wrapped.rawValue, 0) + guard result == 0 else { + try self.cleanupPreSpawn(input: inputPipe, output: outputPipe, error: errorPipe) + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: result) + ) + } + } + if let inputWrite = inputPipe.writeFileDescriptor { + // Close parent side + result = posix_spawn_file_actions_addclose(&fileActions, inputWrite.wrapped.rawValue) + guard result == 0 else { + try self.cleanupPreSpawn(input: inputPipe, output: outputPipe, error: errorPipe) + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: result) + ) + } + } + // Output + if let outputWrite = outputPipe.writeFileDescriptor { + result = posix_spawn_file_actions_adddup2(&fileActions, outputWrite.wrapped.rawValue, 1) + guard result == 0 else { + try self.cleanupPreSpawn(input: inputPipe, output: outputPipe, error: errorPipe) + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: result) + ) + } + } + if let outputRead = outputPipe.readFileDescriptor { + // Close parent side + result = posix_spawn_file_actions_addclose(&fileActions, outputRead.wrapped.rawValue) + guard result == 0 else { + try self.cleanupPreSpawn(input: inputPipe, output: outputPipe, error: errorPipe) + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: result) + ) + } + } + // Error + if let errorWrite = errorPipe.writeFileDescriptor { + result = posix_spawn_file_actions_adddup2(&fileActions, errorWrite.wrapped.rawValue, 2) + guard result == 0 else { + try self.cleanupPreSpawn(input: inputPipe, output: outputPipe, error: errorPipe) + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: result) + ) + } + } + if let errorRead = errorPipe.readFileDescriptor { + // Close parent side + result = posix_spawn_file_actions_addclose(&fileActions, errorRead.wrapped.rawValue) + guard result == 0 else { + try self.cleanupPreSpawn(input: inputPipe, output: outputPipe, error: errorPipe) + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: result) + ) + } + } + // Setup spawnAttributes + posix_spawnattr_init(&spawnAttributes) + defer { + posix_spawnattr_destroy(&spawnAttributes) + } + var noSignals = sigset_t() + var allSignals = sigset_t() + sigemptyset(&noSignals) + sigfillset(&allSignals) + posix_spawnattr_setsigmask(&spawnAttributes, &noSignals) + posix_spawnattr_setsigdefault(&spawnAttributes, &allSignals) + // Configure spawnattr + var spawnAttributeError: Int32 = 0 + var flags: Int32 = POSIX_SPAWN_CLOEXEC_DEFAULT | POSIX_SPAWN_SETSIGMASK | POSIX_SPAWN_SETSIGDEF + if let pgid = self.platformOptions.processGroupID { + flags |= POSIX_SPAWN_SETPGROUP + spawnAttributeError = posix_spawnattr_setpgroup(&spawnAttributes, pid_t(pgid)) + } + spawnAttributeError = posix_spawnattr_setflags(&spawnAttributes, Int16(flags)) + // Set QualityOfService + // spanattr_qos seems to only accept `QOS_CLASS_UTILITY` or `QOS_CLASS_BACKGROUND` + // and returns an error of `EINVAL` if anything else is provided + if spawnAttributeError == 0 && self.platformOptions.qualityOfService == .utility { + spawnAttributeError = posix_spawnattr_set_qos_class_np(&spawnAttributes, QOS_CLASS_UTILITY) + } else if spawnAttributeError == 0 && self.platformOptions.qualityOfService == .background { + spawnAttributeError = posix_spawnattr_set_qos_class_np(&spawnAttributes, QOS_CLASS_BACKGROUND) + } + + // Setup cwd + let intendedWorkingDir = self.workingDirectory.string + let chdirError: Int32 = intendedWorkingDir.withPlatformString { workDir in + return posix_spawn_file_actions_addchdir_np(&fileActions, workDir) + } + + // Error handling + if chdirError != 0 || spawnAttributeError != 0 { + try self.cleanupPreSpawn(input: inputPipe, output: outputPipe, error: errorPipe) + if spawnAttributeError != 0 { + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: spawnAttributeError) + ) + } + + if chdirError != 0 { + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: spawnAttributeError) + ) + } + } + // Run additional config + if let spawnConfig = self.platformOptions.preSpawnProcessConfigurator { + try spawnConfig(&spawnAttributes, &fileActions) + } + + // Spawn + let spawnError: CInt = possibleExecutablePath.withCString { exePath in + return supplementaryGroups.withOptionalUnsafeBufferPointer { sgroups in + return _subprocess_spawn( + &pid, + exePath, + &fileActions, + &spawnAttributes, + argv, + env, + uidPtr, + gidPtr, + Int32(supplementaryGroups?.count ?? 0), + sgroups?.baseAddress, + self.platformOptions.createSession ? 1 : 0 + ) + } + } + // Spawn error + if spawnError != 0 { + if spawnError == ENOENT { + // Move on to another possible path + continue + } + // Throw all other errors + try self.cleanupPreSpawn( + input: inputPipe, + output: outputPipe, + error: errorPipe + ) + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: spawnError) + ) + } + return Execution( + processIdentifier: .init(value: pid), + output: output, + error: error, + outputPipe: outputPipe, + errorPipe: errorPipe + ) + } + + // If we reach this point, it means either the executable path + // or working directory is not valid. Since posix_spawn does not + // provide which one is not valid, here we make a best effort guess + // by checking whether the working directory is valid. This technically + // still causes TOUTOC issue, but it's the best we can do for error recovery. + try self.cleanupPreSpawn(input: inputPipe, output: outputPipe, error: errorPipe) + let workingDirectory = self.workingDirectory.string + guard Configuration.pathAccessible(workingDirectory, mode: F_OK) else { + throw SubprocessError( + code: .init(.failedToChangeWorkingDirectory(workingDirectory)), + underlyingError: .init(rawValue: ENOENT) + ) + } + throw SubprocessError( + code: .init(.executableNotFound(self.executable.description)), + underlyingError: .init(rawValue: ENOENT) + ) + } + } +} + +// Special keys used in Error's user dictionary +extension String { + static let debugDescriptionErrorKey = "NSDebugDescription" +} + // MARK: - Process Monitoring @Sendable internal func monitorProcessTermination( - forProcessWithIdentifier pid: Subprocess.ProcessIdentifier -) async throws -> Subprocess.TerminationStatus { + forProcessWithIdentifier pid: ProcessIdentifier +) async throws -> TerminationStatus { return try await withCheckedThrowingContinuation { continuation in let source = DispatchSource.makeProcessSource( identifier: pid.value, @@ -329,7 +399,12 @@ internal func monitorProcessTermination( var siginfo = siginfo_t() let rc = waitid(P_PID, id_t(pid.value), &siginfo, WEXITED) guard rc == 0 else { - continuation.resume(throwing: POSIXError(.init(rawValue: errno) ?? .ENODEV)) + continuation.resume( + throwing: SubprocessError( + code: .init(.failedToMonitorProcess), + underlyingError: .init(rawValue: errno) + ) + ) return } switch siginfo.si_code { @@ -350,4 +425,4 @@ internal func monitorProcessTermination( } } -#endif // canImport(Darwin) +#endif // canImport(Darwin) diff --git a/Sources/_Subprocess/Platforms/Subprocess+Linux.swift b/Sources/_Subprocess/Platforms/Subprocess+Linux.swift index 9debf2e3..23c9b36e 100644 --- a/Sources/_Subprocess/Platforms/Subprocess+Linux.swift +++ b/Sources/_Subprocess/Platforms/Subprocess+Linux.swift @@ -2,196 +2,210 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors -// Licensed under Apache License v2.0 +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception // -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift.org project authors -// -// SPDX-License-Identifier: Apache-2.0 +// See https://swift.org/LICENSE.txt for license information // //===----------------------------------------------------------------------===// -#if canImport(Glibc) +#if canImport(Glibc) || canImport(Bionic) || canImport(Musl) + +#if canImport(System) +import System +#else +@preconcurrency import SystemPackage +#endif +#if canImport(Glibc) import Glibc -import Dispatch -import SystemPackage -import FoundationEssentials -import _CShims +#elseif canImport(Bionic) +import Bionic +#elseif canImport(Musl) +import Musl +#endif -// Linux specific implementations -extension Subprocess.Configuration { - internal typealias StringOrRawBytes = Subprocess.StringOrRawBytes +internal import Dispatch - internal func spawn( - withInput input: Subprocess.ExecutionInput, - output: Subprocess.ExecutionOutput, - error: Subprocess.ExecutionOutput - ) throws -> Subprocess { +import Synchronization +import _SubprocessCShims + +// Linux specific implementations +extension Configuration { + internal func spawn< + Output: OutputProtocol, + Error: OutputProtocol + >( + withInput inputPipe: CreatedPipe, + output: Output, + outputPipe: CreatedPipe, + error: Error, + errorPipe: CreatedPipe + ) throws -> Execution { _setupMonitorSignalHandler() - let (executablePath, - env, argv, - intendedWorkingDir, - uidPtr, gidPtr, - supplementaryGroups - ) = try self.preSpawn() - var processGroupIDPtr: UnsafeMutablePointer? = nil - if let processGroupID = self.platformOptions.processGroupID { - processGroupIDPtr = .allocate(capacity: 1) - processGroupIDPtr?.pointee = gid_t(processGroupID) - } - defer { - for ptr in env { ptr?.deallocate() } - for ptr in argv { ptr?.deallocate() } - uidPtr?.deallocate() - gidPtr?.deallocate() - processGroupIDPtr?.deallocate() - } + // Instead of checking if every possible executable path + // is valid, spawn each directly and catch ENOENT + let possiblePaths = self.executable.possibleExecutablePaths( + withPathValue: self.environment.pathValue() + ) - let fileDescriptors: [CInt] = [ - input.getReadFileDescriptor()?.rawValue ?? -1, - input.getWriteFileDescriptor()?.rawValue ?? -1, - output.getWriteFileDescriptor()?.rawValue ?? -1, - output.getReadFileDescriptor()?.rawValue ?? -1, - error.getWriteFileDescriptor()?.rawValue ?? -1, - error.getReadFileDescriptor()?.rawValue ?? -1 - ] + return try self.preSpawn { args throws -> Execution in + let (env, uidPtr, gidPtr, supplementaryGroups) = args - var workingDirectory: String? - if intendedWorkingDir != FilePath.currentWorkingDirectory { - // Only pass in working directory if it's different - workingDirectory = intendedWorkingDir.string - } - // Spawn - var pid: pid_t = 0 - let spawnError: CInt = executablePath.withCString { exePath in - return workingDirectory.withOptionalCString { workingDir in - return supplementaryGroups.withOptionalUnsafeBufferPointer { sgroups in - return fileDescriptors.withUnsafeBufferPointer { fds in - return _subprocess_fork_exec( - &pid, exePath, workingDir, - fds.baseAddress!, - argv, env, - uidPtr, gidPtr, - processGroupIDPtr, - CInt(supplementaryGroups?.count ?? 0), sgroups?.baseAddress, - self.platformOptions.createSession ? 1 : 0, - self.platformOptions.preSpawnProcessConfigurator - ) + for possibleExecutablePath in possiblePaths { + var processGroupIDPtr: UnsafeMutablePointer? = nil + if let processGroupID = self.platformOptions.processGroupID { + processGroupIDPtr = .allocate(capacity: 1) + processGroupIDPtr?.pointee = gid_t(processGroupID) + } + // Setup Arguments + let argv: [UnsafeMutablePointer?] = self.arguments.createArgs( + withExecutablePath: possibleExecutablePath + ) + defer { + for ptr in argv { ptr?.deallocate() } + } + // Setup input + let fileDescriptors: [CInt] = [ + inputPipe.readFileDescriptor?.wrapped.rawValue ?? -1, + inputPipe.writeFileDescriptor?.wrapped.rawValue ?? -1, + outputPipe.writeFileDescriptor?.wrapped.rawValue ?? -1, + outputPipe.readFileDescriptor?.wrapped.rawValue ?? -1, + errorPipe.writeFileDescriptor?.wrapped.rawValue ?? -1, + errorPipe.readFileDescriptor?.wrapped.rawValue ?? -1, + ] + + let workingDirectory: String = self.workingDirectory.string + // Spawn + var pid: pid_t = 0 + let spawnError: CInt = possibleExecutablePath.withCString { exePath in + return workingDirectory.withCString { workingDir in + return supplementaryGroups.withOptionalUnsafeBufferPointer { sgroups in + return fileDescriptors.withUnsafeBufferPointer { fds in + return _subprocess_fork_exec( + &pid, + exePath, + workingDir, + fds.baseAddress!, + argv, + env, + uidPtr, + gidPtr, + processGroupIDPtr, + CInt(supplementaryGroups?.count ?? 0), + sgroups?.baseAddress, + self.platformOptions.createSession ? 1 : 0, + self.platformOptions.preSpawnProcessConfigurator + ) + } + } } } + // Spawn error + if spawnError != 0 { + if spawnError == ENOENT { + // Move on to another possible path + continue + } + // Throw all other errors + try self.cleanupPreSpawn( + input: inputPipe, + output: outputPipe, + error: errorPipe + ) + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: spawnError) + ) + } + return Execution( + processIdentifier: .init(value: pid), + output: output, + error: error, + outputPipe: outputPipe, + errorPipe: errorPipe + ) } + + // If we reach this point, it means either the executable path + // or working directory is not valid. Since posix_spawn does not + // provide which one is not valid, here we make a best effort guess + // by checking whether the working directory is valid. This technically + // still causes TOUTOC issue, but it's the best we can do for error recovery. + try self.cleanupPreSpawn(input: inputPipe, output: outputPipe, error: errorPipe) + let workingDirectory = self.workingDirectory.string + guard Configuration.pathAccessible(workingDirectory, mode: F_OK) else { + throw SubprocessError( + code: .init(.failedToChangeWorkingDirectory(workingDirectory)), + underlyingError: .init(rawValue: ENOENT) + ) + } + throw SubprocessError( + code: .init(.executableNotFound(self.executable.description)), + underlyingError: .init(rawValue: ENOENT) + ) } - // Spawn error - if spawnError != 0 { - try self.cleanupAll(input: input, output: output, error: error) - throw POSIXError(.init(rawValue: spawnError) ?? .ENODEV) - } - return Subprocess( - processIdentifier: .init(value: pid), - executionInput: input, - executionOutput: output, - executionError: error - ) } } // MARK: - Platform Specific Options -extension Subprocess { - /// The collection of platform-specific settings - /// to configure the subprocess when running - public struct PlatformOptions: Sendable { - // Set user ID for the subprocess - public var userID: uid_t? = nil - /// Set the real and effective group ID and the saved - /// set-group-ID of the subprocess, equivalent to calling - /// `setgid()` on the child process. - /// Group ID is used to control permissions, particularly - /// for file access. - public var groupID: gid_t? = nil - // Set list of supplementary group IDs for the subprocess - public var supplementaryGroups: [gid_t]? = nil - /// Set the process group for the subprocess, equivalent to - /// calling `setpgid()` on the child process. - /// Process group ID is used to group related processes for - /// controlling signals. - public var processGroupID: pid_t? = nil - // Creates a session and sets the process group ID - // i.e. Detach from the terminal. - public var createSession: Bool = false - /// An ordered list of steps in order to tear down the child - /// process in case the parent task is cancelled before - /// the child proces terminates. - /// Always ends in sending a `.kill` signal at the end. - public var teardownSequence: [TeardownStep] = [] - /// A closure to configure platform-specific - /// spawning constructs. This closure enables direct - /// configuration or override of underlying platform-specific - /// spawn settings that `Subprocess` utilizes internally, - /// in cases where Subprocess does not provide higher-level - /// APIs for such modifications. - /// - /// On Linux, Subprocess uses `fork/exec` as the - /// underlying spawning mechanism. This closure is called - /// after `fork()` but before `exec()`. You may use it to - /// call any necessary process setup functions. - public var preSpawnProcessConfigurator: (@convention(c) @Sendable () -> Void)? = nil - public init() {} - } -} - -extension Subprocess.PlatformOptions: Hashable { - public static func ==( - lhs: Subprocess.PlatformOptions, - rhs: Subprocess.PlatformOptions - ) -> Bool { - // Since we can't compare closure equality, - // as long as preSpawnProcessConfigurator is set - // always returns false so that `PlatformOptions` - // with it set will never equal to each other - if lhs.preSpawnProcessConfigurator != nil || - rhs.preSpawnProcessConfigurator != nil { - return false - } - return lhs.userID == rhs.userID && - lhs.groupID == rhs.groupID && - lhs.supplementaryGroups == rhs.supplementaryGroups && - lhs.processGroupID == rhs.processGroupID && - lhs.createSession == rhs.createSession - } +/// The collection of platform-specific settings +/// to configure the subprocess when running +public struct PlatformOptions: Sendable { + // Set user ID for the subprocess + public var userID: uid_t? = nil + /// Set the real and effective group ID and the saved + /// set-group-ID of the subprocess, equivalent to calling + /// `setgid()` on the child process. + /// Group ID is used to control permissions, particularly + /// for file access. + public var groupID: gid_t? = nil + // Set list of supplementary group IDs for the subprocess + public var supplementaryGroups: [gid_t]? = nil + /// Set the process group for the subprocess, equivalent to + /// calling `setpgid()` on the child process. + /// Process group ID is used to group related processes for + /// controlling signals. + public var processGroupID: pid_t? = nil + // Creates a session and sets the process group ID + // i.e. Detach from the terminal. + public var createSession: Bool = false + /// An ordered list of steps in order to tear down the child + /// process in case the parent task is cancelled before + /// the child proces terminates. + /// Always ends in sending a `.kill` signal at the end. + public var teardownSequence: [TeardownStep] = [] + /// A closure to configure platform-specific + /// spawning constructs. This closure enables direct + /// configuration or override of underlying platform-specific + /// spawn settings that `Subprocess` utilizes internally, + /// in cases where Subprocess does not provide higher-level + /// APIs for such modifications. + /// + /// On Linux, Subprocess uses `fork/exec` as the + /// underlying spawning mechanism. This closure is called + /// after `fork()` but before `exec()`. You may use it to + /// call any necessary process setup functions. + public var preSpawnProcessConfigurator: (@convention(c) @Sendable () -> Void)? = nil - public func hash(into hasher: inout Hasher) { - hasher.combine(userID) - hasher.combine(groupID) - hasher.combine(supplementaryGroups) - hasher.combine(processGroupID) - hasher.combine(createSession) - // Since we can't really hash closures, - // use an UUID such that as long as - // `preSpawnProcessConfigurator` is set, it will - // never equal to other PlatformOptions - if self.preSpawnProcessConfigurator != nil { - hasher.combine(UUID()) - } - } + public init() {} } -extension Subprocess.PlatformOptions : CustomStringConvertible, CustomDebugStringConvertible { +extension PlatformOptions: CustomStringConvertible, CustomDebugStringConvertible { internal func description(withIndent indent: Int) -> String { let indent = String(repeating: " ", count: indent * 4) return """ -PlatformOptions( -\(indent) userID: \(String(describing: userID)), -\(indent) groupID: \(String(describing: groupID)), -\(indent) supplementaryGroups: \(String(describing: supplementaryGroups)), -\(indent) processGroupID: \(String(describing: processGroupID)), -\(indent) createSession: \(createSession), -\(indent) preSpawnProcessConfigurator: \(self.preSpawnProcessConfigurator == nil ? "not set" : "set") -\(indent)) -""" + PlatformOptions( + \(indent) userID: \(String(describing: userID)), + \(indent) groupID: \(String(describing: groupID)), + \(indent) supplementaryGroups: \(String(describing: supplementaryGroups)), + \(indent) processGroupID: \(String(describing: processGroupID)), + \(indent) createSession: \(createSession), + \(indent) preSpawnProcessConfigurator: \(self.preSpawnProcessConfigurator == nil ? "not set" : "set") + \(indent)) + """ } public var description: String { @@ -211,12 +225,13 @@ extension String { // MARK: - Process Monitoring @Sendable internal func monitorProcessTermination( - forProcessWithIdentifier pid: Subprocess.ProcessIdentifier -) async throws -> Subprocess.TerminationStatus { + forProcessWithIdentifier pid: ProcessIdentifier +) async throws -> TerminationStatus { return try await withCheckedThrowingContinuation { continuation in _childProcessContinuations.withLock { continuations in if let existing = continuations.removeValue(forKey: pid.value), - case .status(let existingStatus) = existing { + case .status(let existingStatus) = existing + { // We already have existing status to report continuation.resume(returning: existingStatus) } else { @@ -228,28 +243,26 @@ internal func monitorProcessTermination( } private enum ContinuationOrStatus { - case continuation(CheckedContinuation) - case status(Subprocess.TerminationStatus) + case continuation(CheckedContinuation) + case status(TerminationStatus) } -private let _childProcessContinuations: LockedState< - [pid_t: ContinuationOrStatus] -> = LockedState(initialState: [:]) +private let _childProcessContinuations: + Mutex< + [pid_t: ContinuationOrStatus] + > = Mutex([:]) + +private let signalSource: SendableSourceSignal = SendableSourceSignal() -private var signalSource: (any DispatchSourceSignal)? = nil private let setup: () = { - signalSource = DispatchSource.makeSignalSource( - signal: SIGCHLD, - queue: .global() - ) - signalSource?.setEventHandler { + signalSource.setEventHandler { _childProcessContinuations.withLock { continuations in while true { var siginfo = siginfo_t() guard waitid(P_ALL, id_t(0), &siginfo, WEXITED) == 0 else { return } - var status: Subprocess.TerminationStatus? = nil + var status: TerminationStatus? = nil switch siginfo.si_code { case .init(CLD_EXITED): status = .exited(siginfo._sifields._sigchld.si_status) @@ -265,7 +278,8 @@ private let setup: () = { if let status = status { let pid = siginfo._sifields._sigchld.si_pid if let existing = continuations.removeValue(forKey: pid), - case .continuation(let c) = existing { + case .continuation(let c) = existing + { c.resume(returning: status) } else { // We don't have continuation yet, just state status @@ -275,13 +289,33 @@ private let setup: () = { } } } - signalSource?.resume() + signalSource.resume() }() +/// Unchecked Sendable here since this class is only explicitly +/// initialzied once during the lifetime of the process +final class SendableSourceSignal: @unchecked Sendable { + private let signalSource: DispatchSourceSignal + + func setEventHandler(handler: @escaping DispatchSourceHandler) { + self.signalSource.setEventHandler(handler: handler) + } + + func resume() { + self.signalSource.resume() + } + + init() { + self.signalSource = DispatchSource.makeSignalSource( + signal: SIGCHLD, + queue: .global() + ) + } +} + private func _setupMonitorSignalHandler() { // Only executed once setup } -#endif // canImport(Glibc) - +#endif // canImport(Glibc) || canImport(Bionic) || canImport(Musl) diff --git a/Sources/_Subprocess/Platforms/Subprocess+Unix.swift b/Sources/_Subprocess/Platforms/Subprocess+Unix.swift index bd110301..ae8fd639 100644 --- a/Sources/_Subprocess/Platforms/Subprocess+Unix.swift +++ b/Sources/_Subprocess/Platforms/Subprocess+Unix.swift @@ -2,119 +2,143 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors -// Licensed under Apache License v2.0 +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception // -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift.org project authors -// -// SPDX-License-Identifier: Apache-2.0 +// See https://swift.org/LICENSE.txt for license information // //===----------------------------------------------------------------------===// -#if canImport(Darwin) || canImport(Glibc) +#if canImport(Darwin) || canImport(Glibc) || canImport(Bionic) || canImport(Musl) -#if canImport(FoundationEssentials) -import FoundationEssentials -#elseif canImport(Foundation) -import Foundation +#if canImport(System) +import System +#else +@preconcurrency import SystemPackage #endif +import _SubprocessCShims + #if canImport(Darwin) import Darwin +#elseif canImport(Bionic) +import Bionic #elseif canImport(Glibc) import Glibc +#elseif canImport(Musl) +import Musl #endif -#if FOUNDATION_FRAMEWORK -@_implementationOnly import _FoundationCShims -#else -import _CShims -#endif - -import Dispatch -import SystemPackage +package import Dispatch // MARK: - Signals -extension Subprocess { - /// Signals are standardized messages sent to a running program - /// to trigger specific behavior, such as quitting or error handling. - public struct Signal : Hashable, Sendable { - /// The underlying platform specific value for the signal - public let rawValue: Int32 - - private init(rawValue: Int32) { - self.rawValue = rawValue - } - /// The `.interrupt` signal is sent to a process by its - /// controlling terminal when a user wishes to interrupt - /// the process. - public static var interrupt: Self { .init(rawValue: SIGINT) } - /// The `.terminate` signal is sent to a process to request its - /// termination. Unlike the `.kill` signal, it can be caught - /// and interpreted or ignored by the process. This allows - /// the process to perform nice termination releasing resources - /// and saving state if appropriate. `.interrupt` is nearly - /// identical to `.terminate`. - public static var terminate: Self { .init(rawValue: SIGTERM) } - /// The `.suspend` signal instructs the operating system - /// to stop a process for later resumption. - public static var suspend: Self { .init(rawValue: SIGSTOP) } - /// The `resume` signal instructs the operating system to - /// continue (restart) a process previously paused by the - /// `.suspend` signal. - public static var resume: Self { .init(rawValue: SIGCONT) } - /// The `.kill` signal is sent to a process to cause it to - /// terminate immediately (kill). In contrast to `.terminate` - /// and `.interrupt`, this signal cannot be caught or ignored, - /// and the receiving process cannot perform any - /// clean-up upon receiving this signal. - public static var kill: Self { .init(rawValue: SIGKILL) } - /// The `.terminalClosed` signal is sent to a process when - /// its controlling terminal is closed. In modern systems, - /// this signal usually means that the controlling pseudo - /// or virtual terminal has been closed. - public static var terminalClosed: Self { .init(rawValue: SIGHUP) } - /// The `.quit` signal is sent to a process by its controlling - /// terminal when the user requests that the process quit - /// and perform a core dump. - public static var quit: Self { .init(rawValue: SIGQUIT) } - /// The `.userDefinedOne` signal is sent to a process to indicate - /// user-defined conditions. - public static var userDefinedOne: Self { .init(rawValue: SIGUSR1) } - /// The `.userDefinedTwo` signal is sent to a process to indicate - /// user-defined conditions. - public static var userDefinedTwo: Self { .init(rawValue: SIGUSR2) } - /// The `.alarm` signal is sent to a process when the corresponding - /// time limit is reached. - public static var alarm: Self { .init(rawValue: SIGALRM) } - /// The `.windowSizeChange` signal is sent to a process when - /// its controlling terminal changes its size (a window change). - public static var windowSizeChange: Self { .init(rawValue: SIGWINCH) } +/// Signals are standardized messages sent to a running program +/// to trigger specific behavior, such as quitting or error handling. +public struct Signal: Hashable, Sendable { + /// The underlying platform specific value for the signal + public let rawValue: Int32 + + private init(rawValue: Int32) { + self.rawValue = rawValue + } + + /// The `.interrupt` signal is sent to a process by its + /// controlling terminal when a user wishes to interrupt + /// the process. + public static var interrupt: Self { .init(rawValue: SIGINT) } + /// The `.terminate` signal is sent to a process to request its + /// termination. Unlike the `.kill` signal, it can be caught + /// and interpreted or ignored by the process. This allows + /// the process to perform nice termination releasing resources + /// and saving state if appropriate. `.interrupt` is nearly + /// identical to `.terminate`. + public static var terminate: Self { .init(rawValue: SIGTERM) } + /// The `.suspend` signal instructs the operating system + /// to stop a process for later resumption. + public static var suspend: Self { .init(rawValue: SIGSTOP) } + /// The `resume` signal instructs the operating system to + /// continue (restart) a process previously paused by the + /// `.suspend` signal. + public static var resume: Self { .init(rawValue: SIGCONT) } + /// The `.kill` signal is sent to a process to cause it to + /// terminate immediately (kill). In contrast to `.terminate` + /// and `.interrupt`, this signal cannot be caught or ignored, + /// and the receiving process cannot perform any + /// clean-up upon receiving this signal. + public static var kill: Self { .init(rawValue: SIGKILL) } + /// The `.terminalClosed` signal is sent to a process when + /// its controlling terminal is closed. In modern systems, + /// this signal usually means that the controlling pseudo + /// or virtual terminal has been closed. + public static var terminalClosed: Self { .init(rawValue: SIGHUP) } + /// The `.quit` signal is sent to a process by its controlling + /// terminal when the user requests that the process quit + /// and perform a core dump. + public static var quit: Self { .init(rawValue: SIGQUIT) } + /// The `.userDefinedOne` signal is sent to a process to indicate + /// user-defined conditions. + public static var userDefinedOne: Self { .init(rawValue: SIGUSR1) } + /// The `.userDefinedTwo` signal is sent to a process to indicate + /// user-defined conditions. + public static var userDefinedTwo: Self { .init(rawValue: SIGUSR2) } + /// The `.alarm` signal is sent to a process when the corresponding + /// time limit is reached. + public static var alarm: Self { .init(rawValue: SIGALRM) } + /// The `.windowSizeChange` signal is sent to a process when + /// its controlling terminal changes its size (a window change). + public static var windowSizeChange: Self { .init(rawValue: SIGWINCH) } +} + +// MARK: - ProcessIdentifier + +/// A platform independent identifier for a Subprocess. +public struct ProcessIdentifier: Sendable, Hashable, Codable { + /// The platform specific process identifier value + public let value: pid_t + + public init(value: pid_t) { + self.value = value } +} +extension ProcessIdentifier: CustomStringConvertible, CustomDebugStringConvertible { + public var description: String { "\(self.value)" } + + public var debugDescription: String { "\(self.value)" } +} + +@available(macOS 15.0, *) // FIXME: manually added availability +extension Execution { /// Send the given signal to the child process. /// - Parameters: /// - signal: The signal to send. /// - shouldSendToProcessGroup: Whether this signal should be sent to /// the entire process group. - public func send(_ signal: Signal, toProcessGroup shouldSendToProcessGroup: Bool) throws { + public func send( + signal: Signal, + toProcessGroup shouldSendToProcessGroup: Bool = false + ) throws { let pid = shouldSendToProcessGroup ? -(self.processIdentifier.value) : self.processIdentifier.value guard kill(pid, signal.rawValue) == 0 else { - throw POSIXError(.init(rawValue: errno)!) + throw SubprocessError( + code: .init(.failedToSendSignal(signal.rawValue)), + underlyingError: .init(rawValue: errno) + ) } } - internal func tryTerminate() -> Error? { + internal func tryTerminate() -> Swift.Error? { do { - try self.send(.kill, toProcessGroup: true) + try self.send(signal: .kill) } catch { - guard let posixError: POSIXError = error as? POSIXError else { + guard let posixError: SubprocessError = error as? SubprocessError else { return error } // Ignore ESRCH (no such process) - if posixError.code != .ESRCH { + if let underlyingError = posixError.underlyingError, + underlyingError.rawValue != ESRCH + { return error } } @@ -123,21 +147,32 @@ extension Subprocess { } // MARK: - Environment Resolution -extension Subprocess.Environment { - internal static let pathEnvironmentVariableName = "PATH" +extension Environment { + internal static let pathVariableName = "PATH" internal func pathValue() -> String? { switch self.config { case .inherit(let overrides): // If PATH value exists in overrides, use it - if let value = overrides[.string(Self.pathEnvironmentVariableName)] { - return value.stringValue + if let value = overrides[Self.pathVariableName] { + return value } // Fall back to current process - return ProcessInfo.processInfo.environment[Self.pathEnvironmentVariableName] + return Self.currentEnvironmentValues()[Self.pathVariableName] case .custom(let fullEnvironment): - if let value = fullEnvironment[.string(Self.pathEnvironmentVariableName)] { - return value.stringValue + if let value = fullEnvironment[Self.pathVariableName] { + return value + } + return nil + case .rawBytes(let rawBytesArray): + let needle: [UInt8] = Array("\(Self.pathVariableName)=".utf8) + for row in rawBytesArray { + guard row.starts(with: needle) else { + continue + } + // Attempt to + let pathValue = row.dropFirst(needle.count) + return String(decoding: pathValue, as: UTF8.self) } return nil } @@ -147,8 +182,8 @@ extension Subprocess.Environment { // manually deallocated internal func createEnv() -> [UnsafeMutablePointer?] { func createFullCString( - fromKey keyContainer: Subprocess.StringOrRawBytes, - value valueContainer: Subprocess.StringOrRawBytes + fromKey keyContainer: StringOrRawBytes, + value valueContainer: StringOrRawBytes ) -> UnsafeMutablePointer { let rawByteKey: UnsafeMutablePointer = keyContainer.createRawBytes() let rawByteValue: UnsafeMutablePointer = valueContainer.createRawBytes() @@ -170,21 +205,12 @@ extension Subprocess.Environment { var env: [UnsafeMutablePointer?] = [] switch self.config { case .inherit(let updates): - var current = ProcessInfo.processInfo.environment - for (keyContainer, valueContainer) in updates { - if let stringKey = keyContainer.stringValue { - // Remove the value from current to override it - current.removeValue(forKey: stringKey) - } - // Fast path - if case .string(let stringKey) = keyContainer, - case .string(let stringValue) = valueContainer { - let fullString = "\(stringKey)=\(stringValue)" - env.append(strdup(fullString)) - continue - } - - env.append(createFullCString(fromKey: keyContainer, value: valueContainer)) + var current = Self.currentEnvironmentValues() + for (key, value) in updates { + // Remove the value from current to override it + current.removeValue(forKey: key) + let fullString = "\(key)=\(value)" + env.append(strdup(fullString)) } // Add the rest of `current` to env for (key, value) in current { @@ -192,24 +218,43 @@ extension Subprocess.Environment { env.append(strdup(fullString)) } case .custom(let customValues): - for (keyContainer, valueContainer) in customValues { - // Fast path - if case .string(let stringKey) = keyContainer, - case .string(let stringValue) = valueContainer { - let fullString = "\(stringKey)=\(stringValue)" - env.append(strdup(fullString)) - continue - } - env.append(createFullCString(fromKey: keyContainer, value: valueContainer)) + for (key, value) in customValues { + let fullString = "\(key)=\(value)" + env.append(strdup(fullString)) + } + case .rawBytes(let rawBytesArray): + for rawBytes in rawBytesArray { + env.append(strdup(rawBytes)) } } env.append(nil) return env } + + internal static func withCopiedEnv(_ body: ([UnsafeMutablePointer]) -> R) -> R { + var values: [UnsafeMutablePointer] = [] + // This lock is taken by calls to getenv, so we want as few callouts to other code as possible here. + _subprocess_lock_environ() + guard + let environments: UnsafeMutablePointer?> = + _subprocess_get_environ() + else { + _subprocess_unlock_environ() + return body([]) + } + var curr = environments + while let value = curr.pointee { + values.append(strdup(value)) + curr = curr.advanced(by: 1) + } + _subprocess_unlock_environ() + defer { values.forEach { free($0) } } + return body(values) + } } // MARK: Args Creation -extension Subprocess.Arguments { +extension Arguments { // This method follows the standard "create" rule: `args` needs to be // manually deallocated internal func createArgs(withExecutablePath executablePath: String) -> [UnsafeMutablePointer?] { @@ -225,42 +270,23 @@ extension Subprocess.Arguments { } } -// MARK: - ProcessIdentifier -extension Subprocess { - /// A platform independent identifier for a subprocess. - public struct ProcessIdentifier: Sendable, Hashable, Codable { - /// The platform specific process identifier value - public let value: pid_t - - public init(value: pid_t) { - self.value = value - } - } -} - -extension Subprocess.ProcessIdentifier : CustomStringConvertible, CustomDebugStringConvertible { - public var description: String { "\(self.value)" } - - public var debugDescription: String { "\(self.value)" } -} - // MARK: - Executable Searching -extension Subprocess.Executable { +extension Executable { internal static var defaultSearchPaths: Set { return Set([ "/usr/bin", "/bin", "/usr/sbin", "/sbin", - "/usr/local/bin" + "/usr/local/bin", ]) } - internal func resolveExecutablePath(withPathValue pathValue: String?) -> String? { + internal func resolveExecutablePath(withPathValue pathValue: String?) throws -> String { switch self.storage { case .executable(let executableName): // If the executableName in is already a full path, return it directly - if Subprocess.Configuration.pathAccessible(executableName, mode: X_OK) { + if Configuration.pathAccessible(executableName, mode: X_OK) { return executableName } // Get $PATH from environment @@ -274,50 +300,38 @@ extension Subprocess.Executable { for path in searchPaths { let fullPath = "\(path)/\(executableName)" - let fileExists = Subprocess.Configuration.pathAccessible(fullPath, mode: X_OK) + let fileExists = Configuration.pathAccessible(fullPath, mode: X_OK) if fileExists { return fullPath } } + throw SubprocessError( + code: .init(.executableNotFound(executableName)), + underlyingError: nil + ) case .path(let executablePath): // Use path directly return executablePath.string } - return nil } } -// MARK: - Configuration -extension Subprocess.Configuration { - internal func preSpawn() throws -> ( - executablePath: String, +// MARK: - PreSpawn +extension Configuration { + internal typealias PreSpawnArgs = ( env: [UnsafeMutablePointer?], - argv: [UnsafeMutablePointer?], - intendedWorkingDir: FilePath, uidPtr: UnsafeMutablePointer?, gidPtr: UnsafeMutablePointer?, supplementaryGroups: [gid_t]? - ) { + ) + + internal func preSpawn( + _ work: (PreSpawnArgs) throws -> Result + ) throws -> Result { // Prepare environment let env = self.environment.createEnv() - // Prepare executable path - guard let executablePath = self.executable.resolveExecutablePath( - withPathValue: self.environment.pathValue()) else { - for ptr in env { ptr?.deallocate() } - throw CocoaError(.executableNotLoadable, userInfo: [ - .debugDescriptionErrorKey : "\(self.executable.description) is not an executable" - ]) - } - // Prepare arguments - let argv: [UnsafeMutablePointer?] = self.arguments.createArgs(withExecutablePath: executablePath) - // Prepare workingDir - let intendedWorkingDir = self.workingDirectory - guard Self.pathAccessible(intendedWorkingDir.string, mode: F_OK) else { + defer { for ptr in env { ptr?.deallocate() } - for ptr in argv { ptr?.deallocate() } - throw CocoaError(.fileNoSuchFile, userInfo: [ - .debugDescriptionErrorKey : "Failed to set working directory to \(intendedWorkingDir)" - ]) } var uidPtr: UnsafeMutablePointer? = nil @@ -325,21 +339,28 @@ extension Subprocess.Configuration { uidPtr = .allocate(capacity: 1) uidPtr?.pointee = userID } + defer { + uidPtr?.deallocate() + } var gidPtr: UnsafeMutablePointer? = nil if let groupID = self.platformOptions.groupID { gidPtr = .allocate(capacity: 1) gidPtr?.pointee = groupID } + defer { + gidPtr?.deallocate() + } var supplementaryGroups: [gid_t]? if let groupsValue = self.platformOptions.supplementaryGroups { supplementaryGroups = groupsValue } - return ( - executablePath: executablePath, - env: env, argv: argv, - intendedWorkingDir: intendedWorkingDir, - uidPtr: uidPtr, gidPtr: gidPtr, - supplementaryGroups: supplementaryGroups + return try work( + ( + env: env, + uidPtr: uidPtr, + gidPtr: gidPtr, + supplementaryGroups: supplementaryGroups + ) ) } @@ -359,11 +380,11 @@ extension FileDescriptor { return devnull } - internal var platformDescriptor: Subprocess.PlatformFileDescriptor { + internal var platformDescriptor: PlatformFileDescriptor { return self } - internal func readChunk(upToLength maxLength: Int) async throws -> Data? { + package func readChunk(upToLength maxLength: Int) async throws -> SequenceOutput.Buffer? { return try await withCheckedThrowingContinuation { continuation in DispatchIO.read( fromFileDescriptor: self.rawValue, @@ -371,88 +392,125 @@ extension FileDescriptor { runningHandlerOn: .global() ) { data, error in if error != 0 { - continuation.resume(throwing: POSIXError(.init(rawValue: error) ?? .ENODEV)) + continuation.resume( + throwing: SubprocessError( + code: .init(.failedToReadFromSubprocess), + underlyingError: .init(rawValue: error) + ) + ) return } if data.isEmpty { continuation.resume(returning: nil) } else { - continuation.resume(returning: Data(data)) + continuation.resume(returning: SequenceOutput.Buffer(data: data)) } } } } - internal func readUntilEOF(upToLength maxLength: Int) async throws -> Data { - return try await withCheckedThrowingContinuation { continuation in - let dispatchIO = DispatchIO( - type: .stream, - fileDescriptor: self.rawValue, - queue: .global() - ) { error in - if error != 0 { - continuation.resume(throwing: POSIXError(.init(rawValue: error) ?? .ENODEV)) - } + internal func readUntilEOF( + upToLength maxLength: Int, + resultHandler: sending @escaping (Swift.Result) -> Void + ) { + let dispatchIO = DispatchIO( + type: .stream, + fileDescriptor: self.rawValue, + queue: .global() + ) { error in } + var buffer: DispatchData? + dispatchIO.read( + offset: 0, + length: maxLength, + queue: .global() + ) { done, data, error in + guard error == 0, let chunkData = data else { + dispatchIO.close() + resultHandler( + .failure( + SubprocessError( + code: .init(.failedToReadFromSubprocess), + underlyingError: .init(rawValue: error) + ) + ) + ) + return } - var buffer: Data = Data() - dispatchIO.read( - offset: 0, - length: maxLength, - queue: .global() - ) { done, data, error in - guard error == 0 else { - continuation.resume(throwing: POSIXError(.init(rawValue: error) ?? .ENODEV)) - return - } - if let data = data { - buffer += Data(data) - } - if done { - dispatchIO.close() - continuation.resume(returning: buffer) - } + // Easy case: if we are done and buffer is nil, this means + // there is only one chunk of data + if done && buffer == nil { + dispatchIO.close() + buffer = chunkData + resultHandler(.success(chunkData)) + return + } + + if buffer == nil { + buffer = chunkData + } else { + buffer?.append(chunkData) + } + + if done { + dispatchIO.close() + resultHandler(.success(buffer!)) + return } } } - internal func write(_ data: S) async throws where S.Element == UInt8 { - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) -> Void in - let dispatchData: DispatchData = Array(data).withUnsafeBytes { - return DispatchData(bytes: $0) - } - DispatchIO.write( - toFileDescriptor: self.rawValue, - data: dispatchData, - runningHandlerOn: .global() - ) { _, error in - guard error == 0 else { - continuation.resume( - throwing: POSIXError( - .init(rawValue: error) ?? .ENODEV) + package func write( + _ array: [UInt8] + ) async throws -> Int { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + let dispatchData = array.withUnsafeBytes { + return DispatchData( + bytesNoCopy: $0, + deallocator: .custom( + nil, + { + // noop + } ) - return + ) + } + self.write(dispatchData) { writtenLength, error in + if let error = error { + continuation.resume(throwing: error) + } else { + continuation.resume(returning: writtenLength) } - continuation.resume() } } } -} -extension Subprocess { - internal typealias PlatformFileDescriptor = FileDescriptor -} - -// MARK: - Read Buffer Size -extension Subprocess { - @inline(__always) - internal static var readBufferSize: Int { -#if canImport(Darwin) - return 16384 -#else - // FIXME: Use Platform.pageSize here - return 4096 -#endif // canImport(Darwin) + package func write( + _ dispatchData: DispatchData, + queue: DispatchQueue = .global(), + completion: @escaping (Int, Error?) -> Void + ) { + DispatchIO.write( + toFileDescriptor: self.rawValue, + data: dispatchData, + runningHandlerOn: queue + ) { unwritten, error in + let unwrittenLength = unwritten?.count ?? 0 + let writtenLength = dispatchData.count - unwrittenLength + guard error != 0 else { + completion(writtenLength, nil) + return + } + completion( + writtenLength, + SubprocessError( + code: .init(.failedToWriteToSubprocess), + underlyingError: .init(rawValue: error) + ) + ) + } } } -#endif // canImport(Darwin) || canImport(Glibc) +internal typealias PlatformFileDescriptor = FileDescriptor + +#endif // canImport(Darwin) || canImport(Glibc) || canImport(Bionic) || canImport(Musl) diff --git a/Sources/_Subprocess/Platforms/Subprocess+Windows.swift b/Sources/_Subprocess/Platforms/Subprocess+Windows.swift index 978cb139..aaf53ea0 100644 --- a/Sources/_Subprocess/Platforms/Subprocess+Windows.swift +++ b/Sources/_Subprocess/Platforms/Subprocess+Windows.swift @@ -2,53 +2,66 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors -// Licensed under Apache License v2.0 +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception // -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift.org project authors -// -// SPDX-License-Identifier: Apache-2.0 +// See https://swift.org/LICENSE.txt for license information // //===----------------------------------------------------------------------===// #if canImport(WinSDK) import WinSDK -import Dispatch -import SystemPackage -import FoundationEssentials +internal import Dispatch +#if canImport(System) +import System +#else +@preconcurrency import SystemPackage +#endif // Windows specific implementation -extension Subprocess.Configuration { - internal func spawn( - withInput input: Subprocess.ExecutionInput, - output: Subprocess.ExecutionOutput, - error: Subprocess.ExecutionOutput - ) throws -> Subprocess { +extension Configuration { + internal func spawn< + Output: OutputProtocol, + Error: OutputProtocol + >( + withInput inputPipe: CreatedPipe, + output: Output, + outputPipe: CreatedPipe, + error: Error, + errorPipe: CreatedPipe + ) throws -> Execution { // Spawn differently depending on whether // we need to spawn as a user - if let userCredentials = self.platformOptions.userCredentials { - return try self.spawnAsUser( - withInput: input, - output: output, - error: error, - userCredentials: userCredentials - ) - } else { + guard let userCredentials = self.platformOptions.userCredentials else { return try self.spawnDirect( - withInput: input, + withInput: inputPipe, output: output, - error: error + outputPipe: outputPipe, + error: error, + errorPipe: errorPipe ) } + return try self.spawnAsUser( + withInput: inputPipe, + output: output, + outputPipe: outputPipe, + error: error, + errorPipe: errorPipe, + userCredentials: userCredentials + ) } - internal func spawnDirect( - withInput input: Subprocess.ExecutionInput, - output: Subprocess.ExecutionOutput, - error: Subprocess.ExecutionOutput - ) throws -> Subprocess { + internal func spawnDirect< + Output: OutputProtocol, + Error: OutputProtocol + >( + withInput inputPipe: CreatedPipe, + output: Output, + outputPipe: CreatedPipe, + error: Error, + errorPipe: CreatedPipe + ) throws -> Execution { let ( applicationName, commandAndArgs, @@ -56,9 +69,9 @@ extension Subprocess.Configuration { intendedWorkingDir ) = try self.preSpawn() var startupInfo = try self.generateStartupInfo( - withInput: input, - output: output, - error: error + withInput: inputPipe, + output: outputPipe, + error: errorPipe ) var processInfo: PROCESS_INFORMATION = PROCESS_INFORMATION() var createProcessFlags = self.generateCreateProcessFlag() @@ -78,8 +91,8 @@ extension Subprocess.Configuration { let created = CreateProcessW( applicationNameW, UnsafeMutablePointer(mutating: commandAndArgsW), - nil, // lpProcessAttributes - nil, // lpThreadAttributes + nil, // lpProcessAttributes + nil, // lpThreadAttributes true, // bInheritHandles createProcessFlags, UnsafeMutableRawPointer(mutating: environmentW), @@ -89,14 +102,14 @@ extension Subprocess.Configuration { ) guard created else { let windowsError = GetLastError() - try self.cleanupAll( - input: input, - output: output, - error: error + try self.cleanupPreSpawn( + input: inputPipe, + output: outputPipe, + error: errorPipe ) - throw CocoaError.windowsError( - underlying: windowsError, - errorCode: .fileWriteUnknown + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: windowsError) ) } } @@ -106,47 +119,52 @@ extension Subprocess.Configuration { // We don't need the handle objects, so close it right away guard CloseHandle(processInfo.hThread) else { let windowsError = GetLastError() - try self.cleanupAll( - input: input, - output: output, - error: error + try self.cleanupPreSpawn( + input: inputPipe, + output: outputPipe, + error: errorPipe ) - throw CocoaError.windowsError( - underlying: windowsError, - errorCode: .fileReadUnknown + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: windowsError) ) } guard CloseHandle(processInfo.hProcess) else { let windowsError = GetLastError() - try self.cleanupAll( - input: input, - output: output, - error: error + try self.cleanupPreSpawn( + input: inputPipe, + output: outputPipe, + error: errorPipe ) - throw CocoaError.windowsError( - underlying: windowsError, - errorCode: .fileReadUnknown + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: windowsError) ) } - let pid = Subprocess.ProcessIdentifier( - processID: processInfo.dwProcessId, - threadID: processInfo.dwThreadId + let pid = ProcessIdentifier( + value: processInfo.dwProcessId ) - return Subprocess( + return Execution( processIdentifier: pid, - executionInput: input, - executionOutput: output, - executionError: error, + output: output, + error: error, + outputPipe: outputPipe, + errorPipe: errorPipe, consoleBehavior: self.platformOptions.consoleBehavior ) } - internal func spawnAsUser( - withInput input: Subprocess.ExecutionInput, - output: Subprocess.ExecutionOutput, - error: Subprocess.ExecutionOutput, - userCredentials: Subprocess.PlatformOptions.UserCredentials - ) throws -> Subprocess { + internal func spawnAsUser< + Output: OutputProtocol, + Error: OutputProtocol + >( + withInput inputPipe: CreatedPipe, + output: Output, + outputPipe: CreatedPipe, + error: Error, + errorPipe: CreatedPipe, + userCredentials: PlatformOptions.UserCredentials + ) throws -> Execution { let ( applicationName, commandAndArgs, @@ -154,9 +172,9 @@ extension Subprocess.Configuration { intendedWorkingDir ) = try self.preSpawn() var startupInfo = try self.generateStartupInfo( - withInput: input, - output: output, - error: error + withInput: inputPipe, + output: outputPipe, + error: errorPipe ) var processInfo: PROCESS_INFORMATION = PROCESS_INFORMATION() var createProcessFlags = self.generateCreateProcessFlag() @@ -197,14 +215,14 @@ extension Subprocess.Configuration { ) guard created else { let windowsError = GetLastError() - try self.cleanupAll( - input: input, - output: output, - error: error + try self.cleanupPreSpawn( + input: inputPipe, + output: outputPipe, + error: errorPipe ) - throw CocoaError.windowsError( - underlying: windowsError, - errorCode: .fileWriteUnknown + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: windowsError) ) } } @@ -217,201 +235,175 @@ extension Subprocess.Configuration { // We don't need the handle objects, so close it right away guard CloseHandle(processInfo.hThread) else { let windowsError = GetLastError() - try self.cleanupAll( - input: input, - output: output, - error: error + try self.cleanupPreSpawn( + input: inputPipe, + output: outputPipe, + error: errorPipe ) - throw CocoaError.windowsError( - underlying: windowsError, - errorCode: .fileReadUnknown + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: windowsError) ) } guard CloseHandle(processInfo.hProcess) else { let windowsError = GetLastError() - try self.cleanupAll( - input: input, - output: output, - error: error + try self.cleanupPreSpawn( + input: inputPipe, + output: outputPipe, + error: errorPipe ) - throw CocoaError.windowsError( - underlying: windowsError, - errorCode: .fileReadUnknown + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: windowsError) ) } - let pid = Subprocess.ProcessIdentifier( - processID: processInfo.dwProcessId, - threadID: processInfo.dwThreadId + let pid = ProcessIdentifier( + value: processInfo.dwProcessId ) - return Subprocess( + return Execution( processIdentifier: pid, - executionInput: input, - executionOutput: output, - executionError: error, + output: output, + error: error, + outputPipe: outputPipe, + errorPipe: errorPipe, consoleBehavior: self.platformOptions.consoleBehavior ) } } // MARK: - Platform Specific Options -extension Subprocess { - /// The collection of platform-specific settings - /// to configure the subprocess when running - public struct PlatformOptions: Sendable { - /// A `UserCredentials` to use spawning the subprocess - /// as a different user - public struct UserCredentials: Sendable, Hashable { - // The name of the user. This is the name - // of the user account to run as. - public var username: String - // The clear-text password for the account. - public var password: String - // The name of the domain or server whose account database - // contains the account. - public var domain: String? - } - /// `ConsoleBehavior` defines how should the console appear - /// when spawning a new process - public struct ConsoleBehavior: Sendable, Hashable { - internal enum Storage: Sendable, Hashable { - case createNew - case detatch - case inherit - } +/// The collection of platform-specific settings +/// to configure the subprocess when running +public struct PlatformOptions: Sendable { + /// A `UserCredentials` to use spawning the subprocess + /// as a different user + public struct UserCredentials: Sendable, Hashable { + // The name of the user. This is the name + // of the user account to run as. + public var username: String + // The clear-text password for the account. + public var password: String + // The name of the domain or server whose account database + // contains the account. + public var domain: String? + } - internal let storage: Storage + /// `ConsoleBehavior` defines how should the console appear + /// when spawning a new process + public struct ConsoleBehavior: Sendable, Hashable { + internal enum Storage: Sendable, Hashable { + case createNew + case detatch + case inherit + } - private init(_ storage: Storage) { - self.storage = storage - } + internal let storage: Storage - /// The subprocess has a new console, instead of - /// inheriting its parent's console (the default). - public static let createNew: Self = .init(.createNew) - /// For console processes, the new process does not - /// inherit its parent's console (the default). - /// The new process can call the `AllocConsole` - /// function at a later time to create a console. - public static let detatch: Self = .init(.detatch) - /// The subprocess inherits its parent's console. - public static let inherit: Self = .init(.inherit) + private init(_ storage: Storage) { + self.storage = storage } - /// `ConsoleBehavior` defines how should the window appear - /// when spawning a new process - public struct WindowStyle: Sendable, Hashable { - internal enum Storage: Sendable, Hashable { - case normal - case hidden - case maximized - case minimized - } + /// The subprocess has a new console, instead of + /// inheriting its parent's console (the default). + public static let createNew: Self = .init(.createNew) + /// For console processes, the new process does not + /// inherit its parent's console (the default). + /// The new process can call the `AllocConsole` + /// function at a later time to create a console. + public static let detatch: Self = .init(.detatch) + /// The subprocess inherits its parent's console. + public static let inherit: Self = .init(.inherit) + } - internal let storage: Storage + /// `ConsoleBehavior` defines how should the window appear + /// when spawning a new process + public struct WindowStyle: Sendable, Hashable { + internal enum Storage: Sendable, Hashable { + case normal + case hidden + case maximized + case minimized + } - internal var platformStyle: WORD { - switch self.storage { - case .hidden: return WORD(SW_HIDE) - case .maximized: return WORD(SW_SHOWMAXIMIZED) - case .minimized: return WORD(SW_SHOWMINIMIZED) - default: return WORD(SW_SHOWNORMAL) - } - } + internal let storage: Storage - private init(_ storage: Storage) { - self.storage = storage + internal var platformStyle: WORD { + switch self.storage { + case .hidden: return WORD(SW_HIDE) + case .maximized: return WORD(SW_SHOWMAXIMIZED) + case .minimized: return WORD(SW_SHOWMINIMIZED) + default: return WORD(SW_SHOWNORMAL) } + } - /// Activates and displays a window of normal size - public static let normal: Self = .init(.normal) - /// Does not activate a new window - public static let hidden: Self = .init(.hidden) - /// Activates the window and displays it as a maximized window. - public static let maximized: Self = .init(.maximized) - /// Activates the window and displays it as a minimized window. - public static let minimized: Self = .init(.minimized) + private init(_ storage: Storage) { + self.storage = storage } - /// Sets user credentials when starting the process as another user - public var userCredentials: UserCredentials? = nil - /// The console behavior of the new process, - /// default to inheriting the console from parent process - public var consoleBehavior: ConsoleBehavior = .inherit - /// Window style to use when the process is started - public var windowStyle: WindowStyle = .normal - /// Whether to create a new process group for the new - /// process. The process group includes all processes - /// that are descendants of this root process. - /// The process identifier of the new process group - /// is the same as the process identifier. - public var createProcessGroup: Bool = false - /// A closure to configure platform-specific - /// spawning constructs. This closure enables direct - /// configuration or override of underlying platform-specific - /// spawn settings that `Subprocess` utilizes internally, - /// in cases where Subprocess does not provide higher-level - /// APIs for such modifications. - /// - /// On Windows, Subprocess uses `CreateProcessW()` as the - /// underlying spawning mechanism. This closure allows - /// modification of the `dwCreationFlags` creation flag - /// and startup info `STARTUPINFOW` before - /// they are sent to `CreateProcessW()`. - public var preSpawnProcessConfigurator: ( + /// Activates and displays a window of normal size + public static let normal: Self = .init(.normal) + /// Does not activate a new window + public static let hidden: Self = .init(.hidden) + /// Activates the window and displays it as a maximized window. + public static let maximized: Self = .init(.maximized) + /// Activates the window and displays it as a minimized window. + public static let minimized: Self = .init(.minimized) + } + + /// Sets user credentials when starting the process as another user + public var userCredentials: UserCredentials? = nil + /// The console behavior of the new process, + /// default to inheriting the console from parent process + public var consoleBehavior: ConsoleBehavior = .inherit + /// Window style to use when the process is started + public var windowStyle: WindowStyle = .normal + /// Whether to create a new process group for the new + /// process. The process group includes all processes + /// that are descendants of this root process. + /// The process identifier of the new process group + /// is the same as the process identifier. + public var createProcessGroup: Bool = false + /// An ordered list of steps in order to tear down the child + /// process in case the parent task is cancelled before + /// the child proces terminates. + /// Always ends in forcefully terminate at the end. + public var teardownSequence: [TeardownStep] = [] + /// A closure to configure platform-specific + /// spawning constructs. This closure enables direct + /// configuration or override of underlying platform-specific + /// spawn settings that `Subprocess` utilizes internally, + /// in cases where Subprocess does not provide higher-level + /// APIs for such modifications. + /// + /// On Windows, Subprocess uses `CreateProcessW()` as the + /// underlying spawning mechanism. This closure allows + /// modification of the `dwCreationFlags` creation flag + /// and startup info `STARTUPINFOW` before + /// they are sent to `CreateProcessW()`. + public var preSpawnProcessConfigurator: + ( @Sendable ( inout DWORD, inout STARTUPINFOW ) throws -> Void )? = nil - public init() {} - } -} - -extension Subprocess.PlatformOptions: Hashable { - public static func == ( - lhs: Subprocess.PlatformOptions, - rhs: Subprocess.PlatformOptions - ) -> Bool { - // Since we can't compare closure equality, - // as long as preSpawnProcessConfigurator is set - // always returns false so that `PlatformOptions` - // with it set will never equal to each other - if lhs.preSpawnProcessConfigurator != nil || rhs.preSpawnProcessConfigurator != nil { - return false - } - return lhs.userCredentials == rhs.userCredentials && lhs.consoleBehavior == rhs.consoleBehavior && lhs.windowStyle == rhs.windowStyle && - lhs.createProcessGroup == rhs.createProcessGroup - } - - public func hash(into hasher: inout Hasher) { - hasher.combine(userCredentials) - hasher.combine(consoleBehavior) - hasher.combine(windowStyle) - hasher.combine(createProcessGroup) - // Since we can't really hash closures, - // use an UUID such that as long as - // `preSpawnProcessConfigurator` is set, it will - // never equal to other PlatformOptions - if self.preSpawnProcessConfigurator != nil { - hasher.combine(UUID()) - } - } + public init() {} } -extension Subprocess.PlatformOptions : CustomStringConvertible, CustomDebugStringConvertible { +extension PlatformOptions: CustomStringConvertible, CustomDebugStringConvertible { internal func description(withIndent indent: Int) -> String { let indent = String(repeating: " ", count: indent * 4) return """ -PlatformOptions( -\(indent) userCredentials: \(String(describing: self.userCredentials)), -\(indent) consoleBehavior: \(String(describing: self.consoleBehavior)), -\(indent) windowStyle: \(String(describing: self.windowStyle)), -\(indent) createProcessGroup: \(self.createProcessGroup), -\(indent) preSpawnProcessConfigurator: \(self.preSpawnProcessConfigurator == nil ? "not set" : "set") -\(indent)) -""" + PlatformOptions( + \(indent) userCredentials: \(String(describing: self.userCredentials)), + \(indent) consoleBehavior: \(String(describing: self.consoleBehavior)), + \(indent) windowStyle: \(String(describing: self.windowStyle)), + \(indent) createProcessGroup: \(self.createProcessGroup), + \(indent) preSpawnProcessConfigurator: \(self.preSpawnProcessConfigurator == nil ? "not set" : "set") + \(indent)) + """ } public var description: String { @@ -426,8 +418,8 @@ PlatformOptions( // MARK: - Process Monitoring @Sendable internal func monitorProcessTermination( - forProcessWithIdentifier pid: Subprocess.ProcessIdentifier -) async throws -> Subprocess.TerminationStatus { + forProcessWithIdentifier pid: ProcessIdentifier +) async throws -> TerminationStatus { // Once the continuation resumes, it will need to unregister the wait, so // yield the wait handle back to the calling scope. var waitHandle: HANDLE? @@ -436,11 +428,13 @@ internal func monitorProcessTermination( _ = UnregisterWait(waitHandle) } } - guard let processHandle = OpenProcess( - DWORD(PROCESS_QUERY_INFORMATION | SYNCHRONIZE), - false, - pid.processID - ) else { + guard + let processHandle = OpenProcess( + DWORD(PROCESS_QUERY_INFORMATION | SYNCHRONIZE), + false, + pid.value + ) + else { return .exited(1) } @@ -449,19 +443,29 @@ internal func monitorProcessTermination( // other work. let context = Unmanaged.passRetained(continuation as AnyObject).toOpaque() let callback: WAITORTIMERCALLBACK = { context, _ in - let continuation = Unmanaged.fromOpaque(context!).takeRetainedValue() as! CheckedContinuation + let continuation = + Unmanaged.fromOpaque(context!).takeRetainedValue() as! CheckedContinuation continuation.resume() } // We only want the callback to fire once (and not be rescheduled.) Waiting // may take an arbitrarily long time, so let the thread pool know that too. let flags = ULONG(WT_EXECUTEONLYONCE | WT_EXECUTELONGFUNCTION) - guard RegisterWaitForSingleObject( - &waitHandle, processHandle, callback, context, INFINITE, flags - ) else { - continuation.resume(throwing: CocoaError.windowsError( - underlying: GetLastError(), - errorCode: .fileWriteUnknown) + guard + RegisterWaitForSingleObject( + &waitHandle, + processHandle, + callback, + context, + INFINITE, + flags + ) + else { + continuation.resume( + throwing: SubprocessError( + code: .init(.failedToMonitorProcess), + underlyingError: .init(rawValue: GetLastError()) + ) ) return } @@ -474,58 +478,62 @@ internal func monitorProcessTermination( return .exited(1) } let exitCodeValue = CInt(bitPattern: .init(status)) - if exitCodeValue >= 0 { - return .exited(status) - } else { + guard exitCodeValue >= 0 else { return .unhandledException(status) } + return .exited(status) } // MARK: - Subprocess Control -extension Subprocess { +@available(macOS 15.0, *) // FIXME: manually added availability +extension Execution { /// Terminate the current subprocess with the given exit code /// - Parameter exitCode: The exit code to use for the subprocess. public func terminate(withExitCode exitCode: DWORD) throws { - guard let processHandle = OpenProcess( - // PROCESS_ALL_ACCESS - DWORD(STANDARD_RIGHTS_REQUIRED | SYNCHRONIZE | 0xFFFF), - false, - self.processIdentifier.processID - ) else { - throw CocoaError.windowsError( - underlying: GetLastError(), - errorCode: .fileWriteUnknown + guard + let processHandle = OpenProcess( + // PROCESS_ALL_ACCESS + DWORD(STANDARD_RIGHTS_REQUIRED | SYNCHRONIZE | 0xFFFF), + false, + self.processIdentifier.value + ) + else { + throw SubprocessError( + code: .init(.failedToTerminate), + underlyingError: .init(rawValue: GetLastError()) ) } defer { CloseHandle(processHandle) } guard TerminateProcess(processHandle, exitCode) else { - throw CocoaError.windowsError( - underlying: GetLastError(), - errorCode: .fileWriteUnknown + throw SubprocessError( + code: .init(.failedToTerminate), + underlyingError: .init(rawValue: GetLastError()) ) } } /// Suspend the current subprocess public func suspend() throws { - guard let processHandle = OpenProcess( - // PROCESS_ALL_ACCESS - DWORD(STANDARD_RIGHTS_REQUIRED | SYNCHRONIZE | 0xFFFF), - false, - self.processIdentifier.processID - ) else { - throw CocoaError.windowsError( - underlying: GetLastError(), - errorCode: .fileWriteUnknown + guard + let processHandle = OpenProcess( + // PROCESS_ALL_ACCESS + DWORD(STANDARD_RIGHTS_REQUIRED | SYNCHRONIZE | 0xFFFF), + false, + self.processIdentifier.value + ) + else { + throw SubprocessError( + code: .init(.failedToSuspend), + underlyingError: .init(rawValue: GetLastError()) ) } defer { CloseHandle(processHandle) } - let NTSuspendProcess: Optional<(@convention(c) (HANDLE) -> LONG)> = + let NTSuspendProcess: (@convention(c) (HANDLE) -> LONG)? = unsafeBitCast( GetProcAddress( GetModuleHandleA("ntdll.dll"), @@ -534,53 +542,61 @@ extension Subprocess { to: Optional<(@convention(c) (HANDLE) -> LONG)>.self ) guard let NTSuspendProcess = NTSuspendProcess else { - throw CocoaError(.executableNotLoadable) + throw SubprocessError( + code: .init(.failedToSuspend), + underlyingError: .init(rawValue: GetLastError()) + ) } guard NTSuspendProcess(processHandle) >= 0 else { - throw CocoaError.windowsError( - underlying: GetLastError(), - errorCode: .fileWriteUnknown + throw SubprocessError( + code: .init(.failedToSuspend), + underlyingError: .init(rawValue: GetLastError()) ) } } /// Resume the current subprocess after suspension public func resume() throws { - guard let processHandle = OpenProcess( - // PROCESS_ALL_ACCESS - DWORD(STANDARD_RIGHTS_REQUIRED | SYNCHRONIZE | 0xFFFF), - false, - self.processIdentifier.processID - ) else { - throw CocoaError.windowsError( - underlying: GetLastError(), - errorCode: .fileWriteUnknown + guard + let processHandle = OpenProcess( + // PROCESS_ALL_ACCESS + DWORD(STANDARD_RIGHTS_REQUIRED | SYNCHRONIZE | 0xFFFF), + false, + self.processIdentifier.value + ) + else { + throw SubprocessError( + code: .init(.failedToResume), + underlyingError: .init(rawValue: GetLastError()) ) } defer { CloseHandle(processHandle) } - let NTResumeProcess: Optional<(@convention(c) (HANDLE) -> LONG)> = - unsafeBitCast( - GetProcAddress( - GetModuleHandleA("ntdll.dll"), - "NtResumeProcess" - ), - to: Optional<(@convention(c) (HANDLE) -> LONG)>.self - ) + let NTResumeProcess: (@convention(c) (HANDLE) -> LONG)? = + unsafeBitCast( + GetProcAddress( + GetModuleHandleA("ntdll.dll"), + "NtResumeProcess" + ), + to: Optional<(@convention(c) (HANDLE) -> LONG)>.self + ) guard let NTResumeProcess = NTResumeProcess else { - throw CocoaError(.executableNotLoadable) + throw SubprocessError( + code: .init(.failedToResume), + underlyingError: .init(rawValue: GetLastError()) + ) } guard NTResumeProcess(processHandle) >= 0 else { - throw CocoaError.windowsError( - underlying: GetLastError(), - errorCode: .fileWriteUnknown + throw SubprocessError( + code: .init(.failedToResume), + underlyingError: .init(rawValue: GetLastError()) ) } } - internal func tryTerminate() -> Error? { + internal func tryTerminate() -> Swift.Error? { do { try self.terminate(withExitCode: 0) } catch { @@ -591,35 +607,44 @@ extension Subprocess { } // MARK: - Executable Searching -extension Subprocess.Executable { +extension Executable { // Technically not needed for CreateProcess since // it takes process name. It's here to support // Executable.resolveExecutablePath - internal func resolveExecutablePath(withPathValue pathValue: String?) -> String? { + internal func resolveExecutablePath(withPathValue pathValue: String?) throws -> String { switch self.storage { case .executable(let executableName): - return executableName.withCString( + return try executableName.withCString( encodedAs: UTF16.self - ) { exeName -> String? in - return pathValue.withOptionalCString( + ) { exeName -> String in + return try pathValue.withOptionalCString( encodedAs: UTF16.self - ) { path -> String? in + ) { path -> String in let pathLenth = SearchPathW( path, exeName, - nil, 0, nil, nil + nil, + 0, + nil, + nil ) guard pathLenth > 0 else { - return nil + throw SubprocessError( + code: .init(.executableNotFound(executableName)), + underlyingError: .init(rawValue: GetLastError()) + ) } return withUnsafeTemporaryAllocation( - of: WCHAR.self, capacity: Int(pathLenth) + 1 + of: WCHAR.self, + capacity: Int(pathLenth) + 1 ) { _ = SearchPathW( path, - exeName, nil, + exeName, + nil, pathLenth + 1, - $0.baseAddress, nil + $0.baseAddress, + nil ) return String(decodingCString: $0.baseAddress!, as: UTF16.self) } @@ -633,49 +658,60 @@ extension Subprocess.Executable { } // MARK: - Environment Resolution -extension Subprocess.Environment { - internal static let pathEnvironmentVariableName = "Path" +extension Environment { + internal static let pathVariableName = "Path" internal func pathValue() -> String? { switch self.config { case .inherit(let overrides): // If PATH value exists in overrides, use it - if let value = overrides[.string(Self.pathEnvironmentVariableName)] { - return value.stringValue + if let value = overrides[Self.pathVariableName] { + return value } // Fall back to current process - return ProcessInfo.processInfo.environment[Self.pathEnvironmentVariableName] + return Self.currentEnvironmentValues()[Self.pathVariableName] case .custom(let fullEnvironment): - if let value = fullEnvironment[.string(Self.pathEnvironmentVariableName)] { - return value.stringValue + if let value = fullEnvironment[Self.pathVariableName] { + return value } return nil } } + + internal static func withCopiedEnv(_ body: ([UnsafeMutablePointer]) -> R) -> R { + var values: [UnsafeMutablePointer] = [] + guard let pwszEnvironmentBlock = GetEnvironmentStringsW() else { + return body([]) + } + defer { FreeEnvironmentStringsW(pwszEnvironmentBlock) } + + var pwszEnvironmentEntry: LPWCH? = pwszEnvironmentBlock + while let value = pwszEnvironmentEntry { + let entry = String(decodingCString: value, as: UTF16.self) + if entry.isEmpty { break } + values.append(entry.withCString { _strdup($0)! }) + pwszEnvironmentEntry = pwszEnvironmentEntry?.advanced(by: wcslen(value) + 1) + } + defer { values.forEach { free($0) } } + return body(values) + } } // MARK: - ProcessIdentifier -extension Subprocess { - /// A platform independent identifier for a subprocess. - public struct ProcessIdentifier: Sendable, Hashable, Codable { - /// Windows specifc process identifier value - public let processID: DWORD - /// Windows specific thread identifier associated with process - public let threadID: DWORD - - internal init( - processID: DWORD, - threadID: DWORD - ) { - self.processID = processID - self.threadID = threadID - } + +/// A platform independent identifier for a subprocess. +public struct ProcessIdentifier: Sendable, Hashable, Codable { + /// Windows specifc process identifier value + public let value: DWORD + + internal init(value: DWORD) { + self.value = value } } -extension Subprocess.ProcessIdentifier: CustomStringConvertible, CustomDebugStringConvertible { +extension ProcessIdentifier: CustomStringConvertible, CustomDebugStringConvertible { public var description: String { - return "(processID: \(self.processID), threadID: \(self.threadID))" + return "(processID: \(self.value))" } public var debugDescription: String { @@ -684,7 +720,7 @@ extension Subprocess.ProcessIdentifier: CustomStringConvertible, CustomDebugStri } // MARK: - Private Utils -extension Subprocess.Configuration { +extension Configuration { private func preSpawn() throws -> ( applicationName: String?, commandAndArgs: String, @@ -692,44 +728,33 @@ extension Subprocess.Configuration { intendedWorkingDir: String ) { // Prepare environment - var env: [String : String] = [:] + var env: [String: String] = [:] switch self.environment.config { case .custom(let customValues): // Use the custom values directly - for customKey in customValues.keys { - guard case .string(let stringKey) = customKey, - let valueContainer = customValues[customKey], - case .string(let stringValue) = valueContainer else { - fatalError("Windows does not support non unicode String as environments") - } - env.updateValue(stringValue, forKey: stringKey) - } + env = customValues case .inherit(let updateValues): // Combine current environment - env = ProcessInfo.processInfo.environment - for updatingKey in updateValues.keys { - // Override the current environment values - guard case .string(let stringKey) = updatingKey, - let valueContainer = updateValues[updatingKey], - case .string(let stringValue) = valueContainer else { - fatalError("Windows does not support non unicode String as environments") - } - env.updateValue(stringValue, forKey: stringKey) + env = Environment.currentEnvironmentValues() + for (key, value) in updateValues { + env.updateValue(value, forKey: key) } } // On Windows, the PATH is required in order to locate dlls needed by // the process so we should also pass that to the child - let pathVariableName = Subprocess.Environment.pathEnvironmentVariableName + let pathVariableName = Environment.pathVariableName if env[pathVariableName] == nil, - let parentPath = ProcessInfo.processInfo.environment[pathVariableName] { + let parentPath = Environment.currentEnvironmentValues()[pathVariableName] + { env[pathVariableName] = parentPath } // The environment string must be terminated by a double // null-terminator. Otherwise, CreateProcess will fail with // INVALID_PARMETER. - let environmentString = env.map { - $0.key + "=" + $0.value - }.joined(separator: "\0") + "\0\0" + let environmentString = + env.map { + $0.key + "=" + $0.value + }.joined(separator: "\0") + "\0\0" // Prepare arguments let ( @@ -738,9 +763,12 @@ extension Subprocess.Configuration { ) = try self.generateWindowsCommandAndAgruments() // Validate workingDir guard Self.pathAccessible(self.workingDirectory.string) else { - throw CocoaError(.fileNoSuchFile, userInfo: [ - .debugDescriptionErrorKey : "Failed to set working directory to \(self.workingDirectory)" - ]) + throw SubprocessError( + code: .init( + .failedToChangeWorkingDirectory(self.workingDirectory.string) + ), + underlyingError: nil + ) } return ( applicationName: applicationName, @@ -767,9 +795,9 @@ extension Subprocess.Configuration { } private func generateStartupInfo( - withInput input: Subprocess.ExecutionInput, - output: Subprocess.ExecutionOutput, - error: Subprocess.ExecutionOutput + withInput input: CreatedPipe, + output: CreatedPipe, + error: CreatedPipe ) throws -> STARTUPINFOW { var info: STARTUPINFOW = STARTUPINFOW() info.cb = DWORD(MemoryLayout.size) @@ -781,10 +809,10 @@ extension Subprocess.Configuration { } // Bind IOs // Input - if let inputRead = input.getReadFileDescriptor() { + if let inputRead = input.readFileDescriptor { info.hStdInput = inputRead.platformDescriptor } - if let inputWrite = input.getWriteFileDescriptor() { + if let inputWrite = input.writeFileDescriptor { // Set parent side to be uninhertable SetHandleInformation( inputWrite.platformDescriptor, @@ -793,10 +821,10 @@ extension Subprocess.Configuration { ) } // Output - if let outputWrite = output.getWriteFileDescriptor() { + if let outputWrite = output.writeFileDescriptor { info.hStdOutput = outputWrite.platformDescriptor } - if let outputRead = output.getReadFileDescriptor() { + if let outputRead = output.readFileDescriptor { // Set parent side to be uninhertable SetHandleInformation( outputRead.platformDescriptor, @@ -805,10 +833,10 @@ extension Subprocess.Configuration { ) } // Error - if let errorWrite = error.getWriteFileDescriptor() { + if let errorWrite = error.writeFileDescriptor { info.hStdError = errorWrite.platformDescriptor } - if let errorRead = error.getReadFileDescriptor() { + if let errorRead = error.readFileDescriptor { // Set parent side to be uninhertable SetHandleInformation( errorRead.platformDescriptor, @@ -834,12 +862,12 @@ extension Subprocess.Configuration { // actually resolve the path. However, to maintain // the same behavior as other platforms, still check // here to make sure the executable actually exists - guard self.executable.resolveExecutablePath( - withPathValue: self.environment.pathValue() - ) != nil else { - throw CocoaError(.executableNotLoadable, userInfo: [ - .debugDescriptionErrorKey : "\(self.executable.description) is not an executable" - ]) + do { + _ = try self.executable.resolveExecutablePath( + withPathValue: self.environment.pathValue() + ) + } catch { + throw error } executableNameOrPath = name } @@ -876,7 +904,7 @@ extension Subprocess.Configuration { func quoteWindowsCommandArg(arg: String) -> String { // Windows escaping, adapted from Daniel Colascione's "Everyone quotes // command line arguments the wrong way" - Microsoft Developer Blog - if !arg.contains(where: {" \t\n\"".contains($0)}) { + if !arg.contains(where: { " \t\n\"".contains($0) }) { return arg } @@ -911,7 +939,7 @@ extension Subprocess.Configuration { break } let backslashCount = unquoted.distance(from: unquoted.startIndex, to: firstNonBackslash) - if (unquoted[firstNonBackslash] == "\"") { + if unquoted[firstNonBackslash] == "\"" { // This is a string of \ followed by a " e.g. foo\"bar. Escape the // backslashes and the quote quoted.append(String(repeating: "\\", count: backslashCount * 2 + 1)) @@ -939,20 +967,7 @@ extension Subprocess.Configuration { } // MARK: - PlatformFileDescriptor Type -extension Subprocess { - internal typealias PlatformFileDescriptor = HANDLE -} - -// MARK: - Read Buffer Size -extension Subprocess { - @inline(__always) - internal static var readBufferSize: Int { - // FIXME: Use Platform.pageSize here - var sysInfo: SYSTEM_INFO = SYSTEM_INFO() - GetSystemInfo(&sysInfo) - return Int(sysInfo.dwPageSize) - } -} +internal typealias PlatformFileDescriptor = HANDLE // MARK: - Pipe Support extension FileDescriptor { @@ -968,13 +983,14 @@ extension FileDescriptor { var readHandle: HANDLE? = nil var writeHandle: HANDLE? = nil guard CreatePipe(&readHandle, &writeHandle, &saAttributes, 0), - readHandle != INVALID_HANDLE_VALUE, - writeHandle != INVALID_HANDLE_VALUE, - let readHandle: HANDLE = readHandle, - let writeHandle: HANDLE = writeHandle else { - throw CocoaError.windowsError( - underlying: GetLastError(), - errorCode: .fileReadUnknown + readHandle != INVALID_HANDLE_VALUE, + writeHandle != INVALID_HANDLE_VALUE, + let readHandle: HANDLE = readHandle, + let writeHandle: HANDLE = writeHandle + else { + throw SubprocessError( + code: .init(.failedToCreatePipe), + underlyingError: .init(rawValue: GetLastError()) ) } let readFd = _open_osfhandle( @@ -992,147 +1008,138 @@ extension FileDescriptor { ) } - internal static func openDevNull( - withAcessMode mode: FileDescriptor.AccessMode - ) throws -> FileDescriptor { - return try "NUL".withPlatformString { - let handle = CreateFileW( - $0, - DWORD(GENERIC_WRITE), - DWORD(FILE_SHARE_WRITE), - nil, - DWORD(OPEN_EXISTING), - DWORD(FILE_ATTRIBUTE_NORMAL), - nil - ) - guard let handle = handle, - handle != INVALID_HANDLE_VALUE else { - throw CocoaError.windowsError( - underlying: GetLastError(), - errorCode: .fileReadUnknown - ) - } - let devnull = _open_osfhandle( - intptr_t(bitPattern: handle), - mode.rawValue - ) - return FileDescriptor(rawValue: devnull) - } - } - - var platformDescriptor: Subprocess.PlatformFileDescriptor { + var platformDescriptor: PlatformFileDescriptor { return HANDLE(bitPattern: _get_osfhandle(self.rawValue))! } - internal func read(upToLength maxLength: Int) async throws -> Data { - // TODO: Figure out a better way to asynchornously read + internal func readChunk(upToLength maxLength: Int) async throws -> SequenceOutput.Buffer? { return try await withCheckedThrowingContinuation { continuation in - DispatchQueue.global(qos: .userInitiated).async { - var totalBytesRead: Int = 0 - var lastError: DWORD? = nil - let values = Array( - unsafeUninitializedCapacity: maxLength - ) { buffer, initializedCount in - while true { - guard let baseAddress = buffer.baseAddress else { - initializedCount = 0 - break - } - let bufferPtr = baseAddress.advanced(by: totalBytesRead) - var bytesRead: DWORD = 0 - let readSucceed = ReadFile( - self.platformDescriptor, - UnsafeMutableRawPointer(mutating: bufferPtr), - DWORD(maxLength - totalBytesRead), - &bytesRead, - nil - ) - if !readSucceed { - // Windows throws ERROR_BROKEN_PIPE when the pipe is closed - let error = GetLastError() - if error == ERROR_BROKEN_PIPE { - // We are done reading - initializedCount = totalBytesRead - } else { - // We got some error - lastError = error - initializedCount = 0 - } - break - } else { - // We succesfully read the current round - totalBytesRead += Int(bytesRead) - } - - if totalBytesRead >= maxLength { - initializedCount = min(maxLength, totalBytesRead) - break - } - } - } - if let lastError = lastError { - continuation.resume(throwing: CocoaError.windowsError( - underlying: lastError, - errorCode: .fileReadUnknown) - ) - } else { - continuation.resume(returning: Data(values)) + self.readUntilEOF( + upToLength: maxLength + ) { result in + switch result { + case .failure(let error): + continuation.resume(throwing: error) + case .success(let bytes): + continuation.resume(returning: SequenceOutput.Buffer(data: bytes)) } } } } - internal func write(_ data: S) async throws where S.Element == UInt8 { - // TODO: Figure out a better way to asynchornously write - try await withCheckedThrowingContinuation { ( - continuation: CheckedContinuation - ) -> Void in - DispatchQueue.global(qos: .userInitiated).async { - let buffer = Array(data) - buffer.withUnsafeBytes { ptr in - var writtenBytes: DWORD = 0 - let writeSucceed = WriteFile( + internal func readUntilEOF( + upToLength maxLength: Int, + resultHandler: @Sendable @escaping (Swift.Result<[UInt8], any (Error & Sendable)>) -> Void + ) { + DispatchQueue.global(qos: .userInitiated).async { + var totalBytesRead: Int = 0 + var lastError: DWORD? = nil + let values = [UInt8]( + unsafeUninitializedCapacity: maxLength + ) { buffer, initializedCount in + while true { + guard let baseAddress = buffer.baseAddress else { + initializedCount = 0 + break + } + let bufferPtr = baseAddress.advanced(by: totalBytesRead) + var bytesRead: DWORD = 0 + let readSucceed = ReadFile( self.platformDescriptor, - ptr.baseAddress, - DWORD(buffer.count), - &writtenBytes, + UnsafeMutableRawPointer(mutating: bufferPtr), + DWORD(maxLength - totalBytesRead), + &bytesRead, nil ) - if !writeSucceed { - continuation.resume(throwing: CocoaError.windowsError( - underlying: GetLastError(), - errorCode: .fileWriteUnknown) - ) + if !readSucceed { + // Windows throws ERROR_BROKEN_PIPE when the pipe is closed + let error = GetLastError() + if error == ERROR_BROKEN_PIPE { + // We are done reading + initializedCount = totalBytesRead + } else { + // We got some error + lastError = error + initializedCount = 0 + } + break } else { - continuation.resume() + // We succesfully read the current round + totalBytesRead += Int(bytesRead) + } + + if totalBytesRead >= maxLength { + initializedCount = min(maxLength, totalBytesRead) + break } } } + if let lastError = lastError { + let windowsError = SubprocessError( + code: .init(.failedToReadFromSubprocess), + underlyingError: .init(rawValue: lastError) + ) + resultHandler(.failure(windowsError)) + } else { + resultHandler(.success(values)) + } } } -} - -extension String { - static let debugDescriptionErrorKey = "DebugDescription" -} -// MARK: - CocoaError + Win32 -internal let NSUnderlyingErrorKey = "NSUnderlyingError" + internal func write( + _ array: [UInt8] + ) async throws -> Int { + try await withCheckedThrowingContinuation { continuation in + // TODO: Figure out a better way to asynchornously write + DispatchQueue.global(qos: .userInitiated).async { + array.withUnsafeBytes { + self.write($0) { writtenLength, error in + if let error = error { + continuation.resume(throwing: error) + } else { + continuation.resume(returning: writtenLength) + } + } + } + } + } + } -extension CocoaError { - static func windowsError(underlying: DWORD, errorCode: Code) -> CocoaError { - let userInfo = [ - NSUnderlyingErrorKey : Win32Error(underlying) - ] - return CocoaError(errorCode, userInfo: userInfo) + package func write( + _ ptr: UnsafeRawBufferPointer, + completion: @escaping (Int, Swift.Error?) -> Void + ) { + func _write( + _ ptr: UnsafeRawBufferPointer, + count: Int, + completion: @escaping (Int, Swift.Error?) -> Void + ) { + var writtenBytes: DWORD = 0 + let writeSucceed = WriteFile( + self.platformDescriptor, + ptr.baseAddress, + DWORD(count), + &writtenBytes, + nil + ) + if !writeSucceed { + let error = SubprocessError( + code: .init(.failedToWriteToSubprocess), + underlyingError: .init(rawValue: GetLastError()) + ) + completion(Int(writtenBytes), error) + } else { + completion(Int(writtenBytes), nil) + } + } } } -private extension Optional where Wrapped == String { - func withOptionalCString( +extension Optional where Wrapped == String { + fileprivate func withOptionalCString( encodedAs targetEncoding: Encoding.Type, _ body: (UnsafePointer?) throws -> Result - ) rethrows -> Result where Encoding : _UnicodeEncoding { + ) rethrows -> Result where Encoding: _UnicodeEncoding { switch self { case .none: return try body(nil) @@ -1141,7 +1148,7 @@ private extension Optional where Wrapped == String { } } - func withOptionalNTPathRepresentation( + fileprivate func withOptionalNTPathRepresentation( _ body: (UnsafePointer?) throws -> Result ) throws -> Result { switch self { @@ -1159,23 +1166,30 @@ extension String { _ body: (UnsafePointer) throws -> Result ) throws -> Result { guard !isEmpty else { - throw CocoaError(.fileReadInvalidFileName) + throw SubprocessError( + code: .init(.invalidWindowsPath(self)), + underlyingError: nil + ) } var iter = self.utf8.makeIterator() - let bLeadingSlash = if [._slash, ._backslash].contains(iter.next()), iter.next()?.isLetter ?? false, iter.next() == ._colon { true } else { false } + let bLeadingSlash = + if [._slash, ._backslash].contains(iter.next()), iter.next()?.isLetter ?? false, iter.next() == ._colon { + true + } else { false } // Strip the leading `/` on a RFC8089 path (`/[drive-letter]:/...` ). A // leading slash indicates a rooted path on the drive for the current // working directory. - return try Substring(self.utf8.dropFirst(bLeadingSlash ? 1 : 0)).withCString(encodedAs: UTF16.self) { pwszPath in + return try Substring(self.utf8.dropFirst(bLeadingSlash ? 1 : 0)).withCString(encodedAs: UTF16.self) { + pwszPath in // 1. Normalize the path first. let dwLength: DWORD = GetFullPathNameW(pwszPath, 0, nil, nil) return try withUnsafeTemporaryAllocation(of: WCHAR.self, capacity: Int(dwLength)) { guard GetFullPathNameW(pwszPath, DWORD($0.count), $0.baseAddress, nil) > 0 else { - throw CocoaError.windowsError( - underlying: GetLastError(), - errorCode: .fileReadUnknown + throw SubprocessError( + code: .init(.invalidWindowsPath(self)), + underlyingError: .init(rawValue: GetLastError()) ) } @@ -1186,27 +1200,23 @@ extension String { } } -struct Win32Error: Error { - public typealias Code = DWORD - public let code: Code - - public static var errorDomain: String { - return "NSWin32ErrorDomain" - } - - public init(_ code: Code) { - self.code = code - } -} - -internal extension UInt8 { +extension UInt8 { static var _slash: UInt8 { UInt8(ascii: "/") } static var _backslash: UInt8 { UInt8(ascii: "\\") } static var _colon: UInt8 { UInt8(ascii: ":") } var isLetter: Bool? { - return (0x41 ... 0x5a) ~= self || (0x61 ... 0x7a) ~= self + return (0x41...0x5a) ~= self || (0x61...0x7a) ~= self + } +} + +extension OutputProtocol { + internal func output(from data: [UInt8]) throws -> OutputType { + return try data.withUnsafeBytes { + let span = RawSpan(_unsafeBytes: $0) + return try self.output(from: span) + } } } -#endif // canImport(WinSDK) +#endif // canImport(WinSDK) diff --git a/Sources/_Subprocess/Result.swift b/Sources/_Subprocess/Result.swift new file mode 100644 index 00000000..e1f79894 --- /dev/null +++ b/Sources/_Subprocess/Result.swift @@ -0,0 +1,123 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +#if canImport(System) +import System +#else +@preconcurrency import SystemPackage +#endif + +// MARK: - Result + +/// A simple wrapper around the generic result returned by the +/// `run` closures with the corresponding `TerminationStatus` +/// of the child process. +public struct ExecutionResult { + /// The termination status of the child process + public let terminationStatus: TerminationStatus + /// The result returned by the closure passed to `.run` methods + public let value: Result + + internal init(terminationStatus: TerminationStatus, value: Result) { + self.terminationStatus = terminationStatus + self.value = value + } +} + +/// The result of a subprocess execution with its collected +/// standard output and standard error. +public struct CollectedResult< + Output: OutputProtocol, + Error: OutputProtocol +>: Sendable { + /// The process identifier for the executed subprocess + public let processIdentifier: ProcessIdentifier + /// The termination status of the executed subprocess + public let terminationStatus: TerminationStatus + public let standardOutput: Output.OutputType + public let standardError: Error.OutputType + + internal init( + processIdentifier: ProcessIdentifier, + terminationStatus: TerminationStatus, + standardOutput: Output.OutputType, + standardError: Error.OutputType + ) { + self.processIdentifier = processIdentifier + self.terminationStatus = terminationStatus + self.standardOutput = standardOutput + self.standardError = standardError + } +} + +// MARK: - CollectedResult Conformances +extension CollectedResult: Equatable where Output.OutputType: Equatable, Error.OutputType: Equatable {} + +extension CollectedResult: Hashable where Output.OutputType: Hashable, Error.OutputType: Hashable {} + +extension CollectedResult: Codable where Output.OutputType: Codable, Error.OutputType: Codable {} + +extension CollectedResult: CustomStringConvertible +where Output.OutputType: CustomStringConvertible, Error.OutputType: CustomStringConvertible { + public var description: String { + return """ + CollectedResult( + processIdentifier: \(self.processIdentifier), + terminationStatus: \(self.terminationStatus.description), + standardOutput: \(self.standardOutput.description) + standardError: \(self.standardError.description) + ) + """ + } +} + +extension CollectedResult: CustomDebugStringConvertible +where Output.OutputType: CustomDebugStringConvertible, Error.OutputType: CustomDebugStringConvertible { + public var debugDescription: String { + return """ + CollectedResult( + processIdentifier: \(self.processIdentifier), + terminationStatus: \(self.terminationStatus.description), + standardOutput: \(self.standardOutput.debugDescription) + standardError: \(self.standardError.debugDescription) + ) + """ + } +} + +// MARK: - ExecutionResult Conformances +extension ExecutionResult: Equatable where Result: Equatable {} + +extension ExecutionResult: Hashable where Result: Hashable {} + +extension ExecutionResult: Codable where Result: Codable {} + +extension ExecutionResult: CustomStringConvertible where Result: CustomStringConvertible { + public var description: String { + return """ + ExecutionResult( + terminationStatus: \(self.terminationStatus.description), + value: \(self.value.description) + ) + """ + } +} + +extension ExecutionResult: CustomDebugStringConvertible where Result: CustomDebugStringConvertible { + public var debugDescription: String { + return """ + ExecutionResult( + terminationStatus: \(self.terminationStatus.debugDescription), + value: \(self.value.debugDescription) + ) + """ + } +} diff --git a/Sources/_Subprocess/Subprocess+API.swift b/Sources/_Subprocess/Subprocess+API.swift deleted file mode 100644 index f9c8b1ec..00000000 --- a/Sources/_Subprocess/Subprocess+API.swift +++ /dev/null @@ -1,465 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift.org project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -import SystemPackage -#if canImport(FoundationEssentials) -import FoundationEssentials -#elseif canImport(Foundation) -import Foundation -#endif - -extension Subprocess { - /// Run a executable with given parameters and capture its - /// standard output and standard error. - /// - Parameters: - /// - executable: The executable to run. - /// - arguments: The arguments to pass to the executable. - /// - environment: The environment to use for the process. - /// - workingDirectory: The working directory to use for the subprocess. - /// - platformOptions: The platform specific options to use - /// when running the executable. - /// - input: The input to send to the executable. - /// - output: The method to use for collecting the standard output. - /// - error: The method to use for collecting the standard error. - /// - Returns: `CollectedResult` which contains process identifier, - /// termination status, captured standard output and standard error. - public static func run( - _ executable: Executable, - arguments: Arguments = [], - environment: Environment = .inherit, - workingDirectory: FilePath? = nil, - platformOptions: PlatformOptions = PlatformOptions(), - input: InputMethod = .noInput, - output: CollectedOutputMethod = .collect(), - error: CollectedOutputMethod = .collect() - ) async throws -> CollectedResult { - let result = try await self.run( - executable, - arguments: arguments, - environment: environment, - workingDirectory: workingDirectory, - platformOptions: platformOptions, - input: input, - output: .init(method: output.method), - error: .init(method: error.method) - ) { subprocess in - let (standardOutput, standardError) = try await subprocess.captureIOs() - return ( - processIdentifier: subprocess.processIdentifier, - standardOutput: standardOutput, - standardError: standardError - ) - } - return CollectedResult( - processIdentifier: result.value.processIdentifier, - terminationStatus: result.terminationStatus, - standardOutput: result.value.standardOutput, - standardError: result.value.standardError - ) - } - - /// Run a executable with given parameters and capture its - /// standard output and standard error. - /// - Parameters: - /// - executable: The executable to run. - /// - arguments: The arguments to pass to the executable. - /// - environment: The environment to use for the process. - /// - workingDirectory: The working directory to use for the subprocess. - /// - platformOptions: The platform specific options to use - /// when running the executable. - /// - input: The input to send to the executable. - /// - output: The method to use for collecting the standard output. - /// - error: The method to use for collecting the standard error. - /// - Returns: `CollectedResult` which contains process identifier, - /// termination status, captured standard output and standard error. - public static func run( - _ executable: Executable, - arguments: Arguments = [], - environment: Environment = .inherit, - workingDirectory: FilePath? = nil, - platformOptions: PlatformOptions = PlatformOptions(), - input: some Sequence, - output: CollectedOutputMethod = .collect(), - error: CollectedOutputMethod = .collect() - ) async throws -> CollectedResult { - let result = try await self.run( - executable, - arguments: arguments, - environment: environment, - workingDirectory: workingDirectory, - platformOptions: platformOptions, - output: .init(method: output.method), - error: .init(method: output.method) - ) { subprocess, writer in - return try await withThrowingTaskGroup(of: CapturedIOs?.self) { group in - group.addTask { - try await writer.write(input) - try await writer.finish() - return nil - } - group.addTask { - return try await subprocess.captureIOs() - } - var capturedIOs: CapturedIOs! - while let result = try await group.next() { - if result != nil { - capturedIOs = result - } - } - return ( - processIdentifier: subprocess.processIdentifier, - standardOutput: capturedIOs.standardOutput, - standardError: capturedIOs.standardError - ) - } - } - return CollectedResult( - processIdentifier: result.value.processIdentifier, - terminationStatus: result.terminationStatus, - standardOutput: result.value.standardOutput, - standardError: result.value.standardError - ) - } - - /// Run a executable with given parameters and capture its - /// standard output and standard error. - /// - Parameters: - /// - executable: The executable to run. - /// - arguments: The arguments to pass to the executable. - /// - environment: The environment to use for the process. - /// - workingDirectory: The working directory to use for the subprocess. - /// - platformOptions: The platform specific options to use - /// when running the executable. - /// - input: The input to send to the executable. - /// - output: The method to use for collecting the standard output. - /// - error: The method to use for collecting the standard error. - /// - Returns: `CollectedResult` which contains process identifier, - /// termination status, captured standard output and standard error. - public static func run( - _ executable: Executable, - arguments: Arguments = [], - environment: Environment = .inherit, - workingDirectory: FilePath? = nil, - platformOptions: PlatformOptions = PlatformOptions(), - input: S, - output: CollectedOutputMethod = .collect(), - error: CollectedOutputMethod = .collect() - ) async throws -> CollectedResult where S.Element == UInt8 { - let result = try await self.run( - executable, - arguments: arguments, - environment: environment, - workingDirectory: workingDirectory, - platformOptions: platformOptions, - output: .init(method: output.method), - error: .init(method: output.method) - ) { subprocess, writer in - return try await withThrowingTaskGroup(of: CapturedIOs?.self) { group in - group.addTask { - try await writer.write(input) - try await writer.finish() - return nil - } - group.addTask { - return try await subprocess.captureIOs() - } - var capturedIOs: CapturedIOs! - while let result = try await group.next() { - capturedIOs = result - } - return ( - processIdentifier: subprocess.processIdentifier, - standardOutput: capturedIOs.standardOutput, - standardError: capturedIOs.standardError - ) - } - } - return CollectedResult( - processIdentifier: result.value.processIdentifier, - terminationStatus: result.terminationStatus, - standardOutput: result.value.standardOutput, - standardError: result.value.standardError - ) - } -} - -// MARK: Custom Execution Body -extension Subprocess { - /// Run a executable with given parameters and a custom closure - /// to manage the running subprocess' lifetime and its IOs. - /// - Parameters: - /// - executable: The executable to run. - /// - arguments: The arguments to pass to the executable. - /// - environment: The environment in which to run the executable. - /// - workingDirectory: The working directory in which to run the executable. - /// - platformOptions: The platform specific options to use - /// when running the executable. - /// - input: The input to send to the executable. - /// - output: The method to use for redirecting the standard output. - /// - error: The method to use for redirecting the standard error. - /// - body: The custom execution body to manually control the running process - /// - Returns a ExecutableResult type containing the return value - /// of the closure. - public static func run( - _ executable: Executable, - arguments: Arguments = [], - environment: Environment = .inherit, - workingDirectory: FilePath? = nil, - platformOptions: PlatformOptions = PlatformOptions(), - input: InputMethod = .noInput, - output: RedirectedOutputMethod = .redirectToSequence, - error: RedirectedOutputMethod = .redirectToSequence, - _ body: (sending @escaping (Subprocess) async throws -> R) - ) async throws -> ExecutionResult { - return try await Configuration( - executable: executable, - arguments: arguments, - environment: environment, - workingDirectory: workingDirectory, - platformOptions: platformOptions - ) - .run(input: input, output: output, error: error, body) - } - - /// Run a executable with given parameters and a custom closure - /// to manage the running subprocess' lifetime and its IOs. - /// - Parameters: - /// - executable: The executable to run. - /// - arguments: The arguments to pass to the executable. - /// - environment: The environment in which to run the executable. - /// - workingDirectory: The working directory in which to run the executable. - /// - platformOptions: The platform specific options to use - /// when running the executable. - /// - input: The input to send to the executable. - /// - output: The method to use for redirecting the standard output. - /// - error: The method to use for redirecting the standard error. - /// - body: The custom execution body to manually control the running process - /// - Returns a `ExecutableResult` type containing the return value - /// of the closure. - public static func run( - _ executable: Executable, - arguments: Arguments = [], - environment: Environment = .inherit, - workingDirectory: FilePath? = nil, - platformOptions: PlatformOptions = PlatformOptions(), - input: some Sequence, - output: RedirectedOutputMethod = .redirectToSequence, - error: RedirectedOutputMethod = .redirectToSequence, - _ body: (sending @escaping (Subprocess) async throws -> R) - ) async throws -> ExecutionResult { - return try await Configuration( - executable: executable, - arguments: arguments, - environment: environment, - workingDirectory: workingDirectory, - platformOptions: platformOptions - ) - .run(output: output, error: error) { execution, writer in - return try await withThrowingTaskGroup(of: R?.self) { group in - group.addTask { - try await writer.write(input) - try await writer.finish() - return nil - } - group.addTask { - return try await body(execution) - } - var result: R! - while let next = try await group.next() { - result = next - } - return result - } - } - } - - /// Run a executable with given parameters and a custom closure - /// to manage the running subprocess' lifetime and its IOs. - /// - Parameters: - /// - executable: The executable to run. - /// - arguments: The arguments to pass to the executable. - /// - environment: The environment in which to run the executable. - /// - workingDirectory: The working directory in which to run the executable. - /// - platformOptions: The platform specific options to use - /// when running the executable. - /// - input: The input to send to the executable. - /// - output: The method to use for redirecting the standard output. - /// - error: The method to use for redirecting the standard error. - /// - body: The custom execution body to manually control the running process - /// - Returns a `ExecutableResult` type containing the return value - /// of the closure. - public static func run( - _ executable: Executable, - arguments: Arguments = [], - environment: Environment = .inherit, - workingDirectory: FilePath? = nil, - platformOptions: PlatformOptions = PlatformOptions(), - input: S, - output: RedirectedOutputMethod = .redirectToSequence, - error: RedirectedOutputMethod = .redirectToSequence, - _ body: (sending @escaping (Subprocess) async throws -> R) - ) async throws -> ExecutionResult where S.Element == UInt8 { - return try await Configuration( - executable: executable, - arguments: arguments, - environment: environment, - workingDirectory: workingDirectory, - platformOptions: platformOptions - ) - .run(output: output, error: error) { execution, writer in - return try await withThrowingTaskGroup(of: R?.self) { group in - group.addTask { - try await writer.write(input) - try await writer.finish() - return nil - } - group.addTask { - return try await body(execution) - } - var result: R! - while let next = try await group.next() { - result = next - } - return result - } - } - } - - /// Run a executable with given parameters and a custom closure - /// to manage the running subprocess' lifetime and write to its - /// standard input via `StandardInputWriter` - /// - Parameters: - /// - executable: The executable to run. - /// - arguments: The arguments to pass to the executable. - /// - environment: The environment in which to run the executable. - /// - workingDirectory: The working directory in which to run the executable. - /// - platformOptions: The platform specific options to use - /// when running the executable. - /// - output: The method to use for redirecting the standard output. - /// - error: The method to use for redirecting the standard error. - /// - body: The custom execution body to manually control the running process - /// - Returns a ExecutableResult type containing the return value - /// of the closure. - public static func run( - _ executable: Executable, - arguments: Arguments = [], - environment: Environment = .inherit, - workingDirectory: FilePath? = nil, - platformOptions: PlatformOptions = PlatformOptions(), - output: RedirectedOutputMethod = .redirectToSequence, - error: RedirectedOutputMethod = .redirectToSequence, - _ body: (sending @escaping (Subprocess, StandardInputWriter) async throws -> R) - ) async throws -> ExecutionResult { - return try await Configuration( - executable: executable, - arguments: arguments, - environment: environment, - workingDirectory: workingDirectory, - platformOptions: platformOptions - ) - .run(output: output, error: error, body) - } -} - -// MARK: - Configuration Based -extension Subprocess { - /// Run a executable with given parameters specified by a - /// `Subprocess.Configuration` - /// - Parameters: - /// - configuration: The `Subprocess` configuration to run. - /// - output: The method to use for redirecting the standard output. - /// - error: The method to use for redirecting the standard error. - /// - body: The custom configuration body to manually control - /// the running process and write to its standard input. - /// - Returns a ExecutableResult type containing the return value - /// of the closure. - public static func run( - _ configuration: Configuration, - output: RedirectedOutputMethod = .redirectToSequence, - error: RedirectedOutputMethod = .redirectToSequence, - _ body: (sending @escaping (Subprocess, StandardInputWriter) async throws -> R) - ) async throws -> ExecutionResult { - return try await configuration.run(output: output, error: error, body) - } -} - -// MARK: - Detached -extension Subprocess { - /// Run a executable with given parameters and return its process - /// identifier immediately without monitoring the state of the - /// subprocess nor waiting until it exits. - /// - /// This method is useful for launching subprocesses that outlive their - /// parents (for example, daemons and trampolines). - /// - /// - Parameters: - /// - executable: The executable to run. - /// - arguments: The arguments to pass to the executable. - /// - environment: The environment to use for the process. - /// - workingDirectory: The working directory for the process. - /// - platformOptions: The platform specific options to use for the process. - /// - input: A file descriptor to bind to the subprocess' standard input. - /// - output: A file descriptor to bind to the subprocess' standard output. - /// - error: A file descriptor to bind to the subprocess' standard error. - /// - Returns: the process identifier for the subprocess. - public static func runDetached( - _ executable: Executable, - arguments: Arguments = [], - environment: Environment = .inherit, - workingDirectory: FilePath? = nil, - platformOptions: PlatformOptions = PlatformOptions(), - input: FileDescriptor? = nil, - output: FileDescriptor? = nil, - error: FileDescriptor? = nil - ) throws -> ProcessIdentifier { - // Create input - let executionInput: ExecutionInput - let executionOutput: ExecutionOutput - let executionError: ExecutionOutput - if let inputFd = input { - executionInput = .init(storage: .fileDescriptor(inputFd, false)) - } else { - let devnull: FileDescriptor = try .openDevNull(withAcessMode: .readOnly) - executionInput = .init(storage: .noInput(devnull)) - } - if let outputFd = output { - executionOutput = .init(storage: .fileDescriptor(outputFd, false)) - } else { - let devnull: FileDescriptor = try .openDevNull(withAcessMode: .writeOnly) - executionOutput = .init(storage: .discarded(devnull)) - } - if let errorFd = error { - executionError = .init( - storage: .fileDescriptor(errorFd, false) - ) - } else { - let devnull: FileDescriptor = try .openDevNull(withAcessMode: .writeOnly) - executionError = .init(storage: .discarded(devnull)) - } - // Spawn! - let config: Configuration = Configuration( - executable: executable, - arguments: arguments, - environment: environment, - workingDirectory: workingDirectory, - platformOptions: platformOptions - ) - return try config.spawn( - withInput: executionInput, - output: executionOutput, - error: executionError - ).processIdentifier - } -} - diff --git a/Sources/_Subprocess/Subprocess+AsyncDataSequence.swift b/Sources/_Subprocess/Subprocess+AsyncDataSequence.swift deleted file mode 100644 index a3a3e393..00000000 --- a/Sources/_Subprocess/Subprocess+AsyncDataSequence.swift +++ /dev/null @@ -1,86 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift.org project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -import SystemPackage -import Dispatch - -#if canImport(FoundationEssentials) -import FoundationEssentials -#elseif canImport(Foundation) -import Foundation -#endif - -extension Subprocess { - public struct AsyncDataSequence: AsyncSequence, Sendable, _AsyncSequence { - public typealias Error = any Swift.Error - - public typealias Element = Data - - @_nonSendable - public struct Iterator: AsyncIteratorProtocol { - public typealias Element = Data - - private let fileDescriptor: FileDescriptor - private var buffer: [UInt8] - private var currentPosition: Int - private var finished: Bool - - internal init(fileDescriptor: FileDescriptor) { - self.fileDescriptor = fileDescriptor - self.buffer = [] - self.currentPosition = 0 - self.finished = false - } - - public mutating func next() async throws -> Data? { - let data = try await self.fileDescriptor.readChunk( - upToLength: Subprocess.readBufferSize - ) - if data == nil { - // We finished reading. Close the file descriptor now - try self.fileDescriptor.close() - return nil - } - return data - } - } - - private let fileDescriptor: FileDescriptor - - init(fileDescriptor: FileDescriptor) { - self.fileDescriptor = fileDescriptor - } - - public func makeAsyncIterator() -> Iterator { - return Iterator(fileDescriptor: self.fileDescriptor) - } - } -} - -extension RangeReplaceableCollection { - /// Creates a new instance of a collection containing the elements of an asynchronous sequence. - /// - /// - Parameter source: The asynchronous sequence of elements for the new collection. - @inlinable - public init(_ source: Source) async rethrows where Source.Element == Element { - self.init() - for try await item in source { - append(item) - } - } -} - -public protocol _AsyncSequence: AsyncSequence { - associatedtype Error -} diff --git a/Sources/_Subprocess/Subprocess+Configuration.swift b/Sources/_Subprocess/Subprocess+Configuration.swift deleted file mode 100644 index 0b785790..00000000 --- a/Sources/_Subprocess/Subprocess+Configuration.swift +++ /dev/null @@ -1,769 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift.org project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -@preconcurrency import SystemPackage - -#if canImport(Darwin) -import Darwin -#elseif canImport(Glibc) -import Glibc -#elseif canImport(WinSDK) -import WinSDK -#endif - -#if canImport(FoundationEssentials) -import FoundationEssentials -#elseif canImport(Foundation) -import Foundation -#endif - -extension Subprocess { - /// A collection of configurations parameters to use when - /// spawning a subprocess. - public struct Configuration: Sendable, Hashable { - - internal enum RunState: Sendable { - case workBody(Result) - case monitorChildProcess(TerminationStatus) - } - - /// The executable to run. - public var executable: Executable - /// The arguments to pass to the executable. - public var arguments: Arguments - /// The environment to use when running the executable. - public var environment: Environment - /// The working directory to use when running the executable. - public var workingDirectory: FilePath - /// The platform specifc options to use when - /// running the subprocess. - public var platformOptions: PlatformOptions - - public init( - executable: Executable, - arguments: Arguments = [], - environment: Environment = .inherit, - workingDirectory: FilePath? = nil, - platformOptions: PlatformOptions = PlatformOptions() - ) { - self.executable = executable - self.arguments = arguments - self.environment = environment - self.workingDirectory = workingDirectory ?? .currentWorkingDirectory - self.platformOptions = platformOptions - } - - /// Close each input individually, and throw the first error if there's multiple errors thrown - @Sendable - private func cleanup( - process: Subprocess, - childSide: Bool, parentSide: Bool, - attemptToTerminateSubProcess: Bool - ) async throws { - guard childSide || parentSide || attemptToTerminateSubProcess else { - return - } - - // Attempt to teardown the subprocess - if attemptToTerminateSubProcess { - await process.teardown( - using: self.platformOptions.teardownSequence - ) - } - - let inputCloseFunc: () throws -> Void - let outputCloseFunc: () throws -> Void - let errorCloseFunc: () throws -> Void - if childSide && parentSide { - // Close all - inputCloseFunc = process.executionInput.closeAll - outputCloseFunc = process.executionOutput.closeAll - errorCloseFunc = process.executionError.closeAll - } else if childSide { - // Close child only - inputCloseFunc = process.executionInput.closeChildSide - outputCloseFunc = process.executionOutput.closeChildSide - errorCloseFunc = process.executionError.closeChildSide - } else { - // Close parent only - inputCloseFunc = process.executionInput.closeParentSide - outputCloseFunc = process.executionOutput.closeParentSide - errorCloseFunc = process.executionError.closeParentSide - } - - var inputError: Error? - var outputError: Error? - var errorError: Error? // lol - do { - try inputCloseFunc() - } catch { - inputError = error - } - - do { - try outputCloseFunc() - } catch { - outputError = error - } - - do { - try errorCloseFunc() - } catch { - errorError = error // lolol - } - - if let inputError = inputError { - throw inputError - } - - if let outputError = outputError { - throw outputError - } - - if let errorError = errorError { - throw errorError - } - } - - /// Close each input individually, and throw the first error if there's multiple errors thrown - @Sendable - internal func cleanupAll( - input: ExecutionInput, - output: ExecutionOutput, - error: ExecutionOutput - ) throws { - var inputError: Error? - var outputError: Error? - var errorError: Error? - - do { - try input.closeAll() - } catch { - inputError = error - } - - do { - try output.closeAll() - } catch { - outputError = error - } - - do { - try error.closeAll() - } catch { - errorError = error - } - - if let inputError = inputError { - throw inputError - } - if let outputError = outputError { - throw outputError - } - if let errorError = errorError { - throw errorError - } - } - - internal func run( - output: RedirectedOutputMethod, - error: RedirectedOutputMethod, - _ body: sending @escaping (Subprocess, StandardInputWriter) async throws -> R - ) async throws -> ExecutionResult { - let (readFd, writeFd) = try FileDescriptor.pipe() - let executionInput: ExecutionInput = .init(storage: .customWrite(readFd, writeFd)) - let executionOutput: ExecutionOutput = try output.createExecutionOutput() - let executionError: ExecutionOutput = try error.createExecutionOutput() - let process: Subprocess = try self.spawn( - withInput: executionInput, - output: executionOutput, - error: executionError) - // After spawn, cleanup child side fds - try await self.cleanup( - process: process, - childSide: true, - parentSide: false, - attemptToTerminateSubProcess: false - ) - return try await withAsyncTaskCancellationHandler { - return try await withThrowingTaskGroup(of: RunState.self) { group in - group.addTask { - let status = try await monitorProcessTermination( - forProcessWithIdentifier: process.processIdentifier) - return .monitorChildProcess(status) - } - group.addTask { - do { - let result = try await body(process, .init(input: executionInput)) - try await self.cleanup( - process: process, - childSide: false, - parentSide: true, - attemptToTerminateSubProcess: false - ) - return .workBody(result) - } catch { - // Cleanup everything - try await self.cleanup( - process: process, - childSide: false, - parentSide: true, - attemptToTerminateSubProcess: false - ) - throw error - } - } - - var result: R! - var terminationStatus: TerminationStatus! - while let state = try await group.next() { - switch state { - case .monitorChildProcess(let status): - // We don't really care about termination status here - terminationStatus = status - case .workBody(let workResult): - result = workResult - } - } - return ExecutionResult(terminationStatus: terminationStatus, value: result) - } - } onCancel: { - // Attempt to terminate the child process - // Since the task has already been cancelled, - // this is the best we can do - try? await self.cleanup( - process: process, - childSide: true, - parentSide: true, - attemptToTerminateSubProcess: true - ) - } - } - - internal func run( - input: InputMethod, - output: RedirectedOutputMethod, - error: RedirectedOutputMethod, - _ body: (sending @escaping (Subprocess) async throws -> R) - ) async throws -> ExecutionResult { - let executionInput = try input.createExecutionInput() - let executionOutput = try output.createExecutionOutput() - let executionError = try error.createExecutionOutput() - let process = try self.spawn( - withInput: executionInput, - output: executionOutput, - error: executionError) - // After spawn, clean up child side - try await self.cleanup( - process: process, - childSide: true, - parentSide: false, - attemptToTerminateSubProcess: false - ) - return try await withAsyncTaskCancellationHandler { - return try await withThrowingTaskGroup(of: RunState.self) { group in - group.addTask { - let status = try await monitorProcessTermination( - forProcessWithIdentifier: process.processIdentifier) - return .monitorChildProcess(status) - } - group.addTask { - do { - let result = try await body(process) - try await self.cleanup( - process: process, - childSide: false, - parentSide: true, - attemptToTerminateSubProcess: false - ) - return .workBody(result) - } catch { - try await self.cleanup( - process: process, - childSide: false, - parentSide: true, - attemptToTerminateSubProcess: false - ) - throw error - } - } - - var result: R! - var terminationStatus: TerminationStatus! - while let state = try await group.next() { - switch state { - case .monitorChildProcess(let status): - terminationStatus = status - case .workBody(let workResult): - result = workResult - } - } - return ExecutionResult(terminationStatus: terminationStatus, value: result) - } - } onCancel: { - // Attempt to terminate the child process - // Since the task has already been cancelled, - // this is the best we can do - try? await self.cleanup( - process: process, - childSide: true, - parentSide: true, - attemptToTerminateSubProcess: true - ) - } - } - } -} - -extension Subprocess.Configuration : CustomStringConvertible, CustomDebugStringConvertible { - public var description: String { - return """ -Subprocess.Configuration( - executable: \(self.executable.description), - arguments: \(self.arguments.description), - environment: \(self.environment.description), - workingDirectory: \(self.workingDirectory), - platformOptions: \(self.platformOptions.description(withIndent: 1)) -) -""" - } - - public var debugDescription: String { - return """ -Subprocess.Configuration( - executable: \(self.executable.debugDescription), - arguments: \(self.arguments.debugDescription), - environment: \(self.environment.debugDescription), - workingDirectory: \(self.workingDirectory), - platformOptions: \(self.platformOptions.description(withIndent: 1)) -) -""" - } -} - -// MARK: - Executable -extension Subprocess { - /// `Subprocess.Executable` defines how should the executable - /// be looked up for execution. - public struct Executable: Sendable, Hashable { - internal enum Configuration: Sendable, Hashable { - case executable(String) - case path(FilePath) - } - - internal let storage: Configuration - - private init(_config: Configuration) { - self.storage = _config - } - - /// Locate the executable by its name. - /// `Subprocess` will use `PATH` value to - /// determine the full path to the executable. - public static func named(_ executableName: String) -> Self { - return .init(_config: .executable(executableName)) - } - /// Locate the executable by its full path. - /// `Subprocess` will use this path directly. - public static func at(_ filePath: FilePath) -> Self { - return .init(_config: .path(filePath)) - } - /// Returns the full executable path given the environment value. - public func resolveExecutablePath(in environment: Environment) -> FilePath? { - if let path = self.resolveExecutablePath(withPathValue: environment.pathValue()) { - return FilePath(path) - } - return nil - } - } -} - -extension Subprocess.Executable : CustomStringConvertible, CustomDebugStringConvertible { - public var description: String { - switch storage { - case .executable(let executableName): - return executableName - case .path(let filePath): - return filePath.string - } - } - - public var debugDescription: String { - switch storage { - case .executable(let string): - return "executable(\(string))" - case .path(let filePath): - return "path(\(filePath.string))" - } - } -} - -// MARK: - Arguments -extension Subprocess { - /// A collection of arguments to pass to the subprocess. - public struct Arguments: Sendable, ExpressibleByArrayLiteral, Hashable { - public typealias ArrayLiteralElement = String - - internal let storage: [StringOrRawBytes] - internal let executablePathOverride: StringOrRawBytes? - - /// Create an Arguments object using the given literal values - public init(arrayLiteral elements: String...) { - self.storage = elements.map { .string($0) } - self.executablePathOverride = nil - } - /// Create an Arguments object using the given array - public init(_ array: [String]) { - self.storage = array.map { .string($0) } - self.executablePathOverride = nil - } - -#if !os(Windows) // Windows does NOT support arg0 override - /// Create an `Argument` object using the given values, but - /// override the first Argument value to `executablePathOverride`. - /// If `executablePathOverride` is nil, - /// `Arguments` will automatically use the executable path - /// as the first argument. - /// - Parameters: - /// - executablePathOverride: the value to override the first argument. - /// - remainingValues: the rest of the argument value - public init(executablePathOverride: String?, remainingValues: [String]) { - self.storage = remainingValues.map { .string($0) } - if let executablePathOverride = executablePathOverride { - self.executablePathOverride = .string(executablePathOverride) - } else { - self.executablePathOverride = nil - } - } - - /// Create an `Argument` object using the given values, but - /// override the first Argument value to `executablePathOverride`. - /// If `executablePathOverride` is nil, - /// `Arguments` will automatically use the executable path - /// as the first argument. - /// - Parameters: - /// - executablePathOverride: the value to override the first argument. - /// - remainingValues: the rest of the argument value - public init(executablePathOverride: Data?, remainingValues: [Data]) { - self.storage = remainingValues.map { .rawBytes($0.toArray()) } - if let override = executablePathOverride { - self.executablePathOverride = .rawBytes(override.toArray()) - } else { - self.executablePathOverride = nil - } - } - - public init(_ array: [Data]) { - self.storage = array.map { .rawBytes($0.toArray()) } - self.executablePathOverride = nil - } -#endif - } -} - -extension Subprocess.Arguments : CustomStringConvertible, CustomDebugStringConvertible { - public var description: String { - var result: [String] = self.storage.map(\.description) - - if let override = self.executablePathOverride { - result.insert("override\(override.description)", at: 0) - } - return result.description - } - - public var debugDescription: String { return self.description } -} - -// MARK: - Environment -extension Subprocess { - /// A set of environment variables to use when executing the subprocess. - public struct Environment: Sendable, Hashable { - internal enum Configuration: Sendable, Hashable { - case inherit([StringOrRawBytes : StringOrRawBytes]) - case custom([StringOrRawBytes : StringOrRawBytes]) - } - - internal let config: Configuration - - init(config: Configuration) { - self.config = config - } - /// Child process should inherit the same environment - /// values from its parent process. - public static var inherit: Self { - return .init(config: .inherit([:])) - } - /// Override the provided `newValue` in the existing `Environment` - public func updating(_ newValue: [String : String]) -> Self { - return .init(config: .inherit(newValue.wrapToStringOrRawBytes())) - } - /// Use custom environment variables - public static func custom(_ newValue: [String : String]) -> Self { - return .init(config: .custom(newValue.wrapToStringOrRawBytes())) - } - -#if !os(Windows) - /// Override the provided `newValue` in the existing `Environment` - public func updating(_ newValue: [Data : Data]) -> Self { - return .init(config: .inherit(newValue.wrapToStringOrRawBytes())) - } - /// Use custom environment variables - public static func custom(_ newValue: [Data : Data]) -> Self { - return .init(config: .custom(newValue.wrapToStringOrRawBytes())) - } -#endif - } -} - -extension Subprocess.Environment : CustomStringConvertible, CustomDebugStringConvertible { - public var description: String { - switch self.config { - case .custom(let customDictionary): - return customDictionary.dictionaryDescription - case .inherit(let updateValue): - return "Inherting current environment with updates: \(updateValue.dictionaryDescription)" - } - } - - public var debugDescription: String { - return self.description - } -} - -fileprivate extension Dictionary where Key == String, Value == String { - func wrapToStringOrRawBytes() -> [Subprocess.StringOrRawBytes : Subprocess.StringOrRawBytes] { - var result = Dictionary< - Subprocess.StringOrRawBytes, - Subprocess.StringOrRawBytes - >(minimumCapacity: self.count) - for (key, value) in self { - result[.string(key)] = .string(value) - } - return result - } -} - -fileprivate extension Dictionary where Key == Data, Value == Data { - func wrapToStringOrRawBytes() -> [Subprocess.StringOrRawBytes : Subprocess.StringOrRawBytes] { - var result = Dictionary< - Subprocess.StringOrRawBytes, - Subprocess.StringOrRawBytes - >(minimumCapacity: self.count) - for (key, value) in self { - result[.rawBytes(key.toArray())] = .rawBytes(value.toArray()) - } - return result - } -} - -fileprivate extension Dictionary where Key == Subprocess.StringOrRawBytes, Value == Subprocess.StringOrRawBytes { - var dictionaryDescription: String { - var result = "[\n" - for (key, value) in self { - result += "\t\(key.description) : \(value.description),\n" - } - result += "]" - return result - } -} - -fileprivate extension Data { - func toArray() -> [T] { - return self.withUnsafeBytes { ptr in - return Array(ptr.bindMemory(to: T.self)) - } - } -} - -// MARK: - TerminationStatus -extension Subprocess { - /// An exit status of a subprocess. - @frozen - public enum TerminationStatus: Sendable, Hashable, Codable { - #if canImport(WinSDK) - public typealias Code = DWORD - #else - public typealias Code = CInt - #endif - - /// The subprocess was existed with the given code - case exited(Code) - /// The subprocess was signalled with given exception value - case unhandledException(Code) - /// Whether the current TerminationStatus is successful. - public var isSuccess: Bool { - switch self { - case .exited(let exitCode): - return exitCode == 0 - case .unhandledException(_): - return false - } - } - } -} - -extension Subprocess.TerminationStatus : CustomStringConvertible, CustomDebugStringConvertible { - public var description: String { - switch self { - case .exited(let code): - return "exited(\(code))" - case .unhandledException(let code): - return "unhandledException(\(code))" - } - } - - public var debugDescription: String { - return self.description - } -} - -// MARK: - Internal -extension Subprocess { - internal enum StringOrRawBytes: Sendable, Hashable { - case string(String) - case rawBytes([CChar]) - - // Return value needs to be deallocated manually by callee - func createRawBytes() -> UnsafeMutablePointer { - switch self { - case .string(let string): - return strdup(string) - case .rawBytes(let rawBytes): - return strdup(rawBytes) - } - } - - var stringValue: String? { - switch self { - case .string(let string): - return string - case .rawBytes(let rawBytes): - return String(validatingUTF8: rawBytes) - } - } - - var description: String { - switch self { - case .string(let string): - return string - case .rawBytes(let bytes): - return bytes.description - } - } - - var count: Int { - switch self { - case .string(let string): - return string.count - case .rawBytes(let rawBytes): - return strnlen(rawBytes, Int.max) - } - } - - func hash(into hasher: inout Hasher) { - // If Raw bytes is valid UTF8, hash it as so - switch self { - case .string(let string): - hasher.combine(string) - case .rawBytes(let bytes): - if let stringValue = self.stringValue { - hasher.combine(stringValue) - } else { - hasher.combine(bytes) - } - } - } - } -} - -extension FilePath { - static var currentWorkingDirectory: Self { - let path = getcwd(nil, 0)! - defer { free(path) } - return .init(String(cString: path)) - } -} - -extension Optional where Wrapped : Collection { - func withOptionalUnsafeBufferPointer(_ body: ((UnsafeBufferPointer)?) throws -> R) rethrows -> R { - switch self { - case .some(let wrapped): - guard let array: Array = wrapped as? Array else { - return try body(nil) - } - return try array.withUnsafeBufferPointer { ptr in - return try body(ptr) - } - case .none: - return try body(nil) - } - } -} - -extension Optional where Wrapped == String { - func withOptionalCString(_ body: ((UnsafePointer)?) throws -> R) rethrows -> R { - switch self { - case .none: - return try body(nil) - case .some(let wrapped): - return try wrapped.withCString { - return try body($0) - } - } - } - - var stringValue: String { - return self ?? "nil" - } -} - -// MARK: - Stubs for the one from Foundation -public enum QualityOfService: Int, Sendable { - case userInteractive = 0x21 - case userInitiated = 0x19 - case utility = 0x11 - case background = 0x09 - case `default` = -1 -} - -internal func withAsyncTaskCancellationHandler( - _ body: sending @escaping () async throws -> R, - onCancel handler: sending @escaping () async -> Void -) async rethrows -> R { - return try await withThrowingTaskGroup( - of: R?.self, - returning: R.self - ) { group in - group.addTask { - return try await body() - } - group.addTask { - // wait until cancelled - do { while true { try await Task.sleep(nanoseconds: 1_000_000_000) } } catch {} - // Run task cancel handler - await handler() - return nil - } - - while let result = try await group.next() { - if let result = result { - // As soon as the body finishes, cancel the group - group.cancelAll() - return result - } - } - fatalError("Unreachable") - } -} - diff --git a/Sources/_Subprocess/Subprocess+IO.swift b/Sources/_Subprocess/Subprocess+IO.swift deleted file mode 100644 index 1b43e421..00000000 --- a/Sources/_Subprocess/Subprocess+IO.swift +++ /dev/null @@ -1,437 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift.org project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -#if canImport(FoundationEssentials) -import FoundationEssentials -#elseif canImport(Foundation) -import Foundation -#endif - -import Dispatch -import SystemPackage - -// Naive Mutex so we don't have to update the macOS dependency -final class _Mutex: Sendable { - let lock = Lock() - - var value: Value - - init(_ value: Value) { - self.value = value - } - - func withLock(_ body: (inout Value) throws -> R) rethrows -> R { - self.lock.lock() - defer { self.lock.unlock() } - return try body(&self.value) - } -} - -// MARK: - Input -extension Subprocess { - /// `InputMethod` defines how should the standard input - /// of the subprocess receive inputs. - public struct InputMethod: Sendable, Hashable { - internal enum Storage: Sendable, Hashable { - case noInput - case fileDescriptor(FileDescriptor, Bool) - } - - internal let method: Storage - - internal init(method: Storage) { - self.method = method - } - - internal func createExecutionInput() throws -> ExecutionInput { - switch self.method { - case .noInput: - let devnull: FileDescriptor = try .openDevNull(withAcessMode: .readOnly) - return .init(storage: .noInput(devnull)) - case .fileDescriptor(let fileDescriptor, let closeWhenDone): - return .init(storage: .fileDescriptor(fileDescriptor, closeWhenDone)) - } - } - - /// Subprocess should read no input. This option is equivalent - /// to bind the stanard input to `/dev/null`. - public static var noInput: Self { - return .init(method: .noInput) - } - - /// Subprocess should read input from a given file descriptor. - /// - Parameters: - /// - fd: the file descriptor to read from - /// - closeAfterSpawningProcess: whether the file descriptor - /// should be automatically closed after subprocess is spawned. - public static func readFrom( - _ fd: FileDescriptor, - closeAfterSpawningProcess: Bool - ) -> Self { - return .init(method: .fileDescriptor(fd, closeAfterSpawningProcess)) - } - } -} - -extension Subprocess { - /// `CollectedOutputMethod` defines how should Subprocess collect - /// output from child process' standard output and standard error - public struct CollectedOutputMethod: Sendable, Hashable { - internal enum Storage: Sendable, Hashable { - case discarded - case fileDescriptor(FileDescriptor, Bool) - case collected(Int) - } - - internal let method: Storage - - internal init(method: Storage) { - self.method = method - } - - /// Subprocess shold dicard the child process output. - /// This option is equivalent to binding the child process - /// output to `/dev/null`. - public static var discard: Self { - return .init(method: .discarded) - } - /// Subprocess should write the child process output - /// to the file descriptor specified. - /// - Parameters: - /// - fd: the file descriptor to write to - /// - closeAfterSpawningProcess: whether to close the - /// file descriptor once the process is spawned. - public static func writeTo(_ fd: FileDescriptor, closeAfterSpawningProcess: Bool) -> Self { - return .init(method: .fileDescriptor(fd, closeAfterSpawningProcess)) - } - /// Subprocess should collect the child process output - /// as `Data` with the given limit in bytes. The default - /// limit is 128kb. - public static func collect(upTo limit: Int = 128 * 1024) -> Self { - return .init(method: .collected(limit)) - } - - internal func createExecutionOutput() throws -> ExecutionOutput { - switch self.method { - case .discarded: - // Bind to /dev/null - let devnull: FileDescriptor = try .openDevNull(withAcessMode: .writeOnly) - return .init(storage: .discarded(devnull)) - case .fileDescriptor(let fileDescriptor, let closeWhenDone): - return .init(storage: .fileDescriptor(fileDescriptor, closeWhenDone)) - case .collected(let limit): - let (readFd, writeFd) = try FileDescriptor.pipe() - return .init(storage: .collected(limit, readFd, writeFd)) - } - } - } - - /// `CollectedOutputMethod` defines how should Subprocess redirect - /// output from child process' standard output and standard error. - public struct RedirectedOutputMethod: Sendable, Hashable { - typealias Storage = CollectedOutputMethod.Storage - - internal let method: Storage - - internal init(method: Storage) { - self.method = method - } - - /// Subprocess shold dicard the child process output. - /// This option is equivalent to binding the child process - /// output to `/dev/null`. - public static var discard: Self { - return .init(method: .discarded) - } - /// Subprocess should redirect the child process output - /// to `Subprocess.standardOutput` or `Subprocess.standardError` - /// so they can be consumed as an AsyncSequence - public static var redirectToSequence: Self { - return .init(method: .collected(128 * 1024)) - } - /// Subprocess shold write the child process output - /// to the file descriptor specified. - /// - Parameters: - /// - fd: the file descriptor to write to - /// - closeAfterSpawningProcess: whether to close the - /// file descriptor once the process is spawned. - public static func writeTo( - _ fd: FileDescriptor, - closeAfterSpawningProcess: Bool - ) -> Self { - return .init(method: .fileDescriptor(fd, closeAfterSpawningProcess)) - } - - internal func createExecutionOutput() throws -> ExecutionOutput { - switch self.method { - case .discarded: - // Bind to /dev/null - let devnull: FileDescriptor = try .openDevNull(withAcessMode: .writeOnly) - return .init(storage: .discarded(devnull)) - case .fileDescriptor(let fileDescriptor, let closeWhenDone): - return .init(storage: .fileDescriptor(fileDescriptor, closeWhenDone)) - case .collected(let limit): - let (readFd, writeFd) = try FileDescriptor.pipe() - return .init(storage: .collected(limit, readFd, writeFd)) - } - } - } -} - -// MARK: - Execution IO -extension Subprocess { - internal final class ExecutionInput: Sendable, Hashable { - - - internal enum Storage: Sendable, Hashable { - case noInput(FileDescriptor?) - case customWrite(FileDescriptor?, FileDescriptor?) - case fileDescriptor(FileDescriptor?, Bool) - } - - let storage: _Mutex - - internal init(storage: Storage) { - self.storage = .init(storage) - } - - internal func getReadFileDescriptor() -> FileDescriptor? { - return self.storage.withLock { - switch $0 { - case .noInput(let readFd): - return readFd - case .customWrite(let readFd, _): - return readFd - case .fileDescriptor(let readFd, _): - return readFd - } - } - } - - internal func getWriteFileDescriptor() -> FileDescriptor? { - return self.storage.withLock { - switch $0 { - case .noInput(_), .fileDescriptor(_, _): - return nil - case .customWrite(_, let writeFd): - return writeFd - } - } - } - - internal func closeChildSide() throws { - try self.storage.withLock { - switch $0 { - case .noInput(let devnull): - try devnull?.close() - $0 = .noInput(nil) - case .customWrite(let readFd, let writeFd): - try readFd?.close() - $0 = .customWrite(nil, writeFd) - case .fileDescriptor(let fd, let closeWhenDone): - // User passed in fd - if closeWhenDone { - try fd?.close() - $0 = .fileDescriptor(nil, closeWhenDone) - } - } - } - } - - internal func closeParentSide() throws { - try self.storage.withLock { - switch $0 { - case .noInput(_), .fileDescriptor(_, _): - break - case .customWrite(let readFd, let writeFd): - // The parent fd should have been closed - // in the `body` when writer.finish() is called - // But in case it isn't call it agian - try writeFd?.close() - $0 = .customWrite(readFd, nil) - } - } - } - - internal func closeAll() throws { - try self.storage.withLock { - switch $0 { - case .noInput(let readFd): - try readFd?.close() - $0 = .noInput(nil) - case .customWrite(let readFd, let writeFd): - var readFdCloseError: Error? - var writeFdCloseError: Error? - do { - try readFd?.close() - } catch { - readFdCloseError = error - } - do { - try writeFd?.close() - } catch { - writeFdCloseError = error - } - $0 = .customWrite(nil, nil) - if let readFdCloseError { - throw readFdCloseError - } - if let writeFdCloseError { - throw writeFdCloseError - } - case .fileDescriptor(let fd, let closeWhenDone): - if closeWhenDone { - try fd?.close() - $0 = .fileDescriptor(nil, closeWhenDone) - } - } - } - } - - public func hash(into hasher: inout Hasher) { - self.storage.withLock { - hasher.combine($0) - } - } - - public static func == ( - lhs: Subprocess.ExecutionInput, - rhs: Subprocess.ExecutionInput - ) -> Bool { - return lhs.storage.withLock { lhsStorage in - rhs.storage.withLock { rhsStorage in - return lhsStorage == rhsStorage - } - } - } - } - - internal final class ExecutionOutput: Sendable { - internal enum Storage: Sendable { - case discarded(FileDescriptor?) - case fileDescriptor(FileDescriptor?, Bool) - case collected(Int, FileDescriptor?, FileDescriptor?) - } - - private let storage: _Mutex - - internal init(storage: Storage) { - self.storage = .init(storage) - } - - internal func getWriteFileDescriptor() -> FileDescriptor? { - return self.storage.withLock { - switch $0 { - case .discarded(let writeFd): - return writeFd - case .fileDescriptor(let writeFd, _): - return writeFd - case .collected(_, _, let writeFd): - return writeFd - } - } - } - - internal func getReadFileDescriptor() -> FileDescriptor? { - return self.storage.withLock { - switch $0 { - case .discarded(_), .fileDescriptor(_, _): - return nil - case .collected(_, let readFd, _): - return readFd - } - } - } - - internal func consumeCollectedFileDescriptor() -> (limit: Int, fd: FileDescriptor?)? { - return self.storage.withLock { - switch $0 { - case .discarded(_), .fileDescriptor(_, _): - // The output has been written somewhere else - return nil - case .collected(let limit, let readFd, let writeFd): - $0 = .collected(limit, nil, writeFd) - return (limit, readFd) - } - } - } - - internal func closeChildSide() throws { - try self.storage.withLock { - switch $0 { - case .discarded(let writeFd): - try writeFd?.close() - $0 = .discarded(nil) - case .fileDescriptor(let fd, let closeWhenDone): - // User passed fd - if closeWhenDone { - try fd?.close() - $0 = .fileDescriptor(nil, closeWhenDone) - } - case .collected(let limit, let readFd, let writeFd): - try writeFd?.close() - $0 = .collected(limit, readFd, nil) - } - } - } - - internal func closeParentSide() throws { - try self.storage.withLock { - switch $0 { - case .discarded(_), .fileDescriptor(_, _): - break - case .collected(let limit, let readFd, let writeFd): - try readFd?.close() - $0 = .collected(limit, nil, writeFd) - } - } - } - - internal func closeAll() throws { - try self.storage.withLock { - switch $0 { - case .discarded(let writeFd): - try writeFd?.close() - $0 = .discarded(nil) - case .fileDescriptor(let fd, let closeWhenDone): - if closeWhenDone { - try fd?.close() - $0 = .fileDescriptor(nil, closeWhenDone) - } - case .collected(let limit, let readFd, let writeFd): - var readFdCloseError: Error? - var writeFdCloseError: Error? - do { - try readFd?.close() - } catch { - readFdCloseError = error - } - do { - try writeFd?.close() - } catch { - writeFdCloseError = error - } - $0 = .collected(limit, nil, nil) - if let readFdCloseError { - throw readFdCloseError - } - if let writeFdCloseError { - throw writeFdCloseError - } - } - } - } - } -} - diff --git a/Sources/_Subprocess/Subprocess+Teardown.swift b/Sources/_Subprocess/Subprocess+Teardown.swift deleted file mode 100644 index 8b9af32d..00000000 --- a/Sources/_Subprocess/Subprocess+Teardown.swift +++ /dev/null @@ -1,125 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift.org project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -#if canImport(Darwin) || canImport(Glibc) - -#if canImport(Darwin) -import Darwin -#elseif canImport(Glibc) -import Glibc -#endif - -#if canImport(FoundationEssentials) -import FoundationEssentials -#elseif canImport(Foundation) -import Foundation -#endif - -extension Subprocess { - /// A step in the graceful shutdown teardown sequence. - /// It consists of a signal to send to the child process and the - /// number of nanoseconds allowed for the child process to exit - /// before proceeding to the next step. - public struct TeardownStep: Sendable, Hashable { - internal enum Storage: Sendable, Hashable { - case sendSignal(Signal, allowedNanoseconds: UInt64) - case kill - } - var storage: Storage - - /// Sends `signal` to the process and provides `allowedNanoSecondsToExit` - /// nanoseconds for the process to exit before proceeding to the next step. - /// The final step in the sequence will always send a `.kill` signal. - public static func sendSignal( - _ signal: Signal, - allowedNanoSecondsToExit: UInt64 - ) -> Self { - return Self( - storage: .sendSignal( - signal, - allowedNanoseconds: allowedNanoSecondsToExit - ) - ) - } - } -} - -extension Subprocess { - internal func runTeardownSequence(_ sequence: [TeardownStep]) async { - // First insert the `.kill` step - let finalSequence = sequence + [TeardownStep(storage: .kill)] - for step in finalSequence { - enum TeardownStepCompletion { - case processHasExited - case processStillAlive - case killedTheProcess - } - let stepCompletion: TeardownStepCompletion - - guard self.isAlive() else { - return - } - - switch step.storage { - case .sendSignal(let signal, let allowedNanoseconds): - stepCompletion = await withTaskGroup(of: TeardownStepCompletion.self) { group in - group.addTask { - do { - try await Task.sleep(nanoseconds: allowedNanoseconds) - return .processStillAlive - } catch { - // teardown(using:) cancells this task - // when process has exited - return .processHasExited - } - } - try? self.send(signal, toProcessGroup: false) - return await group.next()! - } - case .kill: - try? self.send(.kill, toProcessGroup: false) - stepCompletion = .killedTheProcess - } - - switch stepCompletion { - case .killedTheProcess, .processHasExited: - return - case .processStillAlive: - // Continue to next step - break - } - } - } -} - -extension Subprocess { - private func isAlive() -> Bool { - return kill(self.processIdentifier.value, 0) == 0 - } -} - -func withUncancelledTask( - returning: R.Type = R.self, - _ body: @Sendable @escaping () async -> R -) async -> R { - // This looks unstructured but it isn't, please note that we `await` `.value` of this task. - // The reason we need this separate `Task` is that in general, we cannot assume that code performs to our - // expectations if the task we run it on is already cancelled. However, in some cases we need the code to - // run regardless -- even if our task is already cancelled. Therefore, we create a new, uncancelled task here. - await Task { - await body() - }.value -} - -#endif // canImport(Darwin) || canImport(Glibc) diff --git a/Sources/_Subprocess/Subprocess.swift b/Sources/_Subprocess/Subprocess.swift deleted file mode 100644 index 35f4a2c2..00000000 --- a/Sources/_Subprocess/Subprocess.swift +++ /dev/null @@ -1,315 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift.org project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -import SystemPackage - -#if canImport(FoundationEssentials) -import FoundationEssentials -#elseif canImport(Foundation) -import Foundation -#endif - -/// An object that represents a subprocess of the current process. -/// -/// Using `Subprocess`, your program can run another program as a subprocess -/// and can monitor that program’s execution. A `Subprocess` object creates a -/// **separate executable** entity; it’s different from `Thread` because it doesn’t -/// share memory space with the process that creates it. -public struct Subprocess: Sendable { - /// The process identifier of the current subprocess - public let processIdentifier: ProcessIdentifier - - internal let executionInput: ExecutionInput - internal let executionOutput: ExecutionOutput - internal let executionError: ExecutionOutput -#if os(Windows) - internal let consoleBehavior: PlatformOptions.ConsoleBehavior -#endif - - /// The standard output of the subprocess. - /// Accessing this property will **fatalError** if - /// - `.output` wasn't set to `.redirectToSequence` when the subprocess was spawned; - /// - This property was accessed multiple times. Subprocess communicates with - /// parent process via pipe under the hood and each pipe can only be consumed ones. - public var standardOutput: some _AsyncSequence { - guard let (_, fd) = self.executionOutput - .consumeCollectedFileDescriptor() else { - fatalError("The standard output was not redirected") - } - guard let fd = fd else { - fatalError("The standard output has already been closed") - } - return AsyncDataSequence(fileDescriptor: fd) - } - - /// The standard error of the subprocess. - /// Accessing this property will **fatalError** if - /// - `.error` wasn't set to `.redirectToSequence` when the subprocess was spawned; - /// - This property was accessed multiple times. Subprocess communicates with - /// parent process via pipe under the hood and each pipe can only be consumed ones. - public var standardError: some _AsyncSequence { - guard let (_, fd) = self.executionError - .consumeCollectedFileDescriptor() else { - fatalError("The standard error was not redirected") - } - guard let fd = fd else { - fatalError("The standard error has already been closed") - } - return AsyncDataSequence(fileDescriptor: fd) - } -} - -// MARK: - Teardown -#if canImport(Darwin) || canImport(Glibc) -extension Subprocess { - /// Performs a sequence of teardown steps on the Subprocess. - /// Teardown sequence always ends with a `.kill` signal - /// - Parameter sequence: The steps to perform. - public func teardown(using sequence: [TeardownStep]) async { - await withUncancelledTask { - await self.runTeardownSequence(sequence) - } - } -} -#endif - -// MARK: - StandardInputWriter -extension Subprocess { - /// A writer that writes to the standard input of the subprocess. - public struct StandardInputWriter { - - private let input: ExecutionInput - - init(input: ExecutionInput) { - self.input = input - } - - /// Write a sequence of UInt8 to the standard input of the subprocess. - /// - Parameter sequence: The sequence of bytes to write. - public func write(_ sequence: S) async throws where S : Sequence, S.Element == UInt8 { - guard let fd: FileDescriptor = self.input.getWriteFileDescriptor() else { - fatalError("Attempting to write to a file descriptor that's already closed") - } - try await fd.write(sequence) - } - - /// Write a sequence of CChar to the standard input of the subprocess. - /// - Parameter sequence: The sequence of bytes to write. - public func write(_ sequence: S) async throws where S : Sequence, S.Element == CChar { - try await self.write(sequence.map { UInt8($0) }) - } - - /// Write a AsyncSequence of CChar to the standard input of the subprocess. - /// - Parameter sequence: The sequence of bytes to write. - public func write(_ asyncSequence: S) async throws where S.Element == CChar { - let sequence = try await Array(asyncSequence).map { UInt8($0) } - try await self.write(sequence) - } - - /// Write a AsyncSequence of UInt8 to the standard input of the subprocess. - /// - Parameter sequence: The sequence of bytes to write. - public func write(_ asyncSequence: S) async throws where S.Element == UInt8 { - let sequence = try await Array(asyncSequence) - try await self.write(sequence) - } - - /// Signal all writes are finished - public func finish() async throws { - try self.input.closeParentSide() - } - } -} - -@available(macOS, unavailable) -@available(iOS, unavailable) -@available(tvOS, unavailable) -@available(watchOS, unavailable) -@available(*, unavailable) -extension Subprocess.StandardInputWriter : Sendable {} - -// MARK: - Result -extension Subprocess { - /// A simple wrapper around the generic result returned by the - /// `run` closures with the corresponding `TerminationStatus` - /// of the child process. - public struct ExecutionResult: Sendable { - /// The termination status of the child process - public let terminationStatus: TerminationStatus - /// The result returned by the closure passed to `.run` methods - public let value: T - - internal init(terminationStatus: TerminationStatus, value: T) { - self.terminationStatus = terminationStatus - self.value = value - } - } - - /// The result of a subprocess execution with its collected - /// standard output and standard error. - public struct CollectedResult: Sendable, Hashable, Codable { - /// The process identifier for the executed subprocess - public let processIdentifier: ProcessIdentifier - /// The termination status of the executed subprocess - public let terminationStatus: TerminationStatus - private let _standardOutput: Data? - private let _standardError: Data? - - /// The collected standard output value for the subprocess. - /// Accessing this property will *fatalError* if the - /// corresponding `CollectedOutputMethod` is not set to - /// `.collect` or `.collect(upTo:)` - public var standardOutput: Data { - guard let output = self._standardOutput else { - fatalError("standardOutput is only available if the Subprocess was ran with .collect as output") - } - return output - } - /// The collected standard error value for the subprocess. - /// Accessing this property will *fatalError* if the - /// corresponding `CollectedOutputMethod` is not set to - /// `.collect` or `.collect(upTo:)` - public var standardError: Data { - guard let output = self._standardError else { - fatalError("standardError is only available if the Subprocess was ran with .collect as error ") - } - return output - } - - internal init( - processIdentifier: ProcessIdentifier, - terminationStatus: TerminationStatus, - standardOutput: Data?, - standardError: Data?) { - self.processIdentifier = processIdentifier - self.terminationStatus = terminationStatus - self._standardOutput = standardOutput - self._standardError = standardError - } - } -} - -extension Subprocess.ExecutionResult: Equatable where T : Equatable {} - -extension Subprocess.ExecutionResult: Hashable where T : Hashable {} - -extension Subprocess.ExecutionResult: Codable where T : Codable {} - -extension Subprocess.ExecutionResult: CustomStringConvertible where T : CustomStringConvertible { - public var description: String { - return """ -Subprocess.ExecutionResult( - terminationStatus: \(self.terminationStatus.description), - value: \(self.value.description) -) -""" - } -} - -extension Subprocess.ExecutionResult: CustomDebugStringConvertible where T : CustomDebugStringConvertible { - public var debugDescription: String { - return """ -Subprocess.ExecutionResult( - terminationStatus: \(self.terminationStatus.debugDescription), - value: \(self.value.debugDescription) -) -""" - } -} - -extension Subprocess.CollectedResult : CustomStringConvertible, CustomDebugStringConvertible { - public var description: String { - return """ -Subprocess.CollectedResult( - processIdentifier: \(self.processIdentifier.description), - terminationStatus: \(self.terminationStatus.description), - standardOutput: \(self._standardOutput?.description ?? "not captured"), - standardError: \(self._standardError?.description ?? "not captured") -) -""" - } - - public var debugDescription: String { - return """ -Subprocess.CollectedResult( - processIdentifier: \(self.processIdentifier.debugDescription), - terminationStatus: \(self.terminationStatus.debugDescription), - standardOutput: \(self._standardOutput?.debugDescription ?? "not captured"), - standardError: \(self._standardError?.debugDescription ?? "not captured") -) -""" - } -} - -// MARK: - Internal -extension Subprocess { - internal enum OutputCapturingState { - case standardOutputCaptured(Data?) - case standardErrorCaptured(Data?) - } - - internal typealias CapturedIOs = (standardOutput: Data?, standardError: Data?) - - private func capture(fileDescriptor: FileDescriptor, maxLength: Int) async throws -> Data { - return try await fileDescriptor.readUntilEOF(upToLength: maxLength) - } - - internal func captureStandardOutput() async throws -> Data? { - guard let (limit, readFd) = self.executionOutput - .consumeCollectedFileDescriptor(), - let readFd = readFd else { - return nil - } - defer { - try? readFd.close() - } - return try await self.capture(fileDescriptor: readFd, maxLength: limit) - } - - internal func captureStandardError() async throws -> Data? { - guard let (limit, readFd) = self.executionError - .consumeCollectedFileDescriptor(), - let readFd = readFd else { - return nil - } - defer { - try? readFd.close() - } - return try await self.capture(fileDescriptor: readFd, maxLength: limit) - } - - internal func captureIOs() async throws -> CapturedIOs { - return try await withThrowingTaskGroup(of: OutputCapturingState.self) { group in - group.addTask { - let stdout = try await self.captureStandardOutput() - return .standardOutputCaptured(stdout) - } - group.addTask { - let stderr = try await self.captureStandardError() - return .standardErrorCaptured(stderr) - } - - var stdout: Data? - var stderror: Data? - while let state = try await group.next() { - switch state { - case .standardOutputCaptured(let output): - stdout = output - case .standardErrorCaptured(let error): - stderror = error - } - } - return (standardOutput: stdout, standardError: stderror) - } - } -} diff --git a/Sources/_Subprocess/SubprocessFoundation/Input+Foundation.swift b/Sources/_Subprocess/SubprocessFoundation/Input+Foundation.swift new file mode 100644 index 00000000..3ec42173 --- /dev/null +++ b/Sources/_Subprocess/SubprocessFoundation/Input+Foundation.swift @@ -0,0 +1,179 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +#if SubprocessFoundation + +#if canImport(Darwin) +// On Darwin always prefer system Foundation +import Foundation +#else +// On other platforms prefer FoundationEssentials +import FoundationEssentials +#endif // canImport(Darwin) + +#if canImport(System) +import System +#else +@preconcurrency import SystemPackage +#endif + +internal import Dispatch + +/// A concrete `Input` type for subprocesses that reads input +/// from a given `Data`. +public struct DataInput: InputProtocol { + private let data: Data + + public func write(with writer: StandardInputWriter) async throws { + _ = try await writer.write(self.data) + } + + internal init(data: Data) { + self.data = data + } +} + +/// A concrete `Input` type for subprocesses that accepts input +/// from a specified sequence of `Data`. +public struct DataSequenceInput< + InputSequence: Sequence & Sendable +>: InputProtocol where InputSequence.Element == Data { + private let sequence: InputSequence + + public func write(with writer: StandardInputWriter) async throws { + var buffer = Data() + for chunk in self.sequence { + buffer.append(chunk) + } + _ = try await writer.write(buffer) + } + + internal init(underlying: InputSequence) { + self.sequence = underlying + } +} + +/// A concrete `Input` type for subprocesses that reads input +/// from a given async sequence of `Data`. +public struct DataAsyncSequenceInput< + InputSequence: AsyncSequence & Sendable +>: InputProtocol where InputSequence.Element == Data { + private let sequence: InputSequence + + private func writeChunk(_ chunk: Data, with writer: StandardInputWriter) async throws { + _ = try await writer.write(chunk) + } + + public func write(with writer: StandardInputWriter) async throws { + for try await chunk in self.sequence { + try await self.writeChunk(chunk, with: writer) + } + } + + internal init(underlying: InputSequence) { + self.sequence = underlying + } +} + +extension InputProtocol { + /// Create a Subprocess input from a `Data` + public static func data(_ data: Data) -> Self where Self == DataInput { + return DataInput(data: data) + } + + /// Create a Subprocess input from a `Sequence` of `Data`. + public static func sequence( + _ sequence: InputSequence + ) -> Self where Self == DataSequenceInput { + return .init(underlying: sequence) + } + + /// Create a Subprocess input from a `AsyncSequence` of `Data`. + public static func sequence( + _ asyncSequence: InputSequence + ) -> Self where Self == DataAsyncSequenceInput { + return .init(underlying: asyncSequence) + } +} + +extension StandardInputWriter { + /// Write a `Data` to the standard input of the subprocess. + /// - Parameter data: The sequence of bytes to write. + /// - Returns number of bytes written. + public func write( + _ data: Data + ) async throws -> Int { + return try await self.fileDescriptor.wrapped.write(data) + } + + /// Write a AsyncSequence of Data to the standard input of the subprocess. + /// - Parameter sequence: The sequence of bytes to write. + /// - Returns number of bytes written. + public func write( + _ asyncSequence: AsyncSendableSequence + ) async throws -> Int where AsyncSendableSequence.Element == Data { + var buffer = Data() + for try await data in asyncSequence { + buffer.append(data) + } + return try await self.write(buffer) + } +} + +extension FileDescriptor { + #if os(Windows) + internal func write( + _ data: Data + ) async throws -> Int { + try await withCheckedThrowingContinuation { continuation in + // TODO: Figure out a better way to asynchornously write + DispatchQueue.global(qos: .userInitiated).async { + data.withUnsafeBytes { + self.write($0) { writtenLength, error in + if let error = error { + continuation.resume(throwing: error) + } else { + continuation.resume(returning: writtenLength) + } + } + } + } + } + } + #else + internal func write( + _ data: Data + ) async throws -> Int { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + let dispatchData = data.withUnsafeBytes { + return DispatchData( + bytesNoCopy: $0, + deallocator: .custom( + nil, + { + // noop + } + ) + ) + } + self.write(dispatchData) { writtenLength, error in + if let error = error { + continuation.resume(throwing: error) + } else { + continuation.resume(returning: writtenLength) + } + } + } + } + #endif +} + +#endif // SubprocessFoundation diff --git a/Sources/_Subprocess/SubprocessFoundation/Output+Foundation.swift b/Sources/_Subprocess/SubprocessFoundation/Output+Foundation.swift new file mode 100644 index 00000000..fac0bb8f --- /dev/null +++ b/Sources/_Subprocess/SubprocessFoundation/Output+Foundation.swift @@ -0,0 +1,53 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +#if SubprocessFoundation + +#if canImport(Darwin) +// On Darwin always prefer system Foundation +import Foundation +#else +// On other platforms prefer FoundationEssentials +import FoundationEssentials +#endif + +/// A concrete `Output` type for subprocesses that collects output +/// from the subprocess as `Data`. This option must be used with +/// the `run()` method that returns a `CollectedResult` +public struct DataOutput: OutputProtocol { + public typealias OutputType = Data + public let maxSize: Int + + public func output(from buffer: some Sequence) throws -> Data { + return Data(buffer) + } + + internal init(limit: Int) { + self.maxSize = limit + } +} + +extension OutputProtocol where Self == DataOutput { + /// Create a `Subprocess` output that collects output as `Data` + /// up to 128kb. + public static var data: Self { + return .data(limit: 128 * 1024) + } + + /// Create a `Subprocess` output that collects output as `Data` + /// with given max number of bytes to collect. + public static func data(limit: Int) -> Self { + return .init(limit: limit) + } +} + + +#endif // SubprocessFoundation diff --git a/Sources/_Subprocess/SubprocessFoundation/Span+SubprocessFoundation.swift b/Sources/_Subprocess/SubprocessFoundation/Span+SubprocessFoundation.swift new file mode 100644 index 00000000..10697091 --- /dev/null +++ b/Sources/_Subprocess/SubprocessFoundation/Span+SubprocessFoundation.swift @@ -0,0 +1,76 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +#if SubprocessFoundation && SubprocessSpan + +#if canImport(Darwin) +// On Darwin always prefer system Foundation +import Foundation +#else +// On other platforms prefer FoundationEssentials +import FoundationEssentials +#endif // canImport(Darwin) + +internal import Dispatch + + +extension Data { + init(_ s: borrowing RawSpan) { + self = s.withUnsafeBytes { Data($0) } + } + + public var bytes: RawSpan { + // FIXME: For demo purpose only + let ptr = self.withUnsafeBytes { ptr in + return ptr + } + let span = RawSpan(_unsafeBytes: ptr) + return _overrideLifetime(of: span, to: self) + } +} + + +extension DataProtocol { + var bytes: RawSpan { + _read { + if self.regions.isEmpty { + let empty = UnsafeRawBufferPointer(start: nil, count: 0) + let span = RawSpan(_unsafeBytes: empty) + yield _overrideLifetime(of: span, to: self) + } else if self.regions.count == 1 { + // Easy case: there is only one region in the data + let ptr = self.regions.first!.withUnsafeBytes { ptr in + return ptr + } + let span = RawSpan(_unsafeBytes: ptr) + yield _overrideLifetime(of: span, to: self) + } else { + // This data contains discontiguous chunks. We have to + // copy and make a contiguous chunk + var contiguous: ContiguousArray? + for region in self.regions { + if contiguous != nil { + contiguous?.append(contentsOf: region) + } else { + contiguous = .init(region) + } + } + let ptr = contiguous!.withUnsafeBytes { ptr in + return ptr + } + let span = RawSpan(_unsafeBytes: ptr) + yield _overrideLifetime(of: span, to: self) + } + } + } +} + +#endif // SubprocessFoundation diff --git a/Sources/_Subprocess/Teardown.swift b/Sources/_Subprocess/Teardown.swift new file mode 100644 index 00000000..b2da85a5 --- /dev/null +++ b/Sources/_Subprocess/Teardown.swift @@ -0,0 +1,217 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +import _SubprocessCShims + +#if canImport(Darwin) +import Darwin +#elseif canImport(Bionic) +import Bionic +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#elseif canImport(WinSDK) +import WinSDK +#endif + +/// A step in the graceful shutdown teardown sequence. +/// It consists of an action to perform on the child process and the +/// duration allowed for the child process to exit before proceeding +/// to the next step. +public struct TeardownStep: Sendable, Hashable { + internal enum Storage: Sendable, Hashable { + #if !os(Windows) + case sendSignal(Signal, allowedDuration: Duration) + #endif + case gracefulShutDown(allowedDuration: Duration) + case kill + } + var storage: Storage + + #if !os(Windows) + /// Sends `signal` to the process and allows `allowedDurationToExit` + /// for the process to exit before proceeding to the next step. + /// The final step in the sequence will always send a `.kill` signal. + public static func send( + signal: Signal, + allowedDurationToNextStep: Duration + ) -> Self { + return Self( + storage: .sendSignal( + signal, + allowedDuration: allowedDurationToNextStep + ) + ) + } + #endif // !os(Windows) + + /// Attempt to perform a graceful shutdown and allows + /// `allowedDurationToNextStep` for the process to exit + /// before proceeding to the next step: + /// - On Unix: send `SIGTERM` + /// - On Windows: + /// 1. Attempt to send `VM_CLOSE` if the child process is a GUI process; + /// 2. Attempt to send `CTRL_C_EVENT` to console; + /// 3. Attempt to send `CTRL_BREAK_EVENT` to process group. + public static func gracefulShutDown( + allowedDurationToNextStep: Duration + ) -> Self { + return Self( + storage: .gracefulShutDown( + allowedDuration: allowedDurationToNextStep + ) + ) + } +} + +@available(macOS 15.0, *) // FIXME: manually added availability +extension Execution { + /// Performs a sequence of teardown steps on the Subprocess. + /// Teardown sequence always ends with a `.kill` signal + /// - Parameter sequence: The steps to perform. + public func teardown(using sequence: some Sequence & Sendable) async { + await withUncancelledTask { + await self.runTeardownSequence(sequence) + } + } +} + +internal enum TeardownStepCompletion { + case processHasExited + case processStillAlive + case killedTheProcess +} + +@available(macOS 15.0, *) // FIXME: manually added availability +extension Execution { + internal func gracefulShutDown( + allowedDurationToNextStep duration: Duration + ) async { + #if os(Windows) + guard + let processHandle = OpenProcess( + DWORD(PROCESS_QUERY_INFORMATION | SYNCHRONIZE), + false, + self.processIdentifier.value + ) + else { + // Nothing more we can do + return + } + defer { + CloseHandle(processHandle) + } + + // 1. Attempt to send WM_CLOSE to the main window + if _subprocess_windows_send_vm_close( + self.processIdentifier.value + ) { + try? await Task.sleep(for: duration) + } + + // 2. Attempt to attach to the console and send CTRL_C_EVENT + if AttachConsole(self.processIdentifier.value) { + // Disable Ctrl-C handling in this process + if SetConsoleCtrlHandler(nil, true) { + if GenerateConsoleCtrlEvent(DWORD(CTRL_C_EVENT), 0) { + // We successfully sent the event. wait for the process to exit + try? await Task.sleep(for: duration) + } + // Re-enable Ctrl-C handling + SetConsoleCtrlHandler(nil, false) + } + // Detach console + FreeConsole() + } + + // 3. Attempt to send CTRL_BREAK_EVENT to the process group + if GenerateConsoleCtrlEvent(DWORD(CTRL_BREAK_EVENT), self.processIdentifier.value) { + // Wait for process to exit + try? await Task.sleep(for: duration) + } + #else + // Send SIGTERM + try? self.send(signal: .terminate) + #endif + } + + internal func runTeardownSequence(_ sequence: some Sequence & Sendable) async { + // First insert the `.kill` step + let finalSequence = sequence + [TeardownStep(storage: .kill)] + for step in finalSequence { + let stepCompletion: TeardownStepCompletion + + switch step.storage { + case .gracefulShutDown(let allowedDuration): + stepCompletion = await withTaskGroup(of: TeardownStepCompletion.self) { group in + group.addTask { + do { + try await Task.sleep(for: allowedDuration) + return .processStillAlive + } catch { + // teardown(using:) cancells this task + // when process has exited + return .processHasExited + } + } + await self.gracefulShutDown(allowedDurationToNextStep: allowedDuration) + return await group.next()! + } + #if !os(Windows) + case .sendSignal(let signal, let allowedDuration): + stepCompletion = await withTaskGroup(of: TeardownStepCompletion.self) { group in + group.addTask { + do { + try await Task.sleep(for: allowedDuration) + return .processStillAlive + } catch { + // teardown(using:) cancells this task + // when process has exited + return .processHasExited + } + } + try? self.send(signal: signal) + return await group.next()! + } + #endif // !os(Windows) + case .kill: + #if os(Windows) + try? self.terminate(withExitCode: 0) + #else + try? self.send(signal: .kill) + #endif + stepCompletion = .killedTheProcess + } + + switch stepCompletion { + case .killedTheProcess, .processHasExited: + return + case .processStillAlive: + // Continue to next step + break + } + } + } +} + +func withUncancelledTask( + returning: Result.Type = Result.self, + _ body: @Sendable @escaping () async -> Result +) async -> Result { + // This looks unstructured but it isn't, please note that we `await` `.value` of this task. + // The reason we need this separate `Task` is that in general, we cannot assume that code performs to our + // expectations if the task we run it on is already cancelled. However, in some cases we need the code to + // run regardless -- even if our task is already cancelled. Therefore, we create a new, uncancelled task here. + await Task { + await body() + }.value +} diff --git a/Sources/_Subprocess/_nio_locks.swift b/Sources/_Subprocess/_nio_locks.swift deleted file mode 100644 index 49053d04..00000000 --- a/Sources/_Subprocess/_nio_locks.swift +++ /dev/null @@ -1,526 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the SwiftNIO open source project -// -// Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of SwiftNIO project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -#if canImport(Darwin) -import Darwin -#elseif os(Windows) -import ucrt -import WinSDK -#elseif canImport(Glibc) -import Glibc -#elseif canImport(Musl) -import Musl -#else -#error("The concurrency lock module was unable to identify your C library.") -#endif - -/// A threading lock based on `libpthread` instead of `libdispatch`. -/// -/// This object provides a lock on top of a single `pthread_mutex_t`. This kind -/// of lock is safe to use with `libpthread`-based threading models, such as the -/// one used by NIO. On Windows, the lock is based on the substantially similar -/// `SRWLOCK` type. -@available(*, deprecated, renamed: "NIOLock") -public final class Lock { -#if os(Windows) - fileprivate let mutex: UnsafeMutablePointer = - UnsafeMutablePointer.allocate(capacity: 1) -#else - fileprivate let mutex: UnsafeMutablePointer = - UnsafeMutablePointer.allocate(capacity: 1) -#endif - - /// Create a new lock. - public init() { -#if os(Windows) - InitializeSRWLock(self.mutex) -#else - var attr = pthread_mutexattr_t() - pthread_mutexattr_init(&attr) - debugOnly { - pthread_mutexattr_settype(&attr, .init(PTHREAD_MUTEX_ERRORCHECK)) - } - - let err = pthread_mutex_init(self.mutex, &attr) - precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)") -#endif - } - - deinit { -#if os(Windows) - // SRWLOCK does not need to be free'd -#else - let err = pthread_mutex_destroy(self.mutex) - precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)") -#endif - mutex.deallocate() - } - - /// Acquire the lock. - /// - /// Whenever possible, consider using `withLock` instead of this method and - /// `unlock`, to simplify lock handling. - public func lock() { -#if os(Windows) - AcquireSRWLockExclusive(self.mutex) -#else - let err = pthread_mutex_lock(self.mutex) - precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)") -#endif - } - - /// Release the lock. - /// - /// Whenever possible, consider using `withLock` instead of this method and - /// `lock`, to simplify lock handling. - public func unlock() { -#if os(Windows) - ReleaseSRWLockExclusive(self.mutex) -#else - let err = pthread_mutex_unlock(self.mutex) - precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)") -#endif - } - - /// Acquire the lock for the duration of the given block. - /// - /// This convenience method should be preferred to `lock` and `unlock` in - /// most situations, as it ensures that the lock will be released regardless - /// of how `body` exits. - /// - /// - Parameter body: The block to execute while holding the lock. - /// - Returns: The value returned by the block. - @inlinable - public func withLock(_ body: () throws -> T) rethrows -> T { - self.lock() - defer { - self.unlock() - } - return try body() - } - - // specialise Void return (for performance) - @inlinable - public func withLockVoid(_ body: () throws -> Void) rethrows -> Void { - try self.withLock(body) - } -} - -/// A `Lock` with a built-in state variable. -/// -/// This class provides a convenience addition to `Lock`: it provides the ability to wait -/// until the state variable is set to a specific value to acquire the lock. -public final class ConditionLock { - private var _value: T - private let mutex: NIOLock -#if os(Windows) - private let cond: UnsafeMutablePointer = - UnsafeMutablePointer.allocate(capacity: 1) -#else - private let cond: UnsafeMutablePointer = - UnsafeMutablePointer.allocate(capacity: 1) -#endif - - /// Create the lock, and initialize the state variable to `value`. - /// - /// - Parameter value: The initial value to give the state variable. - public init(value: T) { - self._value = value - self.mutex = NIOLock() -#if os(Windows) - InitializeConditionVariable(self.cond) -#else - let err = pthread_cond_init(self.cond, nil) - precondition(err == 0, "\(#function) failed in pthread_cond with error \(err)") -#endif - } - - deinit { -#if os(Windows) - // condition variables do not need to be explicitly destroyed -#else - let err = pthread_cond_destroy(self.cond) - precondition(err == 0, "\(#function) failed in pthread_cond with error \(err)") -#endif - self.cond.deallocate() - } - - /// Acquire the lock, regardless of the value of the state variable. - public func lock() { - self.mutex.lock() - } - - /// Release the lock, regardless of the value of the state variable. - public func unlock() { - self.mutex.unlock() - } - - /// The value of the state variable. - /// - /// Obtaining the value of the state variable requires acquiring the lock. - /// This means that it is not safe to access this property while holding the - /// lock: it is only safe to use it when not holding it. - public var value: T { - self.lock() - defer { - self.unlock() - } - return self._value - } - - /// Acquire the lock when the state variable is equal to `wantedValue`. - /// - /// - Parameter wantedValue: The value to wait for the state variable - /// to have before acquiring the lock. - public func lock(whenValue wantedValue: T) { - self.lock() - while true { - if self._value == wantedValue { - break - } - self.mutex.withLockPrimitive { mutex in -#if os(Windows) - let result = SleepConditionVariableSRW(self.cond, mutex, INFINITE, 0) - precondition(result, "\(#function) failed in SleepConditionVariableSRW with error \(GetLastError())") -#else - let err = pthread_cond_wait(self.cond, mutex) - precondition(err == 0, "\(#function) failed in pthread_cond with error \(err)") -#endif - } - } - } - - /// Acquire the lock when the state variable is equal to `wantedValue`, - /// waiting no more than `timeoutSeconds` seconds. - /// - /// - Parameter wantedValue: The value to wait for the state variable - /// to have before acquiring the lock. - /// - Parameter timeoutSeconds: The number of seconds to wait to acquire - /// the lock before giving up. - /// - Returns: `true` if the lock was acquired, `false` if the wait timed out. - public func lock(whenValue wantedValue: T, timeoutSeconds: Double) -> Bool { - precondition(timeoutSeconds >= 0) - -#if os(Windows) - var dwMilliseconds: DWORD = DWORD(timeoutSeconds * 1000) - - self.lock() - while true { - if self._value == wantedValue { - return true - } - - let dwWaitStart = timeGetTime() - if !SleepConditionVariableSRW(self.cond, self.mutex._storage.mutex, - dwMilliseconds, 0) { - let dwError = GetLastError() - if (dwError == ERROR_TIMEOUT) { - self.unlock() - return false - } - fatalError("SleepConditionVariableSRW: \(dwError)") - } - - // NOTE: this may be a spurious wakeup, adjust the timeout accordingly - dwMilliseconds = dwMilliseconds - (timeGetTime() - dwWaitStart) - } -#else - let nsecPerSec: Int64 = 1000000000 - self.lock() - /* the timeout as a (seconds, nano seconds) pair */ - let timeoutNS = Int64(timeoutSeconds * Double(nsecPerSec)) - - var curTime = timeval() - gettimeofday(&curTime, nil) - - let allNSecs: Int64 = timeoutNS + Int64(curTime.tv_usec) * 1000 - var timeoutAbs = timespec(tv_sec: curTime.tv_sec + Int((allNSecs / nsecPerSec)), - tv_nsec: Int(allNSecs % nsecPerSec)) - assert(timeoutAbs.tv_nsec >= 0 && timeoutAbs.tv_nsec < Int(nsecPerSec)) - assert(timeoutAbs.tv_sec >= curTime.tv_sec) - return self.mutex.withLockPrimitive { mutex -> Bool in - while true { - if self._value == wantedValue { - return true - } - switch pthread_cond_timedwait(self.cond, mutex, &timeoutAbs) { - case 0: - continue - case ETIMEDOUT: - self.unlock() - return false - case let e: - fatalError("caught error \(e) when calling pthread_cond_timedwait") - } - } - } -#endif - } - - /// Release the lock, setting the state variable to `newValue`. - /// - /// - Parameter newValue: The value to give to the state variable when we - /// release the lock. - public func unlock(withValue newValue: T) { - self._value = newValue - self.unlock() -#if os(Windows) - WakeAllConditionVariable(self.cond) -#else - let err = pthread_cond_broadcast(self.cond) - precondition(err == 0, "\(#function) failed in pthread_cond with error \(err)") -#endif - } -} - -/// A utility function that runs the body code only in debug builds, without -/// emitting compiler warnings. -/// -/// This is currently the only way to do this in Swift: see -/// https://forums.swift.org/t/support-debug-only-code/11037 for a discussion. -@inlinable -internal func debugOnly(_ body: () -> Void) { - assert({ body(); return true }()) -} - -@available(*, deprecated) -extension Lock: @unchecked Sendable {} -extension ConditionLock: @unchecked Sendable {} - -#if os(Windows) -@usableFromInline -typealias LockPrimitive = SRWLOCK -#else -@usableFromInline -typealias LockPrimitive = pthread_mutex_t -#endif - -@usableFromInline -enum LockOperations { } - -extension LockOperations { - @inlinable - static func create(_ mutex: UnsafeMutablePointer) { - mutex.assertValidAlignment() - -#if os(Windows) - InitializeSRWLock(mutex) -#else - var attr = pthread_mutexattr_t() - pthread_mutexattr_init(&attr) - debugOnly { - pthread_mutexattr_settype(&attr, .init(PTHREAD_MUTEX_ERRORCHECK)) - } - - let err = pthread_mutex_init(mutex, &attr) - precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)") -#endif - } - - @inlinable - static func destroy(_ mutex: UnsafeMutablePointer) { - mutex.assertValidAlignment() - -#if os(Windows) - // SRWLOCK does not need to be free'd -#else - let err = pthread_mutex_destroy(mutex) - precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)") -#endif - } - - @inlinable - static func lock(_ mutex: UnsafeMutablePointer) { - mutex.assertValidAlignment() - -#if os(Windows) - AcquireSRWLockExclusive(mutex) -#else - let err = pthread_mutex_lock(mutex) - precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)") -#endif - } - - @inlinable - static func unlock(_ mutex: UnsafeMutablePointer) { - mutex.assertValidAlignment() - -#if os(Windows) - ReleaseSRWLockExclusive(mutex) -#else - let err = pthread_mutex_unlock(mutex) - precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)") -#endif - } -} - -// Tail allocate both the mutex and a generic value using ManagedBuffer. -// Both the header pointer and the elements pointer are stable for -// the class's entire lifetime. -// -// However, for safety reasons, we elect to place the lock in the "elements" -// section of the buffer instead of the head. The reasoning here is subtle, -// so buckle in. -// -// _As a practical matter_, the implementation of ManagedBuffer ensures that -// the pointer to the header is stable across the lifetime of the class, and so -// each time you call `withUnsafeMutablePointers` or `withUnsafeMutablePointerToHeader` -// the value of the header pointer will be the same. This is because ManagedBuffer uses -// `Builtin.addressOf` to load the value of the header, and that does ~magic~ to ensure -// that it does not invoke any weird Swift accessors that might copy the value. -// -// _However_, the header is also available via the `.header` field on the ManagedBuffer. -// This presents a problem! The reason there's an issue is that `Builtin.addressOf` and friends -// do not interact with Swift's exclusivity model. That is, the various `with` functions do not -// conceptually trigger a mutating access to `.header`. For elements this isn't a concern because -// there's literally no other way to perform the access, but for `.header` it's entirely possible -// to accidentally recursively read it. -// -// Our implementation is free from these issues, so we don't _really_ need to worry about it. -// However, out of an abundance of caution, we store the Value in the header, and the LockPrimitive -// in the trailing elements. We still don't use `.header`, but it's better to be safe than sorry, -// and future maintainers will be happier that we were cautious. -// -// See also: https://github.com/apple/swift/pull/40000 -@usableFromInline -final class LockStorage: ManagedBuffer { - - @inlinable - static func create(value: Value) -> Self { - let buffer = Self.create(minimumCapacity: 1) { _ in - return value - } - let storage = unsafeDowncast(buffer, to: Self.self) - - storage.withUnsafeMutablePointers { _, lockPtr in - LockOperations.create(lockPtr) - } - - return storage - } - - @inlinable - func lock() { - self.withUnsafeMutablePointerToElements { lockPtr in - LockOperations.lock(lockPtr) - } - } - - @inlinable - func unlock() { - self.withUnsafeMutablePointerToElements { lockPtr in - LockOperations.unlock(lockPtr) - } - } - - @inlinable - deinit { - self.withUnsafeMutablePointerToElements { lockPtr in - LockOperations.destroy(lockPtr) - } - } - - @inlinable - func withLockPrimitive(_ body: (UnsafeMutablePointer) throws -> T) rethrows -> T { - try self.withUnsafeMutablePointerToElements { lockPtr in - return try body(lockPtr) - } - } - - @inlinable - func withLockedValue(_ mutate: (inout Value) throws -> T) rethrows -> T { - try self.withUnsafeMutablePointers { valuePtr, lockPtr in - LockOperations.lock(lockPtr) - defer { LockOperations.unlock(lockPtr) } - return try mutate(&valuePtr.pointee) - } - } -} - -extension LockStorage: @unchecked Sendable { } - -/// A threading lock based on `libpthread` instead of `libdispatch`. -/// -/// - note: ``NIOLock`` has reference semantics. -/// -/// This object provides a lock on top of a single `pthread_mutex_t`. This kind -/// of lock is safe to use with `libpthread`-based threading models, such as the -/// one used by NIO. On Windows, the lock is based on the substantially similar -/// `SRWLOCK` type. -public struct NIOLock { - @usableFromInline - internal let _storage: LockStorage - - /// Create a new lock. - @inlinable - public init() { - self._storage = .create(value: ()) - } - - /// Acquire the lock. - /// - /// Whenever possible, consider using `withLock` instead of this method and - /// `unlock`, to simplify lock handling. - @inlinable - public func lock() { - self._storage.lock() - } - - /// Release the lock. - /// - /// Whenever possible, consider using `withLock` instead of this method and - /// `lock`, to simplify lock handling. - @inlinable - public func unlock() { - self._storage.unlock() - } - - @inlinable - internal func withLockPrimitive(_ body: (UnsafeMutablePointer) throws -> T) rethrows -> T { - return try self._storage.withLockPrimitive(body) - } -} - -extension NIOLock { - /// Acquire the lock for the duration of the given block. - /// - /// This convenience method should be preferred to `lock` and `unlock` in - /// most situations, as it ensures that the lock will be released regardless - /// of how `body` exits. - /// - /// - Parameter body: The block to execute while holding the lock. - /// - Returns: The value returned by the block. - @inlinable - public func withLock(_ body: () throws -> T) rethrows -> T { - self.lock() - defer { - self.unlock() - } - return try body() - } - - @inlinable - public func withLockVoid(_ body: () throws -> Void) rethrows -> Void { - try self.withLock(body) - } -} - -extension NIOLock: Sendable {} - -extension UnsafeMutablePointer { - @inlinable - func assertValidAlignment() { - assert(UInt(bitPattern: self) % UInt(MemoryLayout.alignment) == 0) - } -} diff --git a/Sources/_CShims/include/process_shims.h b/Sources/_SubprocessCShims/include/process_shims.h similarity index 74% rename from Sources/_CShims/include/process_shims.h rename to Sources/_SubprocessCShims/include/process_shims.h index 563b517f..35cbd2fe 100644 --- a/Sources/_CShims/include/process_shims.h +++ b/Sources/_SubprocessCShims/include/process_shims.h @@ -2,20 +2,17 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors -// Licensed under Apache License v2.0 +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception // -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift.org project authors -// -// SPDX-License-Identifier: Apache-2.0 +// See https://swift.org/LICENSE.txt for license information // //===----------------------------------------------------------------------===// #ifndef process_shims_h #define process_shims_h -#include "_CShimsTargetConditionals.h" +#include "target_conditionals.h" #if !TARGET_OS_WINDOWS #include @@ -24,6 +21,10 @@ #include #endif +#if __has_include() +vm_size_t _subprocess_vm_size(void); +#endif + #if TARGET_OS_MAC int _subprocess_spawn( pid_t * _Nonnull pid, @@ -60,6 +61,10 @@ int _was_process_signaled(int status); int _get_signal_code(int status); int _was_process_suspended(int status); +void _subprocess_lock_environ(void); +void _subprocess_unlock_environ(void); +char * _Nullable * _Nullable _subprocess_get_environ(void); + #if TARGET_OS_LINUX int _shims_snprintf( char * _Nonnull str, @@ -72,4 +77,15 @@ int _shims_snprintf( #endif // !TARGET_OS_WINDOWS +#if TARGET_OS_WINDOWS + +#ifndef _WINDEF_ +typedef unsigned long DWORD; +typedef int BOOL; +#endif + +BOOL _subprocess_windows_send_vm_close(DWORD pid); + +#endif + #endif /* process_shims_h */ diff --git a/Sources/_CShims/include/_CShimsTargetConditionals.h b/Sources/_SubprocessCShims/include/target_conditionals.h similarity index 76% rename from Sources/_CShims/include/_CShimsTargetConditionals.h rename to Sources/_SubprocessCShims/include/target_conditionals.h index 9e1d80cb..fef2eaf2 100644 --- a/Sources/_CShims/include/_CShimsTargetConditionals.h +++ b/Sources/_SubprocessCShims/include/target_conditionals.h @@ -2,13 +2,11 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors -// Licensed under Apache License v2.0 +// Copyright (c) 2021 - 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception // -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift.org project authors -// -// SPDX-License-Identifier: Apache-2.0 +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors // //===----------------------------------------------------------------------===// diff --git a/Sources/_SubprocessCShims/process_shims.c b/Sources/_SubprocessCShims/process_shims.c new file mode 100644 index 00000000..287e1a8d --- /dev/null +++ b/Sources/_SubprocessCShims/process_shims.c @@ -0,0 +1,682 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +#include "include/target_conditionals.h" + +#if TARGET_OS_LINUX +// For posix_spawn_file_actions_addchdir_np +#define _GNU_SOURCE 1 +#endif + +#include "include/process_shims.h" + +#if TARGET_OS_WINDOWS +#include +#else +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if __has_include() +#include +#elif defined(_WIN32) +#include +#elif __has_include() +#include +extern char **environ; +#endif + +int _was_process_exited(int status) { + return WIFEXITED(status); +} + +int _get_exit_code(int status) { + return WEXITSTATUS(status); +} + +int _was_process_signaled(int status) { + return WIFSIGNALED(status); +} + +int _get_signal_code(int status) { + return WTERMSIG(status); +} + +int _was_process_suspended(int status) { + return WIFSTOPPED(status); +} + +#if TARGET_OS_LINUX +#include + +int _shims_snprintf( + char * _Nonnull str, + int len, + const char * _Nonnull format, + char * _Nonnull str1, + char * _Nonnull str2 +) { + return snprintf(str, len, format, str1, str2); +} +#endif + +#if __has_include() +vm_size_t _subprocess_vm_size(void) { + // This shim exists because vm_page_size is not marked const, and therefore looks like global mutable state to Swift. + return vm_page_size; +} +#endif + +// MARK: - Darwin (posix_spawn) +#if TARGET_OS_MAC +static int _subprocess_spawn_prefork( + pid_t * _Nonnull pid, + const char * _Nonnull exec_path, + const posix_spawn_file_actions_t _Nullable * _Nonnull file_actions, + const posix_spawnattr_t _Nullable * _Nonnull spawn_attrs, + char * _Nullable const args[_Nonnull], + char * _Nullable const env[_Nullable], + uid_t * _Nullable uid, + gid_t * _Nullable gid, + int number_of_sgroups, const gid_t * _Nullable sgroups, + int create_session +) { + // Set `POSIX_SPAWN_SETEXEC` flag since we are forking ourselves + short flags = 0; + int rc = posix_spawnattr_getflags(spawn_attrs, &flags); + if (rc != 0) { + return rc; + } + + rc = posix_spawnattr_setflags( + (posix_spawnattr_t *)spawn_attrs, flags | POSIX_SPAWN_SETEXEC + ); + if (rc != 0) { + return rc; + } + // Setup pipe to catch exec failures from child + int pipefd[2]; + if (pipe(pipefd) != 0) { + return errno; + } + // Set FD_CLOEXEC so the pipe is automatically closed when exec succeeds + flags = fcntl(pipefd[0], F_GETFD); + if (flags == -1) { + close(pipefd[0]); + close(pipefd[1]); + return errno; + } + flags |= FD_CLOEXEC; + if (fcntl(pipefd[0], F_SETFD, flags) == -1) { + close(pipefd[0]); + close(pipefd[1]); + return errno; + } + + flags = fcntl(pipefd[1], F_GETFD); + if (flags == -1) { + close(pipefd[0]); + close(pipefd[1]); + return errno; + } + flags |= FD_CLOEXEC; + if (fcntl(pipefd[1], F_SETFD, flags) == -1) { + close(pipefd[0]); + close(pipefd[1]); + return errno; + } + + // Finally, fork +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated" + pid_t childPid = fork(); +#pragma GCC diagnostic pop + if (childPid == -1) { + close(pipefd[0]); + close(pipefd[1]); + return errno; + } + + if (childPid == 0) { + // Child process + close(pipefd[0]); // Close unused read end + + // Perform setups + if (number_of_sgroups > 0 && sgroups != NULL) { + if (setgroups(number_of_sgroups, sgroups) != 0) { + int error = errno; + write(pipefd[1], &error, sizeof(error)); + close(pipefd[1]); + _exit(EXIT_FAILURE); + } + } + + if (uid != NULL) { + if (setuid(*uid) != 0) { + int error = errno; + write(pipefd[1], &error, sizeof(error)); + close(pipefd[1]); + _exit(EXIT_FAILURE); + } + } + + if (gid != NULL) { + if (setgid(*gid) != 0) { + int error = errno; + write(pipefd[1], &error, sizeof(error)); + close(pipefd[1]); + _exit(EXIT_FAILURE); + } + } + + if (create_session != 0) { + (void)setsid(); + } + + // Use posix_spawnas exec + int error = posix_spawn(pid, exec_path, file_actions, spawn_attrs, args, env); + // If we reached this point, something went wrong + write(pipefd[1], &error, sizeof(error)); + close(pipefd[1]); + _exit(EXIT_FAILURE); + } else { + // Parent process + close(pipefd[1]); // Close unused write end + // Communicate child pid back + *pid = childPid; + // Read from the pipe until pipe is closed + // Eitehr due to exec succeeds or error is written + int childError = 0; + if (read(pipefd[0], &childError, sizeof(childError)) > 0) { + // We encountered error + close(pipefd[0]); + return childError; + } else { + // Child process exec was successful + close(pipefd[0]); + return 0; + } + } +} + +int _subprocess_spawn( + pid_t * _Nonnull pid, + const char * _Nonnull exec_path, + const posix_spawn_file_actions_t _Nullable * _Nonnull file_actions, + const posix_spawnattr_t _Nullable * _Nonnull spawn_attrs, + char * _Nullable const args[_Nonnull], + char * _Nullable const env[_Nullable], + uid_t * _Nullable uid, + gid_t * _Nullable gid, + int number_of_sgroups, const gid_t * _Nullable sgroups, + int create_session +) { + int require_pre_fork = uid != NULL || + gid != NULL || + number_of_sgroups > 0 || + create_session > 0; + + if (require_pre_fork != 0) { + int rc = _subprocess_spawn_prefork( + pid, + exec_path, + file_actions, spawn_attrs, + args, env, + uid, gid, number_of_sgroups, sgroups, create_session + ); + return rc; + } + + // Spawn + return posix_spawn(pid, exec_path, file_actions, spawn_attrs, args, env); +} + +#endif // TARGET_OS_MAC + +// MARK: - Linux (fork/exec + posix_spawn fallback) +#if TARGET_OS_LINUX + +#if _POSIX_SPAWN +static int _subprocess_is_addchdir_np_available() { +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 29) + // Glibc versions prior to 2.29 don't support posix_spawn_file_actions_addchdir_np, impacting: + // - Amazon Linux 2 (EoL mid-2025) + return 0; +#elif defined(__OpenBSD__) || defined(__QNX__) + // Currently missing as of: + // - OpenBSD 7.5 (April 2024) + // - QNX 8 (December 2023) + return 0; +#elif defined(__GLIBC__) || TARGET_OS_DARWIN || defined(__FreeBSD__) || (defined(__ANDROID__) && __ANDROID_API__ >= 34) || defined(__musl__) + // Pre-standard posix_spawn_file_actions_addchdir_np version available in: + // - Solaris 11.3 (October 2015) + // - Glibc 2.29 (February 2019) + // - macOS 10.15 (October 2019) + // - musl 1.1.24 (October 2019) + // - FreeBSD 13.1 (May 2022) + // - Android 14 (October 2023) + return 1; +#else + // Standardized posix_spawn_file_actions_addchdir version (POSIX.1-2024, June 2024) available in: + // - Solaris 11.4 (August 2018) + // - NetBSD 10.0 (March 2024) + return 1; +#endif +} + +static int _subprocess_addchdir_np( + posix_spawn_file_actions_t *file_actions, + const char * __restrict path +) { +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 29) + // Glibc versions prior to 2.29 don't support posix_spawn_file_actions_addchdir_np, impacting: + // - Amazon Linux 2 (EoL mid-2025) + // noop +#elif defined(__OpenBSD__) || defined(__QNX__) + // Currently missing as of: + // - OpenBSD 7.5 (April 2024) + // - QNX 8 (December 2023) + // noop +#elif defined(__GLIBC__) || TARGET_OS_DARWIN || defined(__FreeBSD__) || (defined(__ANDROID__) && __ANDROID_API__ >= 34) || defined(__musl__) + // Pre-standard posix_spawn_file_actions_addchdir_np version available in: + // - Solaris 11.3 (October 2015) + // - Glibc 2.29 (February 2019) + // - macOS 10.15 (October 2019) + // - musl 1.1.24 (October 2019) + // - FreeBSD 13.1 (May 2022) + // - Android 14 (October 2023) + return posix_spawn_file_actions_addchdir_np(file_actions, path); +#else + // Standardized posix_spawn_file_actions_addchdir version (POSIX.1-2024, June 2024) available in: + // - Solaris 11.4 (August 2018) + // - NetBSD 10.0 (March 2024) + return posix_spawn_file_actions_addchdir(file_actions, path); +#endif +} + +static int _subprocess_posix_spawn_fallback( + pid_t * _Nonnull pid, + const char * _Nonnull exec_path, + const char * _Nullable working_directory, + const int file_descriptors[_Nonnull], + char * _Nullable const args[_Nonnull], + char * _Nullable const env[_Nullable], + gid_t * _Nullable process_group_id +) { + // Setup stdin, stdout, and stderr + posix_spawn_file_actions_t file_actions; + + int rc = posix_spawn_file_actions_init(&file_actions); + if (rc != 0) { return rc; } + if (file_descriptors[0] >= 0) { + rc = posix_spawn_file_actions_adddup2( + &file_actions, file_descriptors[0], STDIN_FILENO + ); + if (rc != 0) { return rc; } + } + if (file_descriptors[2] >= 0) { + rc = posix_spawn_file_actions_adddup2( + &file_actions, file_descriptors[2], STDOUT_FILENO + ); + if (rc != 0) { return rc; } + } + if (file_descriptors[4] >= 0) { + rc = posix_spawn_file_actions_adddup2( + &file_actions, file_descriptors[4], STDERR_FILENO + ); + if (rc != 0) { return rc; } + } + // Setup working directory + rc = _subprocess_addchdir_np(&file_actions, working_directory); + if (rc != 0) { + return rc; + } + + // Close parent side + if (file_descriptors[1] >= 0) { + rc = posix_spawn_file_actions_addclose(&file_actions, file_descriptors[1]); + if (rc != 0) { return rc; } + } + if (file_descriptors[3] >= 0) { + rc = posix_spawn_file_actions_addclose(&file_actions, file_descriptors[3]); + if (rc != 0) { return rc; } + } + if (file_descriptors[5] >= 0) { + rc = posix_spawn_file_actions_addclose(&file_actions, file_descriptors[5]); + if (rc != 0) { return rc; } + } + + // Setup spawnattr + posix_spawnattr_t spawn_attr; + rc = posix_spawnattr_init(&spawn_attr); + if (rc != 0) { return rc; } + // Masks + sigset_t no_signals; + sigset_t all_signals; + sigemptyset(&no_signals); + sigfillset(&all_signals); + rc = posix_spawnattr_setsigmask(&spawn_attr, &no_signals); + if (rc != 0) { return rc; } + rc = posix_spawnattr_setsigdefault(&spawn_attr, &all_signals); + if (rc != 0) { return rc; } + // Flags + short flags = POSIX_SPAWN_SETSIGMASK | POSIX_SPAWN_SETSIGDEF; + if (process_group_id != NULL) { + flags |= POSIX_SPAWN_SETPGROUP; + rc = posix_spawnattr_setpgroup(&spawn_attr, *process_group_id); + if (rc != 0) { return rc; } + } + rc = posix_spawnattr_setflags(&spawn_attr, flags); + + // Spawn! + rc = posix_spawn( + pid, exec_path, + &file_actions, &spawn_attr, + args, env + ); + posix_spawn_file_actions_destroy(&file_actions); + posix_spawnattr_destroy(&spawn_attr); + return rc; +} +#endif // _POSIX_SPAWN + +int _subprocess_fork_exec( + pid_t * _Nonnull pid, + const char * _Nonnull exec_path, + const char * _Nullable working_directory, + const int file_descriptors[_Nonnull], + char * _Nullable const args[_Nonnull], + char * _Nullable const env[_Nullable], + uid_t * _Nullable uid, + gid_t * _Nullable gid, + gid_t * _Nullable process_group_id, + int number_of_sgroups, const gid_t * _Nullable sgroups, + int create_session, + void (* _Nullable configurator)(void) +) { + int require_pre_fork = _subprocess_is_addchdir_np_available() == 0 || + uid != NULL || + gid != NULL || + process_group_id != NULL || + (number_of_sgroups > 0 && sgroups != NULL) || + create_session || + configurator != NULL; + +#if _POSIX_SPAWN + // If posix_spawn is available on this platform and + // we do not require prefork, use posix_spawn if possible. + // + // (Glibc's posix_spawn does not support + // `POSIX_SPAWN_SETEXEC` therefore we have to keep + // using fork/exec if `require_pre_fork` is true. + if (require_pre_fork == 0) { + return _subprocess_posix_spawn_fallback( + pid, exec_path, + working_directory, + file_descriptors, + args, env, + process_group_id + ); + } +#endif + + // Setup pipe to catch exec failures from child + int pipefd[2]; + if (pipe(pipefd) != 0) { + return errno; + } + // Set FD_CLOEXEC so the pipe is automatically closed when exec succeeds + short flags = fcntl(pipefd[0], F_GETFD); + if (flags == -1) { + close(pipefd[0]); + close(pipefd[1]); + return errno; + } + flags |= FD_CLOEXEC; + if (fcntl(pipefd[0], F_SETFD, flags) == -1) { + close(pipefd[0]); + close(pipefd[1]); + return errno; + } + + flags = fcntl(pipefd[1], F_GETFD); + if (flags == -1) { + close(pipefd[0]); + close(pipefd[1]); + return errno; + } + flags |= FD_CLOEXEC; + if (fcntl(pipefd[1], F_SETFD, flags) == -1) { + close(pipefd[0]); + close(pipefd[1]); + return errno; + } + + // Finally, fork +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated" + pid_t childPid = fork(); +#pragma GCC diagnostic pop + if (childPid == -1) { + close(pipefd[0]); + close(pipefd[1]); + return errno; + } + + if (childPid == 0) { + // Child process + close(pipefd[0]); // Close unused read end + + // Perform setups + if (working_directory != NULL) { + if (chdir(working_directory) != 0) { + int error = errno; + write(pipefd[1], &error, sizeof(error)); + close(pipefd[1]); + _exit(EXIT_FAILURE); + } + } + + + if (uid != NULL) { + if (setuid(*uid) != 0) { + int error = errno; + write(pipefd[1], &error, sizeof(error)); + close(pipefd[1]); + _exit(EXIT_FAILURE); + } + } + + if (gid != NULL) { + if (setgid(*gid) != 0) { + int error = errno; + write(pipefd[1], &error, sizeof(error)); + close(pipefd[1]); + _exit(EXIT_FAILURE); + } + } + + if (number_of_sgroups > 0 && sgroups != NULL) { + if (setgroups(number_of_sgroups, sgroups) != 0) { + int error = errno; + write(pipefd[1], &error, sizeof(error)); + close(pipefd[1]); + _exit(EXIT_FAILURE); + } + } + + if (create_session != 0) { + (void)setsid(); + } + + if (process_group_id != NULL) { + (void)setpgid(0, *process_group_id); + } + + // Bind stdin, stdout, and stderr + int rc = 0; + if (file_descriptors[0] >= 0) { + rc = dup2(file_descriptors[0], STDIN_FILENO); + if (rc < 0) { + int error = errno; + write(pipefd[1], &error, sizeof(error)); + close(pipefd[1]); + _exit(EXIT_FAILURE); + } + } + if (file_descriptors[2] >= 0) { + rc = dup2(file_descriptors[2], STDOUT_FILENO); + if (rc < 0) { + int error = errno; + write(pipefd[1], &error, sizeof(error)); + close(pipefd[1]); + _exit(EXIT_FAILURE); + } + } + if (file_descriptors[4] >= 0) { + rc = dup2(file_descriptors[4], STDERR_FILENO); + if (rc < 0) { + int error = errno; + write(pipefd[1], &error, sizeof(error)); + close(pipefd[1]); + _exit(EXIT_FAILURE); + } + } + // Close parent side + if (file_descriptors[1] >= 0) { + rc = close(file_descriptors[1]); + } + if (file_descriptors[3] >= 0) { + rc = close(file_descriptors[3]); + } + if (file_descriptors[4] >= 0) { + rc = close(file_descriptors[5]); + } + if (rc != 0) { + int error = errno; + write(pipefd[1], &error, sizeof(error)); + close(pipefd[1]); + _exit(EXIT_FAILURE); + } + // Run custom configuratior + if (configurator != NULL) { + configurator(); + } + // Finally, exec + execve(exec_path, args, env); + // If we reached this point, something went wrong + int error = errno; + write(pipefd[1], &error, sizeof(error)); + close(pipefd[1]); + _exit(EXIT_FAILURE); + } else { + // Parent process + close(pipefd[1]); // Close unused write end + // Communicate child pid back + *pid = childPid; + // Read from the pipe until pipe is closed + // Eitehr due to exec succeeds or error is written + int childError = 0; + if (read(pipefd[0], &childError, sizeof(childError)) > 0) { + // We encountered error + close(pipefd[0]); + return childError; + } else { + // Child process exec was successful + close(pipefd[0]); + return 0; + } + } +} + +#endif // TARGET_OS_LINUX + +#endif // !TARGET_OS_WINDOWS + +#pragma mark - Environment Locking + +#if __has_include() +#import +void _subprocess_lock_environ(void) { + environ_lock_np(); +} + +void _subprocess_unlock_environ(void) { + environ_unlock_np(); +} +#else +void _subprocess_lock_environ(void) { /* noop */ } +void _subprocess_unlock_environ(void) { /* noop */ } +#endif + +char ** _subprocess_get_environ(void) { +#if __has_include() + return *_NSGetEnviron(); +#elif defined(_WIN32) +#include + return _environ; +#elif TARGET_OS_WASI + return __wasilibc_get_environ(); +#elif __has_include() + return environ; +#endif +} + + +#if TARGET_OS_WINDOWS + +typedef struct { + DWORD pid; + HWND mainWindow; +} CallbackContext; + +static BOOL CALLBACK enumWindowsCallback( + HWND hwnd, + LPARAM lParam +) { + CallbackContext *context = (CallbackContext *)lParam; + DWORD pid; + GetWindowThreadProcessId(hwnd, &pid); + if (pid == context->pid) { + context->mainWindow = hwnd; + return FALSE; // Stop enumeration + } + return TRUE; // Continue enumeration +} + +BOOL _subprocess_windows_send_vm_close( + DWORD pid +) { + // First attempt to find the Window associate + // with this process + CallbackContext context = {0}; + context.pid = pid; + EnumWindows(enumWindowsCallback, (LPARAM)&context); + + if (context.mainWindow != NULL) { + if (SendMessage(context.mainWindow, WM_CLOSE, 0, 0)) { + return TRUE; + } + } + + return FALSE; +} + +#endif + From bcdb03aeff0c0dfa55223f691f95d149f3c1269f Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Wed, 18 Jun 2025 14:40:42 +0900 Subject: [PATCH 03/37] fix unacceptablelanguageignore for new paths --- .unacceptablelanguageignore | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.unacceptablelanguageignore b/.unacceptablelanguageignore index 0310d1c3..92a1d726 100644 --- a/.unacceptablelanguageignore +++ b/.unacceptablelanguageignore @@ -2,16 +2,5 @@ Sources/SwiftJavaBootstrapJavaTool/SwiftJavaBootstrapJavaTool.swift Sources/_Subprocess/Platforms/Subprocess+Darwin.swift Sources/_Subprocess/Platforms/Subprocess+Linux.swift Sources/_Subprocess/Platforms/Subprocess+Unix.swift -Sources/_Subprocess/Platforms/Subprocess+Unix.swift -Sources/_Subprocess/Platforms/Subprocess+Unix.swift -Sources/_Subprocess/Platforms/Subprocess+Unix.swift -Sources/_Subprocess/Platforms/Subprocess+Unix.swift -Sources/_Subprocess/Platforms/Subprocess+Unix.swift -Sources/_Subprocess/Subprocess+Teardown.swift -Sources/_Subprocess/Subprocess+Teardown.swift -Sources/_Subprocess/Subprocess+Teardown.swift -Sources/_Subprocess/Subprocess+Teardown.swift -Sources/_Subprocess/Subprocess+Teardown.swift -Sources/_Subprocess/Subprocess+Teardown.swift -Sources/_Subprocess/Subprocess+Teardown.swift +Sources/_Subprocess/Teardown.swift Sources/_Subprocess/Subprocess.swift \ No newline at end of file From 62e0190e5aedc8c68b5af8d39deafdcba6480aa7 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Wed, 18 Jun 2025 14:41:09 +0900 Subject: [PATCH 04/37] ignore .swift-version --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6712182f..b3dbac45 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.swift-version + .DS_Store .build .idea From b9fbf6e32f8646191ec4bd696a8b326349c0d3b5 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Wed, 18 Jun 2025 14:53:41 +0900 Subject: [PATCH 05/37] license ignore: the vendored subprocess has a slightly differnetly phrased apache 2 license header; ignore it for now, it is compatible but we'll want to move to the package instead anyway shortly --- .licenseignore | 3 ++- Package.swift | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.licenseignore b/.licenseignore index df3de377..003cac25 100644 --- a/.licenseignore +++ b/.licenseignore @@ -47,5 +47,6 @@ Plugins/**/_PluginsShared Plugins/**/0_PLEASE_SYMLINK* Plugins/PluginsShared/JavaKitConfigurationShared Samples/JavaDependencySampleApp/gradle -Sources/_Subprocess/_nio_locks.swift +Sources/_Subprocess/** +Sources/_SubprocessCShims/** Samples/gradle diff --git a/Package.swift b/Package.swift index 80879ca3..c7a30957 100644 --- a/Package.swift +++ b/Package.swift @@ -11,6 +11,7 @@ import Foundation // Library/Java/JavaVirtualMachines/openjdk-21.jdk/Contents/Home. func findJavaHome() -> String { if let home = ProcessInfo.processInfo.environment["JAVA_HOME"] { + print("JAVA_HOME = \(home)") return home } From 5377f9fd971933a93f0022be8a6037ed8dd9ab56 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Wed, 18 Jun 2025 15:02:50 +0900 Subject: [PATCH 06/37] start using wrap-java subcommand --- Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift | 6 ++++-- Sources/SwiftJavaTool/SwiftJava.swift | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift b/Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift index 3c989db1..36bcdb95 100644 --- a/Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift +++ b/Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift @@ -87,8 +87,10 @@ struct SwiftJavaBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { searchForConfigFiles(in: dependency) } - var arguments: [String] = [] - arguments += argumentsSwiftModuleDeprecated(sourceModule: sourceModule) + var arguments: [String] = [ + "wrap-java" + ] + arguments += argumentsSwiftModule(sourceModule: sourceModule) arguments += argumentsOutputDirectory(context: context) arguments += argumentsDependedOnConfigs(dependentConfigFiles) arguments.append(configFile.path(percentEncoded: false)) diff --git a/Sources/SwiftJavaTool/SwiftJava.swift b/Sources/SwiftJavaTool/SwiftJava.swift index b54a7b58..d1bef9f3 100644 --- a/Sources/SwiftJavaTool/SwiftJava.swift +++ b/Sources/SwiftJavaTool/SwiftJava.swift @@ -35,6 +35,7 @@ struct SwiftJava: SwiftJavaBaseAsyncParsableCommand, HasCommonJVMOptions { // FI subcommands: [ ConfigureCommand.self, ResolveCommand.self, + WrapJavaCommand.self, ]) // FIXME: this must be removed when we move things out from the SwiftJava main file! From 76d33b8e515829efd7c7603827c4f4d025b58112 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Wed, 18 Jun 2025 21:26:14 +0900 Subject: [PATCH 07/37] New commands: wrap-java and resolve work from swift-java CLI tool --- Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift | 17 +++------ .../Configuration.swift | 6 +-- .../Commands/ConfigureCommand.swift | 4 +- .../Commands/SwiftJava+JExtract.swift | 26 +++++++++++++ .../Commands/WrapJavaCommand.swift | 37 +++++++++++++------ Sources/SwiftJavaTool/CommonOptions.swift | 20 +++++++--- Sources/SwiftJavaTool/SwiftJava.swift | 26 ++++++------- .../SwiftJavaBaseAsyncParsableCommand.swift | 12 +----- 8 files changed, 91 insertions(+), 57 deletions(-) diff --git a/Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift b/Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift index 36bcdb95..a057978d 100644 --- a/Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift +++ b/Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift @@ -87,13 +87,10 @@ struct SwiftJavaBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { searchForConfigFiles(in: dependency) } - var arguments: [String] = [ - "wrap-java" - ] + var arguments: [String] = [] arguments += argumentsSwiftModule(sourceModule: sourceModule) arguments += argumentsOutputDirectory(context: context) arguments += argumentsDependedOnConfigs(dependentConfigFiles) - arguments.append(configFile.path(percentEncoded: false)) let classes = config.classes ?? [:] print("[swift-java-plugin] Classes to wrap (\(classes.count)): \(classes.map(\.key))") @@ -158,12 +155,7 @@ struct SwiftJavaBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { executable: executable, arguments: ["resolve"] + argumentsOutputDirectory(context: context, generated: false) - + argumentsSwiftModule(sourceModule: sourceModule) - // + [ - // // explicitly provide config path - // configFile.path(percentEncoded: false) - // ] - , + + argumentsSwiftModule(sourceModule: sourceModule), environment: [:], inputFiles: [configFile], outputFiles: fetchDependenciesOutputFiles @@ -174,13 +166,16 @@ struct SwiftJavaBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { } if !outputSwiftFiles.isEmpty { + arguments += [ configFile.path(percentEncoded: false) ] + let displayName = "Wrapping \(classes.count) Java classes in Swift target '\(sourceModule.name)'" log("Prepared: \(displayName)") commands += [ .buildCommand( displayName: displayName, executable: executable, - arguments: arguments, + arguments: ["wrap-java"] + + arguments, inputFiles: compiledClassFiles + fetchDependenciesOutputFiles + [ configFile ], outputFiles: outputSwiftFiles ) diff --git a/Sources/JavaKitConfigurationShared/Configuration.swift b/Sources/JavaKitConfigurationShared/Configuration.swift index 1953167e..f38a487b 100644 --- a/Sources/JavaKitConfigurationShared/Configuration.swift +++ b/Sources/JavaKitConfigurationShared/Configuration.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Copyright (c) 2024-2025 Apple Inc. and the Swift.org project authors // Licensed under Apache License v2.0 // // See LICENSE.txt for license information @@ -156,7 +156,7 @@ public func findSwiftJavaClasspaths(in basePath: String = FileManager.default.cu let baseURL = URL(fileURLWithPath: basePath) var classpathEntries: [String] = [] - print("[debug][swift-java] Searching for *.swift-java.classpath files in: \(baseURL)") + print("[debug][swift-java] Searching for *.swift-java.classpath files in: \(baseURL.absoluteString)") guard let enumerator = fileManager.enumerator(at: baseURL, includingPropertiesForKeys: []) else { print("[warning][swift-java] Failed to get enumerator for \(baseURL)") return [] @@ -164,7 +164,7 @@ public func findSwiftJavaClasspaths(in basePath: String = FileManager.default.cu for case let fileURL as URL in enumerator { if fileURL.lastPathComponent.hasSuffix(".swift-java.classpath") { - print("[debug][swift-java] Constructing classpath with entries from: \(fileURL.relativePath)") + print("[debug][swift-java] Constructing classpath with entries from: \(fileURL.path)") if let contents = try? String(contentsOf: fileURL) { let entries = contents.split(separator: ":").map(String.init) for entry in entries { diff --git a/Sources/SwiftJavaTool/Commands/ConfigureCommand.swift b/Sources/SwiftJavaTool/Commands/ConfigureCommand.swift index 54859d3d..2355ea81 100644 --- a/Sources/SwiftJavaTool/Commands/ConfigureCommand.swift +++ b/Sources/SwiftJavaTool/Commands/ConfigureCommand.swift @@ -66,8 +66,8 @@ extension SwiftJava { extension SwiftJava.ConfigureCommand { mutating func runSwiftJavaCommand(config: inout Configuration) async throws { - let classpathEntries = - self.configureCommandJVMClasspath(effectiveSwiftModuleURL: self.effectiveSwiftModuleURL, config: config) + let classpathEntries = self.configureCommandJVMClasspath( + searchDirs: [self.effectiveSwiftModuleURL], config: config) let jvm = try self.makeJVM(classpathEntries: classpathEntries) diff --git a/Sources/SwiftJavaTool/Commands/SwiftJava+JExtract.swift b/Sources/SwiftJavaTool/Commands/SwiftJava+JExtract.swift index 79c96d3e..157d7960 100644 --- a/Sources/SwiftJavaTool/Commands/SwiftJava+JExtract.swift +++ b/Sources/SwiftJavaTool/Commands/SwiftJava+JExtract.swift @@ -31,6 +31,32 @@ import JavaKitConfigurationShared /// ``` extension SwiftJava { + struct JExtractCommand: SwiftJavaBaseAsyncParsableCommand, HasCommonOptions { + static let configuration = CommandConfiguration( + commandName: "jextract", // TODO: wrap-swift? + abstract: "Resolve dependencies and write the resulting swift-java.classpath file") + + @OptionGroup var commonOptions: SwiftJava.CommonOptions + + @Option(help: "The mode of generation to use for the output files. Used with jextract mode.") + var mode: GenerationMode = .ffm + + @Option(help: "The name of the Swift module into which the resulting Swift types will be generated.") + var swiftModule: String + + var effectiveSwiftModule: String { + swiftModule + } + } +} + +extension SwiftJava.JExtractCommand { + func runSwiftJavaCommand(config: inout Configuration) async throws { + fatalError("not implemented yet") + } +} + +extension SwiftJava { mutating func jextractSwift( config: Configuration ) throws { diff --git a/Sources/SwiftJavaTool/Commands/WrapJavaCommand.swift b/Sources/SwiftJavaTool/Commands/WrapJavaCommand.swift index 7fdedcbf..73329555 100644 --- a/Sources/SwiftJavaTool/Commands/WrapJavaCommand.swift +++ b/Sources/SwiftJavaTool/Commands/WrapJavaCommand.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Copyright (c) 2024-2025 Apple Inc. and the Swift.org project authors // Licensed under Apache License v2.0 // // See LICENSE.txt for license information @@ -52,6 +52,9 @@ extension SwiftJava { @Option(help: "The names of Java classes whose declared native methods will be implemented in Swift.") var swiftNativeImplementation: [String] = [] + @Option(help: "Cache directory for intermediate results and other outputs between runs") + var cacheDirectory: String? + @Argument(help: "Path to .jar file whose Java classes should be wrapped using Swift bindings") var input: String } @@ -60,34 +63,46 @@ extension SwiftJava { extension SwiftJava.WrapJavaCommand { mutating func runSwiftJavaCommand(config: inout Configuration) async throws { + // Perform any config overrides by command options: if let javaPackage { config.javaPackage = javaPackage } - // Load all of the dependent configurations and associate them with Swift - // modules. - let dependentConfigs = try loadDependentConfigs() - // Configure our own classpath based on config - var classpathEntries = - self.configureCommandJVMClasspath(effectiveSwiftModuleURL: self.effectiveSwiftModuleURL, config: config) + + // Get base classpath configuration for this target and configuration + var classpathSearchDirs = [self.effectiveSwiftModuleURL] + if let cacheDir = self.cacheDirectory { + print("[trace][swift-java] Cache directory: \(cacheDir)") + classpathSearchDirs += [URL(fileURLWithPath: cacheDir)] + } else { + print("[trace][swift-java] Cache directory: none") + } + print("[trace][swift-java] INPUT: \(input)") + + var classpathEntries = self.configureCommandJVMClasspath( + searchDirs: classpathSearchDirs, config: config) + + // Load all of the dependent configurations and associate them with Swift modules. + let dependentConfigs = try self.loadDependentConfigs() + print("[debug][swift-java] Dependent configs: \(dependentConfigs.count)") // Include classpath entries which libs we depend on require... for (fromModule, config) in dependentConfigs { + print("[trace][swift-java] Add dependent config (\(fromModule)) classpath elements: \(config.classpathEntries.count)") // TODO: may need to resolve the dependent configs rather than just get their configs // TODO: We should cache the resolved classpaths as well so we don't do it many times - config.classpath.map { entry in + for entry in config.classpathEntries { print("[trace][swift-java] Add dependent config (\(fromModule)) classpath element: \(entry)") classpathEntries.append(entry) } } - let completeClasspath = classpathEntries.joined(separator: ":") let jvm = try self.makeJVM(classpathEntries: classpathEntries) try self.generateWrappers( config: config, - classpath: completeClasspath, + // classpathEntries: classpathEntries, dependentConfigs: dependentConfigs, environment: jvm.environment() ) @@ -118,7 +133,7 @@ extension SwiftJava.WrapJavaCommand { extension SwiftJava.WrapJavaCommand { mutating func generateWrappers( config: Configuration, - classpath: String, + // classpathEntries: [String], dependentConfigs: [(String, Configuration)], environment: JNIEnvironment ) throws { diff --git a/Sources/SwiftJavaTool/CommonOptions.swift b/Sources/SwiftJavaTool/CommonOptions.swift index 54ed8131..e3ddfa44 100644 --- a/Sources/SwiftJavaTool/CommonOptions.swift +++ b/Sources/SwiftJavaTool/CommonOptions.swift @@ -73,22 +73,30 @@ extension HasCommonJVMOptions { } extension HasCommonJVMOptions { - func configureCommandJVMClasspath(effectiveSwiftModuleURL: Foundation.URL, config: Configuration) -> [String] { + + /// Collect classpath information from various sources such as CLASSPATH, `-cp` option and + /// swift-java.classpath files as configured. + /// Parameters: + /// - searchDirs: search directories where we can find swift.java.classpath files to include in the configuration + func configureCommandJVMClasspath(searchDirs: [Foundation.URL], config: Configuration) -> [String] { // Form a class path from all of our input sources: // * Command-line option --classpath let classpathOptionEntries: [String] = self.classpathEntries let classpathFromEnv = ProcessInfo.processInfo.environment["CLASSPATH"]?.split(separator: ":").map(String.init) ?? [] + print("[debug][swift-java] Base classpath from CLASSPATH environment: \(classpathFromEnv)") let classpathFromConfig: [String] = config.classpath?.split(separator: ":").map(String.init) ?? [] print("[debug][swift-java] Base classpath from config: \(classpathFromConfig)") var classpathEntries: [String] = classpathFromConfig - let classPathFilesSearchDirectory = effectiveSwiftModuleURL.absoluteString - print("[debug][swift-java] Search *.swift-java.classpath in: \(classPathFilesSearchDirectory)") - let swiftJavaCachedModuleClasspath = findSwiftJavaClasspaths(in: classPathFilesSearchDirectory) + for searchDir in searchDirs { + let classPathFilesSearchDirectory = searchDir.path + print("[debug][swift-java] Search *.swift-java.classpath in: \(classPathFilesSearchDirectory)") + let foundSwiftJavaClasspath = findSwiftJavaClasspaths(in: classPathFilesSearchDirectory) - print("[debug][swift-java] Classpath from *.swift-java.classpath files: \(swiftJavaCachedModuleClasspath)") - classpathEntries += swiftJavaCachedModuleClasspath + print("[debug][swift-java] Classpath from *.swift-java.classpath files: \(foundSwiftJavaClasspath)") + classpathEntries += foundSwiftJavaClasspath + } if !classpathOptionEntries.isEmpty { print("[debug][swift-java] Classpath from options: \(classpathOptionEntries)") diff --git a/Sources/SwiftJavaTool/SwiftJava.swift b/Sources/SwiftJavaTool/SwiftJava.swift index d1bef9f3..062f3541 100644 --- a/Sources/SwiftJavaTool/SwiftJava.swift +++ b/Sources/SwiftJavaTool/SwiftJava.swift @@ -77,19 +77,19 @@ struct SwiftJava: SwiftJavaBaseAsyncParsableCommand, HasCommonJVMOptions { // FI @Option(help: "The mode of generation to use for the output files. Used with jextract mode.") var mode: GenerationMode = .ffm - @Option(name: .shortAndLong, help: "Directory where to write cached values (e.g. swift-java.classpath files)") - var cacheDirectory: String? = nil +// @Option(name: .shortAndLong, help: "Directory where to write cached values (e.g. swift-java.classpath files)") +// var cacheDirectory: String? = nil @OptionGroup var commonOptions: SwiftJava.CommonOptions @OptionGroup var commonJVMOptions: SwiftJava.CommonJVMOptions - var effectiveCacheDirectory: String? { - if let cacheDirectory { - return cacheDirectory - } else { - return nil - } - } +// var effectiveCacheDirectory: String? { +// if let cacheDirectory { +// return cacheDirectory +// } else { +// return nil +// } +// } // @Argument( // help: "The input file, which is either a Java2Swift configuration file or (if '-jar' was specified) a Jar file." @@ -193,10 +193,10 @@ struct SwiftJava: SwiftJavaBaseAsyncParsableCommand, HasCommonJVMOptions { // FI var classpathEntries: [String] = classpathFromConfig - let swiftJavaCachedModuleClasspath = findSwiftJavaClasspaths( - in: self.effectiveCacheDirectory ?? FileManager.default.currentDirectoryPath) - print("[debug][swift-java] Classpath from *.swift-java.classpath files: \(swiftJavaCachedModuleClasspath)") - classpathEntries += swiftJavaCachedModuleClasspath +// let swiftJavaCachedModuleClasspath = findSwiftJavaClasspaths( +// in: self.effectiveCacheDirectory ?? FileManager.default.currentDirectoryPath) +// print("[debug][swift-java] Classpath from *.swift-java.classpath files: \(swiftJavaCachedModuleClasspath)") +// classpathEntries += swiftJavaCachedModuleClasspath if !classpathOptionEntries.isEmpty { print("[debug][swift-java] Classpath from options: \(classpathOptionEntries)") diff --git a/Sources/SwiftJavaTool/SwiftJavaBaseAsyncParsableCommand.swift b/Sources/SwiftJavaTool/SwiftJavaBaseAsyncParsableCommand.swift index 92b3f84c..a0e736f7 100644 --- a/Sources/SwiftJavaTool/SwiftJavaBaseAsyncParsableCommand.swift +++ b/Sources/SwiftJavaTool/SwiftJavaBaseAsyncParsableCommand.swift @@ -46,7 +46,7 @@ extension SwiftJavaBaseAsyncParsableCommand { extension SwiftJavaBaseAsyncParsableCommand { public mutating func run() async { print("[info][swift-java] Run \(Self.self): \(CommandLine.arguments.joined(separator: " "))") - print("[info][swift-java] Current work directory: \(URL(fileURLWithPath: "."))") + print("[info][swift-java] Current work directory: \(URL(fileURLWithPath: ".").path)") do { var config = try readInitialConfiguration(command: self) @@ -64,16 +64,6 @@ extension SwiftJavaBaseAsyncParsableCommand { } extension SwiftJavaBaseAsyncParsableCommand { -// mutating func writeContents( -// _ contents: String, -// to filename: String, description: String) throws { -// try writeContents( -// contents, -// outputDirectoryOverride: self.actualOutputDirectory, -// to: filename, -// description: description) -// } - mutating func writeContents( _ contents: String, outputDirectory: Foundation.URL?, From ec05d34a06f1bc4e8bb7f19c68f5f4a095c10614 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Wed, 18 Jun 2025 21:28:47 +0900 Subject: [PATCH 08/37] cleanups, remove dead code, commented code etc --- .../Commands/SwiftJava+JExtract.swift | 2 + .../Commands/WrapJavaCommand.swift | 2 - Sources/SwiftJavaTool/SwiftJava.swift | 184 +----------------- .../SwiftJavaBaseAsyncParsableCommand.swift | 10 +- 4 files changed, 7 insertions(+), 191 deletions(-) diff --git a/Sources/SwiftJavaTool/Commands/SwiftJava+JExtract.swift b/Sources/SwiftJavaTool/Commands/SwiftJava+JExtract.swift index 157d7960..0275bb3b 100644 --- a/Sources/SwiftJavaTool/Commands/SwiftJava+JExtract.swift +++ b/Sources/SwiftJavaTool/Commands/SwiftJava+JExtract.swift @@ -64,3 +64,5 @@ extension SwiftJava { } } + +extension GenerationMode: ExpressibleByArgument {} diff --git a/Sources/SwiftJavaTool/Commands/WrapJavaCommand.swift b/Sources/SwiftJavaTool/Commands/WrapJavaCommand.swift index 73329555..49b4815d 100644 --- a/Sources/SwiftJavaTool/Commands/WrapJavaCommand.swift +++ b/Sources/SwiftJavaTool/Commands/WrapJavaCommand.swift @@ -68,8 +68,6 @@ extension SwiftJava.WrapJavaCommand { config.javaPackage = javaPackage } - - // Get base classpath configuration for this target and configuration var classpathSearchDirs = [self.effectiveSwiftModuleURL] if let cacheDir = self.cacheDirectory { diff --git a/Sources/SwiftJavaTool/SwiftJava.swift b/Sources/SwiftJavaTool/SwiftJava.swift index 062f3541..2f91c7ef 100644 --- a/Sources/SwiftJavaTool/SwiftJava.swift +++ b/Sources/SwiftJavaTool/SwiftJava.swift @@ -36,34 +36,16 @@ struct SwiftJava: SwiftJavaBaseAsyncParsableCommand, HasCommonJVMOptions { // FI ConfigureCommand.self, ResolveCommand.self, WrapJavaCommand.self, + JExtractCommand.self ]) - // FIXME: this must be removed when we move things out from the SwiftJava main file! - @Option( - name: .long, - help: "The name of the Swift module into which the resulting Swift types will be generated.") - var swiftModuleDeprecated: String? - var effectiveSwiftModule: String { - if let module = swiftModuleDeprecated { - module - } else if let module = self.outputSwift?.split(separator: "/").last { + if let module = self.outputSwift?.split(separator: "/").last { String(module) } else { "UnknownSwiftModule" } } -// -// @Option( -// help: -// "A Java2Swift configuration file for a given Swift module name on which this module depends, e.g., JavaKitJar=Sources/JavaKitJar/Java2Swift.config. There should be one of these options for each Swift module that this module depends on (transitively) that contains wrapped Java sources." -// ) -// var dependsOn: [String] = [] -// -// @Option( -// help: "The names of Java classes whose declared native methods will be implemented in Swift." -// ) -// var swiftNativeImplementation: [String] = [] @Option(help: "The directory where generated Swift files should be written. Generally used with jextract mode.") var outputSwift: String? = nil @@ -71,42 +53,14 @@ struct SwiftJava: SwiftJavaBaseAsyncParsableCommand, HasCommonJVMOptions { // FI @Option(help: "The directory where generated Java files should be written. Generally used with jextract mode.") var outputJava: String? = nil -// @Option(help: "The Java package the generated Java code should be emitted into.") -// var javaPackage: String? = nil - - @Option(help: "The mode of generation to use for the output files. Used with jextract mode.") - var mode: GenerationMode = .ffm - -// @Option(name: .shortAndLong, help: "Directory where to write cached values (e.g. swift-java.classpath files)") -// var cacheDirectory: String? = nil - @OptionGroup var commonOptions: SwiftJava.CommonOptions @OptionGroup var commonJVMOptions: SwiftJava.CommonJVMOptions -// var effectiveCacheDirectory: String? { -// if let cacheDirectory { -// return cacheDirectory -// } else { -// return nil -// } -// } - // @Argument( // help: "The input file, which is either a Java2Swift configuration file or (if '-jar' was specified) a Jar file." // ) var input: String? // FIXME: top level command cannot have input argument like this - // FIXME: this is subcommands - /// Describes what kind of generation action is being performed by swift-java. - enum ToolMode { -// /// Generate Swift wrappers for Java classes based on the given -// /// configuration. -// case classWrappers - - /// Extract Java bindings from provided Swift sources. - case jextract // TODO: carry jextract specific config here? - } - mutating func runSwiftJavaCommand(config: inout Configuration) async throws { guard CommandLine.arguments.count >= 2 else { // there's no "default" command, print USAGE when no arguments/parameters are passed. @@ -114,13 +68,6 @@ struct SwiftJava: SwiftJavaBaseAsyncParsableCommand, HasCommonJVMOptions { // FI return } -// if let javaPackage { -// config.javaPackage = javaPackage -// } - - // Determine the mode in which we'll execute. - let toolMode: ToolMode = .jextract - // TODO: some options are exclusive to each other so we should detect that if let inputSwift = commonOptions.inputSwift { guard let inputSwift = commonOptions.inputSwift else { @@ -139,124 +86,11 @@ struct SwiftJava: SwiftJavaBaseAsyncParsableCommand, HasCommonJVMOptions { // FI config.inputSwiftDirectory = inputSwift config.outputSwiftDirectory = outputSwift config.outputJavaDirectory = outputJava - -// toolMode = .jextract -// } else if fetch { -// guard let input else { -// fatalError("Mode 'fetch' requires path\n\(Self.helpMessage())") -// } -// config = try JavaTranslator.readConfiguration(from: URL(fileURLWithPath: input)) -// guard let dependencies = config.dependencies else { -// print("[swift-java] Running in 'fetch dependencies' mode but dependencies list was empty!") -// print("[swift-java] Nothing to do: done.") -// return -// } -// toolMode = .fetchDependencies - } -// else { -// guard let input else { -// fatalError("Mode -jar requires path\n\(Self.helpMessage())") -// } -// config = try JavaTranslator.readConfiguration(from: URL(fileURLWithPath: input)) -// toolMode = .classWrappers -// } - - print("[debug][swift-java] Running swift-java in mode: " + "\(toolMode.prettyName)".bold) - - let swiftModule: String = - self.effectiveSwiftModule -// ?? -// self.effectiveSwiftModule.split(separator: "/").dropLast().last.map(String.init) ?? "__UnknownModule" - -// // Load all of the dependent configurations and associate them with Swift -// // modules. -// let dependentConfigs = try dependsOn.map { dependentConfig in -// guard let equalLoc = dependentConfig.firstIndex(of: "=") else { -// throw JavaToSwiftError.badConfigOption(dependentConfig) -// } -// -// let afterEqual = dependentConfig.index(after: equalLoc) -// let swiftModuleName = String(dependentConfig[.. (javaClassName: String, swiftName: String) { @@ -295,13 +129,3 @@ extension JavaToSwiftError: CustomStringConvertible { } } -extension SwiftJava.ToolMode { - var prettyName: String { - switch self { -// case .classWrappers: "Wrap Java classes" - case .jextract: "JExtract Swift for Java" - } - } -} - -extension GenerationMode: ExpressibleByArgument {} diff --git a/Sources/SwiftJavaTool/SwiftJavaBaseAsyncParsableCommand.swift b/Sources/SwiftJavaTool/SwiftJavaBaseAsyncParsableCommand.swift index a0e736f7..20b177af 100644 --- a/Sources/SwiftJavaTool/SwiftJavaBaseAsyncParsableCommand.swift +++ b/Sources/SwiftJavaTool/SwiftJavaBaseAsyncParsableCommand.swift @@ -149,15 +149,7 @@ extension SwiftJavaBaseAsyncParsableCommand { // For generated Swift sources, put them into a "generated" subdirectory. // The configuration file goes at the top level. - let outputDir: Foundation.URL - // if jar { - // precondition(self.input != nil, "-jar mode requires path to jar to be specified as input path") - outputDir = baseDir - // } else { - // outputDir = baseDir - // .appendingPathComponent("generated", isDirectory: true) - // } - + let outputDir: Foundation.URL = baseDir return outputDir } From 6ab9ee6d4d5124ca8e9f325f29085db3979ac78f Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Wed, 18 Jun 2025 22:00:34 +0900 Subject: [PATCH 09/37] add explicit dependency on swift-system --- Package.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Package.swift b/Package.swift index c7a30957..0ad8f295 100644 --- a/Package.swift +++ b/Package.swift @@ -385,6 +385,7 @@ let package = Package( .product(name: "SwiftSyntax", package: "swift-syntax"), .product(name: "SwiftSyntaxBuilder", package: "swift-syntax"), .product(name: "ArgumentParser", package: "swift-argument-parser"), + .product(name: "SystemPackage", package: "swift-system"), "JavaKit", "JavaKitJar", "JavaKitNetwork", From c2f66d5b1d0e5aceb45c659a0c5c9d6e79a2b02a Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Wed, 18 Jun 2025 22:24:19 +0900 Subject: [PATCH 10/37] fixing subprocess build on linux --- Package.swift | 13 +++++++-- .../Commands/ResolveCommand.swift | 29 ++++--------------- 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/Package.swift b/Package.swift index 0ad8f295..f64befea 100644 --- a/Package.swift +++ b/Package.swift @@ -369,7 +369,7 @@ let package = Package( "JavaKitShared", "JavaKitConfigurationShared", // .product(name: "Subprocess", package: "swift-subprocess") - "_Subprocess", // using process spawning + "_Subprocess", ], swiftSettings: [ .swiftLanguageMode(.v5), @@ -394,11 +394,14 @@ let package = Package( "JavaKitShared", "JavaKitConfigurationShared", ], - swiftSettings: [ .swiftLanguageMode(.v5), .unsafeFlags(["-I\(javaIncludePath)", "-I\(javaPlatformIncludePath)"]), .enableUpcomingFeature("BareSlashRegexLiterals"), + .define( + "SYSTEM_PACKAGE_DARWIN", + .when(platforms: [.macOS, .macCatalyst, .iOS, .watchOS, .tvOS, .visionOS])), + .define("SYSTEM_PACKAGE"), ] ), @@ -508,7 +511,11 @@ let package = Package( .product(name: "SystemPackage", package: "swift-system"), ], swiftSettings: [ - .swiftLanguageMode(.v5) + .swiftLanguageMode(.v5), + .define( + "SYSTEM_PACKAGE_DARWIN", + .when(platforms: [.macOS, .macCatalyst, .iOS, .watchOS, .tvOS, .visionOS])), + .define("SYSTEM_PACKAGE"), ] ), ] diff --git a/Sources/SwiftJavaTool/Commands/ResolveCommand.swift b/Sources/SwiftJavaTool/Commands/ResolveCommand.swift index 66f0eec4..4f022d71 100644 --- a/Sources/SwiftJavaTool/Commands/ResolveCommand.swift +++ b/Sources/SwiftJavaTool/Commands/ResolveCommand.swift @@ -22,7 +22,11 @@ import SwiftJavaLib import JavaKitConfigurationShared import JavaKitShared import _Subprocess +#if canImport(System) import System +#else +@preconcurrency import SystemPackage +#endif typealias Configuration = JavaKitConfigurationShared.Configuration @@ -58,31 +62,10 @@ extension SwiftJava.ResolveCommand { var printRuntimeClasspathTaskName: String { "printRuntimeClasspath" } mutating func runSwiftJavaCommand(config: inout Configuration) async throws { - // Form a class path from all of our input sources: - let classpathOptionEntries: [String] = self.classpathEntries - let classpathFromEnv = self.classpathEnvEntries - let classpathFromConfig: [String] = config.classpath?.split(separator: ":").map(String.init) ?? [] - print("[debug][swift-java] Base classpath from config: \(classpathFromConfig)") - - var classpathEntries: [String] = classpathFromConfig - - let classPathFilesSearchDirectory = self.effectiveSwiftModuleURL.absoluteString - print("[debug][swift-java] Search *.swift-java.classpath in: \(classPathFilesSearchDirectory)") - let swiftJavaCachedModuleClasspath = findSwiftJavaClasspaths(in: classPathFilesSearchDirectory) - - print("[debug][swift-java] Classpath from *.swift-java.classpath files: \(swiftJavaCachedModuleClasspath)") - classpathEntries += swiftJavaCachedModuleClasspath - - if logLevel >= .debug { - let classpathString = classpathEntries.joined(separator: ":") - print("[debug][swift-java] Initialize JVM with classpath: \(classpathString)") - } - let jvm = try JavaVirtualMachine.shared(classpath: classpathEntries) - var dependenciesToResolve: [JavaDependencyDescriptor] = [] if let input, let inputDependencies = parseDependencyDescriptor(input) { dependenciesToResolve.append(inputDependencies) - } + } if let dependencies = config.dependencies { dependenciesToResolve += dependencies } @@ -148,7 +131,7 @@ extension SwiftJava.ResolveCommand { "--rerun-tasks", "\(printRuntimeClasspathTaskName)", ], - workingDirectory: FilePath(resolverDir.path), + workingDirectory: Optional(FilePath(resolverDir.path)), // TODO: we could move to stream processing the outputs output: .string(limit: Int.max, encoding: UTF8.self), // Don't limit output, we know it will be reasonable size error: .string(limit: Int.max, encoding: UTF8.self) // Don't limit output, we know it will be reasonable size From 3f15bc1a6a0306e70b0bc2527a32270b9459a8ad Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Thu, 19 Jun 2025 18:35:08 +0900 Subject: [PATCH 11/37] introduce swift-java jextract subcommand, completing our migration --- .../JExtractSwiftCommandPlugin.swift | 2 +- .../JExtractSwiftPlugin.swift | 4 +- .../SwiftAndJavaJarSampleLib/Package.swift | 2 +- Samples/SwiftAndJavaJarSampleLib/build.gradle | 10 +- ...ift2JavaGenerator+SwiftThunkPrinting.swift | 4 +- .../JNI/JNISwift2JavaGenerator.swift | 2 +- Sources/JExtractSwiftLib/Swift2Java.swift | 3 +- .../Configuration.swift | 2 +- .../GenerationMode.swift | 2 +- .../Commands/JExtractCommand.swift | 94 +++++++++++++++++++ .../Commands/SwiftJava+JExtract.swift | 68 -------------- .../Commands/WrapJavaCommand.swift | 8 -- Sources/SwiftJavaTool/SwiftJava.swift | 49 +--------- .../Asserts/TextAssertions.swift | 2 +- 14 files changed, 114 insertions(+), 138 deletions(-) create mode 100644 Sources/SwiftJavaTool/Commands/JExtractCommand.swift delete mode 100644 Sources/SwiftJavaTool/Commands/SwiftJava+JExtract.swift diff --git a/Plugins/JExtractSwiftCommandPlugin/JExtractSwiftCommandPlugin.swift b/Plugins/JExtractSwiftCommandPlugin/JExtractSwiftCommandPlugin.swift index f2327dad..c68110fb 100644 --- a/Plugins/JExtractSwiftCommandPlugin/JExtractSwiftCommandPlugin.swift +++ b/Plugins/JExtractSwiftCommandPlugin/JExtractSwiftCommandPlugin.swift @@ -71,6 +71,7 @@ final class JExtractSwiftCommandPlugin: SwiftJavaPluginProtocol, BuildToolPlugin let configuration = try readConfiguration(sourceDir: "\(sourceDir)") var arguments: [String] = [ + /*subcommand=*/"jextract", "--input-swift", sourceDir, "--swift-module", sourceModule.name, "--output-java", context.outputJavaDirectory.path(percentEncoded: false), @@ -80,7 +81,6 @@ final class JExtractSwiftCommandPlugin: SwiftJavaPluginProtocol, BuildToolPlugin // as it depends on the contents of the input files. Therefore we have to implement this as a prebuild plugin. // We'll have to make up some caching inside the tool so we don't re-parse files which have not changed etc. ] - // arguments.append(sourceDir) // TODO: we could do this shape maybe? to have the dirs last? if let package = configuration?.javaPackage, !package.isEmpty { arguments += ["--java-package", package] } diff --git a/Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift b/Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift index 4b52df26..a9347be7 100644 --- a/Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift +++ b/Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift @@ -54,8 +54,9 @@ struct JExtractSwiftBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { let outputSwiftDirectory = context.outputSwiftDirectory var arguments: [String] = [ - "--input-swift", sourceDir, + /*subcommand=*/"jextract", "--swift-module", sourceModule.name, + "--input-swift", sourceDir, "--output-java", outputJavaDirectory.path(percentEncoded: false), "--output-swift", outputSwiftDirectory.path(percentEncoded: false), // TODO: "--build-cache-directory", ... @@ -63,7 +64,6 @@ struct JExtractSwiftBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { // as it depends on the contents of the input files. Therefore we have to implement this as a prebuild plugin. // We'll have to make up some caching inside the tool so we don't re-parse files which have not changed etc. ] - // arguments.append(sourceDir) if !javaPackage.isEmpty { arguments.append(contentsOf: ["--java-package", javaPackage]) } diff --git a/Samples/SwiftAndJavaJarSampleLib/Package.swift b/Samples/SwiftAndJavaJarSampleLib/Package.swift index 5f132239..c4c604ee 100644 --- a/Samples/SwiftAndJavaJarSampleLib/Package.swift +++ b/Samples/SwiftAndJavaJarSampleLib/Package.swift @@ -43,7 +43,7 @@ let javaIncludePath = "\(javaHome)/include" let package = Package( name: "SwiftAndJavaJarSampleLib", platforms: [ - .macOS(.v10_15) + .macOS(.v15) ], products: [ .library( diff --git a/Samples/SwiftAndJavaJarSampleLib/build.gradle b/Samples/SwiftAndJavaJarSampleLib/build.gradle index 2125011d..f4fb63f9 100644 --- a/Samples/SwiftAndJavaJarSampleLib/build.gradle +++ b/Samples/SwiftAndJavaJarSampleLib/build.gradle @@ -51,8 +51,8 @@ dependencies { // This is for development, when we edit the Swift swift-java project, the outputs of the generated sources may change. // Thus, we also need to watch and re-build the top level project. -def compileSwiftJExtractPlugin = tasks.register("compileSwiftJExtractPlugin", Exec) { - description = "Rebuild the swift-java root project" +def compileSwiftJExtractPlugin = tasks.register("compileSwiftJava", Exec) { + description = "Rebuild the swift-java root project (" + swiftBuildConfiguration() + ")" inputs.file(new File(rootDir, "Package.swift")) inputs.dir(new File(rootDir, "Sources")) @@ -62,9 +62,7 @@ def compileSwiftJExtractPlugin = tasks.register("compileSwiftJExtractPlugin", Ex commandLine "swift" args("build", "-c", swiftBuildConfiguration(), - "--product", "SwiftKitSwift", - "--product", "JExtractSwiftPlugin", - "--product", "JExtractSwiftCommandPlugin") + "--product", "swift-java") } def jextract = tasks.register("jextract", Exec) { @@ -96,7 +94,7 @@ def jextract = tasks.register("jextract", Exec) { workingDir = layout.projectDirectory commandLine "swift" - args("package", "jextract", "-v", "--log-level", "info") // TODO: pass log level from Gradle build + args("run", "swift-java", "jextract", "-v", "--log-level", "info") // TODO: pass log level from Gradle build } diff --git a/Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator+SwiftThunkPrinting.swift b/Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator+SwiftThunkPrinting.swift index 02d715a0..79bcfd04 100644 --- a/Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator+SwiftThunkPrinting.swift +++ b/Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator+SwiftThunkPrinting.swift @@ -34,7 +34,7 @@ extension FFMSwift2JavaGenerator { javaPackagePath: nil, filename: moduleFilename) { - print("[swift-java] Generated: \(moduleFilenameBase.bold).swift (at \(outputFile)") + print("[swift-java] Generated: \(moduleFilenameBase.bold).swift (at \(outputFile))") } } catch { log.warning("Failed to write to Swift thunks: \(moduleFilename)") @@ -54,7 +54,7 @@ extension FFMSwift2JavaGenerator { javaPackagePath: nil, filename: filename) { - print("[swift-java] Generated: \(fileNameBase.bold).swift (at \(outputFile)") + print("[swift-java] Generated: \(fileNameBase.bold).swift (at \(outputFile))") } } catch { log.warning("Failed to write to Swift thunks: \(filename)") diff --git a/Sources/JExtractSwiftLib/JNI/JNISwift2JavaGenerator.swift b/Sources/JExtractSwiftLib/JNI/JNISwift2JavaGenerator.swift index b15688dc..8ec4b437 100644 --- a/Sources/JExtractSwiftLib/JNI/JNISwift2JavaGenerator.swift +++ b/Sources/JExtractSwiftLib/JNI/JNISwift2JavaGenerator.swift @@ -87,7 +87,7 @@ extension JNISwift2JavaGenerator { javaPackagePath: javaPackagePath, filename: moduleFilename ) { - print("[swift-java] Generated: \(moduleFilenameBase.bold).swift (at \(outputFile)") + print("[swift-java] Generated: \(moduleFilenameBase.bold).swift (at \(outputFile))") } } catch { logger.warning("Failed to write to Swift thunks: \(moduleFilename)") diff --git a/Sources/JExtractSwiftLib/Swift2Java.swift b/Sources/JExtractSwiftLib/Swift2Java.swift index 9b1b06bc..ac746c1d 100644 --- a/Sources/JExtractSwiftLib/Swift2Java.swift +++ b/Sources/JExtractSwiftLib/Swift2Java.swift @@ -39,12 +39,11 @@ public struct SwiftToJava { translator.log.warning("Configured java package is '', consider specifying concrete package for generated sources.") } - print("===== CONFIG ==== \(config)") - guard let inputSwift = config.inputSwiftDirectory else { fatalError("Missing '--swift-input' directory!") } + translator.log.info("Input swift = \(inputSwift)") let inputPaths = inputSwift.split(separator: ",").map { URL(string: String($0))! } translator.log.info("Input paths = \(inputPaths)") diff --git a/Sources/JavaKitConfigurationShared/Configuration.swift b/Sources/JavaKitConfigurationShared/Configuration.swift index f38a487b..33eb6757 100644 --- a/Sources/JavaKitConfigurationShared/Configuration.swift +++ b/Sources/JavaKitConfigurationShared/Configuration.swift @@ -38,7 +38,7 @@ public struct Configuration: Codable { public var outputJavaDirectory: String? - public var mode: GenerationMode? + public var mode: JExtractGenerationMode? // ==== java 2 swift --------------------------------------------------------- diff --git a/Sources/JavaKitConfigurationShared/GenerationMode.swift b/Sources/JavaKitConfigurationShared/GenerationMode.swift index ea30a436..b4a96476 100644 --- a/Sources/JavaKitConfigurationShared/GenerationMode.swift +++ b/Sources/JavaKitConfigurationShared/GenerationMode.swift @@ -12,7 +12,7 @@ // //===----------------------------------------------------------------------===// -public enum GenerationMode: String, Codable { +public enum JExtractGenerationMode: String, Codable { /// Foreign Value and Memory API case ffm diff --git a/Sources/SwiftJavaTool/Commands/JExtractCommand.swift b/Sources/SwiftJavaTool/Commands/JExtractCommand.swift new file mode 100644 index 00000000..da845eb3 --- /dev/null +++ b/Sources/SwiftJavaTool/Commands/JExtractCommand.swift @@ -0,0 +1,94 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation +import ArgumentParser +import SwiftJavaLib +import JavaKit +import JavaKitJar +import SwiftJavaLib +import JExtractSwiftLib +import JavaKitConfigurationShared + +/// Extract Java bindings from Swift sources or interface files. +/// +/// Example usage: +/// ``` +/// > swift-java jextract +// --input-swift Sources/SwiftyBusiness \ +/// --output-swift .build/.../outputs/SwiftyBusiness \ +/// --output-Java .build/.../outputs/Java +/// ``` +extension SwiftJava { + + struct JExtractCommand: SwiftJavaBaseAsyncParsableCommand, HasCommonOptions { + static let configuration = CommandConfiguration( + commandName: "jextract", // TODO: wrap-swift? + abstract: "Wrap Swift functions and types with Java bindings, making them available to be called from Java") + + @OptionGroup var commonOptions: SwiftJava.CommonOptions + + @Option(help: "The mode of generation to use for the output files. Used with jextract mode.") + var mode: JExtractGenerationMode = .ffm + + @Option(help: "The name of the Swift module into which the resulting Swift types will be generated.") + var swiftModule: String + + var effectiveSwiftModule: String { + swiftModule + } + + @Option(help: "The Java package the generated Java code should be emitted into.") + var javaPackage: String? = nil + + @Option(help: "The directory where generated Swift files should be written. Generally used with jextract mode.") + var outputSwift: String + + @Option(help: "The directory where generated Java files should be written. Generally used with jextract mode.") + var outputJava: String + } +} + +extension SwiftJava.JExtractCommand { + func runSwiftJavaCommand(config: inout Configuration) async throws { + if let javaPackage { + config.javaPackage = javaPackage + } + config.swiftModule = self.effectiveSwiftModule + config.outputJavaDirectory = outputJava + config.outputSwiftDirectory = outputSwift + + if let inputSwift = commonOptions.inputSwift { + config.inputSwiftDirectory = inputSwift + } else if let swiftModule = config.swiftModule { + // This is a "good guess" technically a target can be somewhere else, but then you can use --input-swift + config.inputSwiftDirectory = "\(FileManager.default.currentDirectoryPath)/Sources/\(swiftModule)" + } + + print("[debug][swift-java] Running swift-java in mode: " + "\(self.mode)".bold) + + try jextractSwift(config: config) + } +} + +extension SwiftJava.JExtractCommand { + func jextractSwift( + config: Configuration + ) throws { + try SwiftToJava(config: config).run() + } + +} + +extension JExtractGenerationMode: ExpressibleByArgument {} diff --git a/Sources/SwiftJavaTool/Commands/SwiftJava+JExtract.swift b/Sources/SwiftJavaTool/Commands/SwiftJava+JExtract.swift deleted file mode 100644 index 0275bb3b..00000000 --- a/Sources/SwiftJavaTool/Commands/SwiftJava+JExtract.swift +++ /dev/null @@ -1,68 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift.org project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -import Foundation -import ArgumentParser -import SwiftJavaLib -import JavaKit -import JavaKitJar -import SwiftJavaLib -import JExtractSwiftLib -import JavaKitConfigurationShared - -/// Extract Java bindings from Swift sources or interface files. -/// -/// Example usage: -/// ``` -/// > swift-java --input-swift Sources/SwiftyBusiness \ -/// --output-swift .build/.../outputs/SwiftyBusiness \ -/// --output-Java .build/.../outputs/Java -/// ``` -extension SwiftJava { - - struct JExtractCommand: SwiftJavaBaseAsyncParsableCommand, HasCommonOptions { - static let configuration = CommandConfiguration( - commandName: "jextract", // TODO: wrap-swift? - abstract: "Resolve dependencies and write the resulting swift-java.classpath file") - - @OptionGroup var commonOptions: SwiftJava.CommonOptions - - @Option(help: "The mode of generation to use for the output files. Used with jextract mode.") - var mode: GenerationMode = .ffm - - @Option(help: "The name of the Swift module into which the resulting Swift types will be generated.") - var swiftModule: String - - var effectiveSwiftModule: String { - swiftModule - } - } -} - -extension SwiftJava.JExtractCommand { - func runSwiftJavaCommand(config: inout Configuration) async throws { - fatalError("not implemented yet") - } -} - -extension SwiftJava { - mutating func jextractSwift( - config: Configuration - ) throws { - try SwiftToJava(config: config).run() - } - -} - -extension GenerationMode: ExpressibleByArgument {} diff --git a/Sources/SwiftJavaTool/Commands/WrapJavaCommand.swift b/Sources/SwiftJavaTool/Commands/WrapJavaCommand.swift index 49b4815d..349144ab 100644 --- a/Sources/SwiftJavaTool/Commands/WrapJavaCommand.swift +++ b/Sources/SwiftJavaTool/Commands/WrapJavaCommand.swift @@ -46,9 +46,6 @@ extension SwiftJava { ) var dependsOn: [String] = [] - @Option(help: "The Java package the generated Java code should be emitted into.") - var javaPackage: String? = nil - @Option(help: "The names of Java classes whose declared native methods will be implemented in Swift.") var swiftNativeImplementation: [String] = [] @@ -63,11 +60,6 @@ extension SwiftJava { extension SwiftJava.WrapJavaCommand { mutating func runSwiftJavaCommand(config: inout Configuration) async throws { - // Perform any config overrides by command options: - if let javaPackage { - config.javaPackage = javaPackage - } - // Get base classpath configuration for this target and configuration var classpathSearchDirs = [self.effectiveSwiftModuleURL] if let cacheDir = self.cacheDirectory { diff --git a/Sources/SwiftJavaTool/SwiftJava.swift b/Sources/SwiftJavaTool/SwiftJava.swift index 2f91c7ef..8a325f90 100644 --- a/Sources/SwiftJavaTool/SwiftJava.swift +++ b/Sources/SwiftJavaTool/SwiftJava.swift @@ -27,7 +27,7 @@ import JavaKitShared /// Command-line utility to drive the export of Java classes into Swift types. @main -struct SwiftJava: SwiftJavaBaseAsyncParsableCommand, HasCommonJVMOptions { // FIXME: this is just a normal async command, no parsing happening here +struct SwiftJava: AsyncParsableCommand { // FIXME: this is just a normal async command, no parsing happening here static var _commandName: String { "swift-java" } static let configuration = CommandConfiguration( @@ -40,57 +40,18 @@ struct SwiftJava: SwiftJavaBaseAsyncParsableCommand, HasCommonJVMOptions { // FI ]) var effectiveSwiftModule: String { - if let module = self.outputSwift?.split(separator: "/").last { - String(module) - } else { - "UnknownSwiftModule" - } + fatalError("SHOULD NOT BE USED") } - @Option(help: "The directory where generated Swift files should be written. Generally used with jextract mode.") - var outputSwift: String? = nil - - @Option(help: "The directory where generated Java files should be written. Generally used with jextract mode.") - var outputJava: String? = nil - - @OptionGroup var commonOptions: SwiftJava.CommonOptions - @OptionGroup var commonJVMOptions: SwiftJava.CommonJVMOptions - -// @Argument( -// help: "The input file, which is either a Java2Swift configuration file or (if '-jar' was specified) a Jar file." -// ) - var input: String? // FIXME: top level command cannot have input argument like this - - mutating func runSwiftJavaCommand(config: inout Configuration) async throws { + mutating func run() async throws { guard CommandLine.arguments.count >= 2 else { // there's no "default" command, print USAGE when no arguments/parameters are passed. print("error: Must specify mode subcommand (e.g. configure, resolve, jextract, ...).\n\n\(Self.helpMessage())") return } - // TODO: some options are exclusive to each other so we should detect that - if let inputSwift = commonOptions.inputSwift { - guard let inputSwift = commonOptions.inputSwift else { - print("[swift-java] --input-swift enabled 'jextract' mode, however no --output-swift directory was provided!\n\(Self.helpMessage())") - return - } - guard let outputSwift else { - print("[swift-java] --output-swift enabled 'jextract' mode, however no --output-swift directory was provided!\n\(Self.helpMessage())") - return - } - guard let outputJava else { - print("[swift-java] --output-java enabled 'jextract' mode, however no --output-java directory was provided!\n\(Self.helpMessage())") - return - } - config.swiftModule = self.effectiveSwiftModule - config.inputSwiftDirectory = inputSwift - config.outputSwiftDirectory = outputSwift - config.outputJavaDirectory = outputJava - } - - print("[debug][swift-java] Running swift-java in mode: " + "jextract (legacy, to be moved into subcommand)") - - try jextractSwift(config: config) + print("error: Must specify subcommand to execute.\n\n\(Self.helpMessage())") + return } private func names(from javaClassNameOpt: String) -> (javaClassName: String, swiftName: String) { diff --git a/Tests/JExtractSwiftTests/Asserts/TextAssertions.swift b/Tests/JExtractSwiftTests/Asserts/TextAssertions.swift index b3a09a22..61866525 100644 --- a/Tests/JExtractSwiftTests/Asserts/TextAssertions.swift +++ b/Tests/JExtractSwiftTests/Asserts/TextAssertions.swift @@ -25,7 +25,7 @@ enum RenderKind { func assertOutput( dump: Bool = false, input: String, - _ mode: GenerationMode, + _ mode: JExtractGenerationMode, _ renderKind: RenderKind, swiftModuleName: String = "SwiftModule", detectChunkByInitialLines: Int = 4, From 799e666b42a27c95857c91f365306f53e94aa21e Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Thu, 19 Jun 2025 19:21:12 +0900 Subject: [PATCH 12/37] debug output on linux --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 8fa774bd..2df3c596 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -36,7 +36,7 @@ jobs: - name: Gradle :SwiftKit:build run: ./gradlew build -x test - name: Gradle :SwiftKit:check - run: ./gradlew :SwiftKit:check --info + run: ./gradlew :SwiftKit:check --debug - name: Gradle compile JMH benchmarks run: ./gradlew compileJmh --info From 0813c5fed5deaf980d9451751b4129cecd7f37ee Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Thu, 19 Jun 2025 22:18:41 +0900 Subject: [PATCH 13/37] refactor jextract plugin to be a BUILD plugin; unblocks swift 6.2 --- Package.swift | 32 +++---- .../JExtractSwiftCommandPlugin.swift | 2 +- .../JExtractSwiftPlugin.swift | 49 +++++++++- .../JavaCompilerPlugin.swift | 21 +++-- Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift | 4 +- .../Sources/MySwiftLibrary/MySwiftClass.swift | 69 ++++++++++++++ .../MySwiftLibrary/MySwiftLibrary.swift | 92 +++++++++---------- Samples/SwiftAndJavaJarSampleLib/build.gradle | 48 +++++----- Samples/SwiftKitSampleApp/build.gradle | 2 +- ...ift2JavaGenerator+SwiftThunkPrinting.swift | 1 + .../Configuration.swift | 2 +- 11 files changed, 220 insertions(+), 102 deletions(-) create mode 100644 Samples/SwiftAndJavaJarSampleLib/Sources/MySwiftLibrary/MySwiftClass.swift diff --git a/Package.swift b/Package.swift index f64befea..f6aba9de 100644 --- a/Package.swift +++ b/Package.swift @@ -175,12 +175,12 @@ let package = Package( "JExtractSwiftPlugin" ] ), - .plugin( - name: "JExtractSwiftCommandPlugin", - targets: [ - "JExtractSwiftCommandPlugin" - ] - ), +// .plugin( +// name: "JExtractSwiftCommandPlugin", +// targets: [ +// "JExtractSwiftCommandPlugin" +// ] +// ), // ==== Examples @@ -429,16 +429,16 @@ let package = Package( "SwiftJavaTool" ] ), - .plugin( - name: "JExtractSwiftCommandPlugin", - capability: .command( - intent: .custom(verb: "jextract", description: "Extract Java accessors from Swift module"), - permissions: [ - ]), - dependencies: [ - "SwiftJavaTool" - ] - ), +// .plugin( +// name: "JExtractSwiftCommandPlugin", +// capability: .command( +// intent: .custom(verb: "jextract", description: "Extract Java accessors from Swift module"), +// permissions: [ +// ]), +// dependencies: [ +// "SwiftJavaTool" +// ] +// ), .testTarget( name: "JavaKitTests", diff --git a/Plugins/JExtractSwiftCommandPlugin/JExtractSwiftCommandPlugin.swift b/Plugins/JExtractSwiftCommandPlugin/JExtractSwiftCommandPlugin.swift index c68110fb..d91d4108 100644 --- a/Plugins/JExtractSwiftCommandPlugin/JExtractSwiftCommandPlugin.swift +++ b/Plugins/JExtractSwiftCommandPlugin/JExtractSwiftCommandPlugin.swift @@ -72,8 +72,8 @@ final class JExtractSwiftCommandPlugin: SwiftJavaPluginProtocol, BuildToolPlugin var arguments: [String] = [ /*subcommand=*/"jextract", - "--input-swift", sourceDir, "--swift-module", sourceModule.name, + "--input-swift", sourceDir, "--output-java", context.outputJavaDirectory.path(percentEncoded: false), "--output-swift", context.outputSwiftDirectory.path(percentEncoded: false), // TODO: "--build-cache-directory", ... diff --git a/Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift b/Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift index a9347be7..304f80fb 100644 --- a/Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift +++ b/Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift @@ -40,6 +40,10 @@ struct JExtractSwiftBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { } let sourceDir = target.directory.string + + // The name of the configuration file JavaKit.config from the target for + // which we are generating Swift wrappers for Java classes. + let configFile = URL(filePath: sourceDir).appending(path: "swift-java.config") let configuration = try readConfiguration(sourceDir: "\(sourceDir)") guard let javaPackage = configuration?.javaPackage else { @@ -65,18 +69,53 @@ struct JExtractSwiftBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { // We'll have to make up some caching inside the tool so we don't re-parse files which have not changed etc. ] if !javaPackage.isEmpty { - arguments.append(contentsOf: ["--java-package", javaPackage]) + arguments += ["--java-package", javaPackage] + } + + let swiftFiles = sourceModule.sourceFiles.map { $0.url }.filter { + $0.pathExtension == "swift" + } + + let outputSwiftFiles: [URL] = swiftFiles.compactMap { sourceFileURL in + guard sourceFileURL.isFileURL else { + return nil as URL? + } + + let sourceFilePath = sourceFileURL.path + guard sourceFilePath.starts(with: sourceDir) else { + fatalError("Could not get relative path for source file \(sourceFilePath)") + } + var outputURL = outputSwiftDirectory + .appending(path: String(sourceFilePath.dropFirst(sourceDir.count).dropLast(sourceFileURL.lastPathComponent.count + 1))) + + let inputFileName = sourceFileURL.deletingPathExtension().lastPathComponent + print(" inputFileName = \(inputFileName)") + let isModuleFile = inputFileName.contains("Library") // FIXME: this is a hack + print(" isModuleFile = \(isModuleFile)") + + return if isModuleFile { + outputURL.appending(path: "\(inputFileName)Module+SwiftJava.swift") + } else { + outputURL.appending(path: "\(inputFileName)+SwiftJava.swift") + } } return [ - .prebuildCommand( + .buildCommand( displayName: "Generate Java wrappers for Swift types", executable: toolURL, arguments: arguments, - // inputFiles: [ configFile ] + swiftFiles, - // outputFiles: outputJavaFiles - outputFilesDirectory: outputSwiftDirectory + inputFiles: [ configFile ] + swiftFiles, + outputFiles: outputSwiftFiles ) +// .prebuildCommand( +// displayName: "Generate Java wrappers for Swift types", +// executable: toolURL, +// arguments: arguments, +// // inputFiles: [ configFile ] + swiftFiles, +// // outputFiles: outputJavaFiles +// outputFilesDirectory: outputSwiftDirectory +// ) ] } } diff --git a/Plugins/JavaCompilerPlugin/JavaCompilerPlugin.swift b/Plugins/JavaCompilerPlugin/JavaCompilerPlugin.swift index b93fe403..55955180 100644 --- a/Plugins/JavaCompilerPlugin/JavaCompilerPlugin.swift +++ b/Plugins/JavaCompilerPlugin/JavaCompilerPlugin.swift @@ -34,7 +34,7 @@ struct JavaCompilerBuildToolPlugin: BuildToolPlugin { // The name of the configuration file JavaKit.config from the target for // which we are generating Swift wrappers for Java classes. - let configFile = URL(filePath: sourceDir).appending(path: "Java2Swift.config") + let configFile = URL(filePath: sourceDir).appending(path: "swift-java.config") let config: Configuration? if let configData = try? Data(contentsOf: configFile) { @@ -45,18 +45,21 @@ struct JavaCompilerBuildToolPlugin: BuildToolPlugin { // The class files themselves will be generated into the build directory // for this target. - let classFiles = javaFiles.map { sourceFileURL in + let classFiles = javaFiles.compactMap { sourceFileURL in + guard sourceFileURL.isFileURL else { + return nil as URL? + } + let sourceFilePath = sourceFileURL.path guard sourceFilePath.starts(with: sourceDir) else { fatalError("Could not get relative path for source file \(sourceFilePath)") } - return URL( - filePath: context.pluginWorkDirectoryURL.path - ).appending(path: "Java") - .appending(path: String(sourceFilePath.dropFirst(sourceDir.count))) - .deletingPathExtension() - .appendingPathExtension("class") + return URL(filePath: context.pluginWorkDirectoryURL.path) + .appending(path: "Java") + .appending(path: String(sourceFilePath.dropFirst(sourceDir.count))) + .deletingPathExtension() + .appendingPathExtension("class") } let javaHome = URL(filePath: findJavaHome()) @@ -73,7 +76,7 @@ struct JavaCompilerBuildToolPlugin: BuildToolPlugin { "-parameters", // keep parameter names, which allows us to emit them in generated Swift decls ] + (config?.compilerVersionArgs ?? []), inputFiles: javaFiles, - outputFiles: classFiles + outputFiles: classFiles // FIXME: this is not quite enough, javac may generate more files for closures etc, which we don't know about unless we compile first ) ] } diff --git a/Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift b/Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift index a057978d..cbbbe425 100644 --- a/Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift +++ b/Plugins/SwiftJavaPlugin/SwiftJavaPlugin.swift @@ -44,7 +44,7 @@ struct SwiftJavaBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { log("Config was: \(config)") var javaDependencies = config.dependencies ?? [] - /// Find the manifest files from other Java2Swift executions in any targets + /// Find the manifest files from other swift-java executions in any targets /// this target depends on. var dependentConfigFiles: [(String, URL)] = [] func searchForConfigFiles(in target: any Target) { @@ -95,7 +95,7 @@ struct SwiftJavaBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { let classes = config.classes ?? [:] print("[swift-java-plugin] Classes to wrap (\(classes.count)): \(classes.map(\.key))") - /// Determine the set of Swift files that will be emitted by the Java2Swift tool. + /// Determine the set of Swift files that will be emitted by the swift-java tool. // TODO: this is not precise and won't work with more advanced Java files, e.g. lambdas etc. let outputDirectoryGenerated = self.outputDirectory(context: context, generated: true) let outputSwiftFiles = classes.map { (javaClassName, swiftName) in diff --git a/Samples/SwiftAndJavaJarSampleLib/Sources/MySwiftLibrary/MySwiftClass.swift b/Samples/SwiftAndJavaJarSampleLib/Sources/MySwiftLibrary/MySwiftClass.swift new file mode 100644 index 00000000..c842715c --- /dev/null +++ b/Samples/SwiftAndJavaJarSampleLib/Sources/MySwiftLibrary/MySwiftClass.swift @@ -0,0 +1,69 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +// This is a "plain Swift" file containing various types of declarations, +// that is exported to Java by using the `jextract-swift` tool. +// +// No annotations are necessary on the Swift side to perform the export. + +#if os(Linux) +import Glibc +#else +import Darwin.C +#endif + +public class MySwiftClass { + + public var len: Int + public var cap: Int + + public init(len: Int, cap: Int) { + self.len = len + self.cap = cap + + p("\(MySwiftClass.self).len = \(self.len)") + p("\(MySwiftClass.self).cap = \(self.cap)") + let addr = unsafeBitCast(self, to: UInt64.self) + p("initializer done, self = 0x\(String(addr, radix: 16, uppercase: true))") + } + + deinit { + let addr = unsafeBitCast(self, to: UInt64.self) + p("Deinit, self = 0x\(String(addr, radix: 16, uppercase: true))") + } + + public var counter: Int32 = 0 + + public func voidMethod() { + p("") + } + + public func takeIntMethod(i: Int) { + p("i:\(i)") + } + + public func echoIntMethod(i: Int) -> Int { + p("i:\(i)") + return i + } + + public func makeIntMethod() -> Int { + p("make int -> 12") + return 12 + } + + public func makeRandomIntMethod() -> Int { + return Int.random(in: 1..<256) + } +} \ No newline at end of file diff --git a/Samples/SwiftAndJavaJarSampleLib/Sources/MySwiftLibrary/MySwiftLibrary.swift b/Samples/SwiftAndJavaJarSampleLib/Sources/MySwiftLibrary/MySwiftLibrary.swift index 84e4618f..39688f08 100644 --- a/Samples/SwiftAndJavaJarSampleLib/Sources/MySwiftLibrary/MySwiftLibrary.swift +++ b/Samples/SwiftAndJavaJarSampleLib/Sources/MySwiftLibrary/MySwiftLibrary.swift @@ -38,55 +38,55 @@ public func globalTakeIntInt(i: Int, j: Int) { public func globalCallMeRunnable(run: () -> ()) { run() } - -public class MySwiftClass { - - public var len: Int - public var cap: Int - - public init(len: Int, cap: Int) { - self.len = len - self.cap = cap - - p("\(MySwiftClass.self).len = \(self.len)") - p("\(MySwiftClass.self).cap = \(self.cap)") - let addr = unsafeBitCast(self, to: UInt64.self) - p("initializer done, self = 0x\(String(addr, radix: 16, uppercase: true))") - } - - deinit { - let addr = unsafeBitCast(self, to: UInt64.self) - p("Deinit, self = 0x\(String(addr, radix: 16, uppercase: true))") - } - - public var counter: Int32 = 0 - - public func voidMethod() { - p("") - } - - public func takeIntMethod(i: Int) { - p("i:\(i)") - } - - public func echoIntMethod(i: Int) -> Int { - p("i:\(i)") - return i - } - - public func makeIntMethod() -> Int { - p("make int -> 12") - return 12 - } - - public func makeRandomIntMethod() -> Int { - return Int.random(in: 1..<256) - } -} +// +//public class MySwiftClass { +// +// public var len: Int +// public var cap: Int +// +// public init(len: Int, cap: Int) { +// self.len = len +// self.cap = cap +// +// p("\(MySwiftClass.self).len = \(self.len)") +// p("\(MySwiftClass.self).cap = \(self.cap)") +// let addr = unsafeBitCast(self, to: UInt64.self) +// p("initializer done, self = 0x\(String(addr, radix: 16, uppercase: true))") +// } +// +// deinit { +// let addr = unsafeBitCast(self, to: UInt64.self) +// p("Deinit, self = 0x\(String(addr, radix: 16, uppercase: true))") +// } +// +// public var counter: Int32 = 0 +// +// public func voidMethod() { +// p("") +// } +// +// public func takeIntMethod(i: Int) { +// p("i:\(i)") +// } +// +// public func echoIntMethod(i: Int) -> Int { +// p("i:\(i)") +// return i +// } +// +// public func makeIntMethod() -> Int { +// p("make int -> 12") +// return 12 +// } +// +// public func makeRandomIntMethod() -> Int { +// return Int.random(in: 1..<256) +// } +//} // ==== Internal helpers -private func p(_ msg: String, file: String = #fileID, line: UInt = #line, function: String = #function) { +func p(_ msg: String, file: String = #fileID, line: UInt = #line, function: String = #function) { print("[swift][\(file):\(line)](\(function)) \(msg)") fflush(stdout) } diff --git a/Samples/SwiftAndJavaJarSampleLib/build.gradle b/Samples/SwiftAndJavaJarSampleLib/build.gradle index f4fb63f9..2ccc3e87 100644 --- a/Samples/SwiftAndJavaJarSampleLib/build.gradle +++ b/Samples/SwiftAndJavaJarSampleLib/build.gradle @@ -49,25 +49,25 @@ dependencies { testImplementation("org.junit.jupiter:junit-jupiter") } -// This is for development, when we edit the Swift swift-java project, the outputs of the generated sources may change. -// Thus, we also need to watch and re-build the top level project. -def compileSwiftJExtractPlugin = tasks.register("compileSwiftJava", Exec) { - description = "Rebuild the swift-java root project (" + swiftBuildConfiguration() + ")" - - inputs.file(new File(rootDir, "Package.swift")) - inputs.dir(new File(rootDir, "Sources")) - outputs.dir(new File(rootDir, ".build")) - - workingDir = rootDir - commandLine "swift" - args("build", - "-c", swiftBuildConfiguration(), - "--product", "swift-java") -} +//// This is for development, when we edit the Swift swift-java project, the outputs of the generated sources may change. +//// Thus, we also need to watch and re-build the top level project. +//def compileSwiftJExtractPlugin = tasks.register("compileSwiftJava", Exec) { +// description = "Rebuild the swift-java root project (" + swiftBuildConfiguration() + ")" +// +// inputs.file(new File(rootDir, "Package.swift")) +// inputs.dir(new File(rootDir, "Sources")) +// outputs.dir(new File(rootDir, ".build")) +// +// workingDir = rootDir +// commandLine "swift" +// args("build", +// "-c", swiftBuildConfiguration(), +// "--product", "swift-java") +//} def jextract = tasks.register("jextract", Exec) { - description = "Builds swift sources, including swift-java source generation" - dependsOn compileSwiftJExtractPlugin + description = "Generate Java wrappers for swift target" + // dependsOn compileSwiftJExtractPlugin // only because we depend on "live developing" the plugin while using this project to test it inputs.file(new File(rootDir, "Package.swift")) @@ -94,7 +94,16 @@ def jextract = tasks.register("jextract", Exec) { workingDir = layout.projectDirectory commandLine "swift" - args("run", "swift-java", "jextract", "-v", "--log-level", "info") // TODO: pass log level from Gradle build + args("run", + "swift-java", "jextract", + "--swift-module", "MySwiftLibrary", + "--java-module", "MyJavaLibrary", + // "--java-package", "com.example.myjava", + "--java-output", "${layout.buildDirectory.dir(".build/plugins/outputs/${layout.projectDirectory.asFile.getName().toLowerCase()}/JExtractSwiftPlugin/src/generated/java")}", + "--swift-output", "${layout.buildDirectory.dir(".build/plugins/outputs/${layout.projectDirectory.asFile.getName().toLowerCase()}/JExtractSwiftPlugin/Sources")}", +// "-v", +// "--log-level", "info" // TODO: pass log level from Gradle build + ) } @@ -124,7 +133,6 @@ tasks.named('test', Test) { // ==== Jar publishing List swiftProductDylibPaths() { - def process = ['swift', 'package', 'describe', '--type', 'json'].execute() process.waitFor() @@ -141,8 +149,6 @@ List swiftProductDylibPaths() { target.product_memberships }.flatten() - - def productDylibPaths = products.collect { logger.info("[swift-java] Include Swift product: '${it}' in product resource paths.") "${layout.projectDirectory}/.build/${swiftBuildConfiguration()}/lib${it}.dylib" diff --git a/Samples/SwiftKitSampleApp/build.gradle b/Samples/SwiftKitSampleApp/build.gradle index 18a55f44..bcd8af96 100644 --- a/Samples/SwiftKitSampleApp/build.gradle +++ b/Samples/SwiftKitSampleApp/build.gradle @@ -81,7 +81,7 @@ def jextract = tasks.register("jextract", Exec) { workingDir = layout.projectDirectory commandLine "swift" - args("package", "jextract", "-v", "--log-level", "debug") // TODO: pass log level from Gradle build + args("run", "swift-java", "jextract", "-v", "--log-level", "info") // TODO: pass log level from Gradle build } // Add the java-swift generated Java sources diff --git a/Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator+SwiftThunkPrinting.swift b/Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator+SwiftThunkPrinting.swift index 79bcfd04..c1ad19b6 100644 --- a/Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator+SwiftThunkPrinting.swift +++ b/Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator+SwiftThunkPrinting.swift @@ -41,6 +41,7 @@ extension FFMSwift2JavaGenerator { } // === All types + // FIXME: write them all into the same file they were declared from +SwiftJava for (_, ty) in self.analysis.importedTypes.sorted(by: { (lhs, rhs) in lhs.key < rhs.key }) { let fileNameBase = "\(ty.swiftNominal.qualifiedName)+SwiftJava" let filename = "\(fileNameBase).swift" diff --git a/Sources/JavaKitConfigurationShared/Configuration.swift b/Sources/JavaKitConfigurationShared/Configuration.swift index 33eb6757..26b01702 100644 --- a/Sources/JavaKitConfigurationShared/Configuration.swift +++ b/Sources/JavaKitConfigurationShared/Configuration.swift @@ -42,7 +42,7 @@ public struct Configuration: Codable { // ==== java 2 swift --------------------------------------------------------- - /// The Java class path that should be passed along to the Java2Swift tool. + /// The Java class path that should be passed along to the swift-java tool. public var classpath: String? = nil public var classpathEntries: [String] { From d8b9429d14a7a87c88d9d115f40c3b0b4f13b2f1 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Fri, 20 Jun 2025 15:14:58 +0900 Subject: [PATCH 14/37] Correct the SwiftAndJavaJarSampleLib example and actually verify created jar --- Package.swift | 16 -- .../JExtractSwiftCommandPlugin.swift | 159 ------------------ .../JExtractSwiftCommandPlugin/_PluginsShared | 1 - .../JExtractSwiftPlugin.swift | 8 - Samples/SwiftAndJavaJarSampleLib/build.gradle | 73 ++++---- .../SwiftAndJavaJarSampleLib/ci-validate.sh | 16 ++ Samples/SwiftKitSampleApp/build.gradle | 18 -- Sources/SwiftJavaTool/SwiftJava.swift | 6 +- .../org/swift/swiftkit/SwiftValueLayout.java | 4 +- 9 files changed, 63 insertions(+), 238 deletions(-) delete mode 100644 Plugins/JExtractSwiftCommandPlugin/JExtractSwiftCommandPlugin.swift delete mode 120000 Plugins/JExtractSwiftCommandPlugin/_PluginsShared create mode 100755 Samples/SwiftAndJavaJarSampleLib/ci-validate.sh diff --git a/Package.swift b/Package.swift index f6aba9de..a82ecce3 100644 --- a/Package.swift +++ b/Package.swift @@ -175,12 +175,6 @@ let package = Package( "JExtractSwiftPlugin" ] ), -// .plugin( -// name: "JExtractSwiftCommandPlugin", -// targets: [ -// "JExtractSwiftCommandPlugin" -// ] -// ), // ==== Examples @@ -429,16 +423,6 @@ let package = Package( "SwiftJavaTool" ] ), -// .plugin( -// name: "JExtractSwiftCommandPlugin", -// capability: .command( -// intent: .custom(verb: "jextract", description: "Extract Java accessors from Swift module"), -// permissions: [ -// ]), -// dependencies: [ -// "SwiftJavaTool" -// ] -// ), .testTarget( name: "JavaKitTests", diff --git a/Plugins/JExtractSwiftCommandPlugin/JExtractSwiftCommandPlugin.swift b/Plugins/JExtractSwiftCommandPlugin/JExtractSwiftCommandPlugin.swift deleted file mode 100644 index d91d4108..00000000 --- a/Plugins/JExtractSwiftCommandPlugin/JExtractSwiftCommandPlugin.swift +++ /dev/null @@ -1,159 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift.org project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -import Foundation -import PackagePlugin - -@main -final class JExtractSwiftCommandPlugin: SwiftJavaPluginProtocol, BuildToolPlugin, CommandPlugin { - - var pluginName: String = "swift-java-command" - var verbose: Bool = getEnvironmentBool("SWIFT_JAVA_VERBOSE") - - /// Build the target before attempting to extract from it. - /// This avoids trying to extract from broken sources. - /// - /// You may disable this if confident that input targets sources are correct and there's no need to kick off a pre-build for some reason. - var buildInputs: Bool = true - - /// Build the target once swift-java sources have been generated. - /// This helps verify that the generated output is correct, and won't miscompile on the next build. - var buildOutputs: Bool = true - - func createBuildCommands(context: PluginContext, target: any Target) async throws -> [Command] { - // FIXME: This is not a build plugin but SwiftPM forces us to impleme the protocol anyway? rdar://139556637 - return [] - } - - func performCommand(context: PluginContext, arguments: [String]) throws { - // Plugin can't have dependencies, so we have some naive argument parsing instead: - self.verbose = arguments.contains("-v") || arguments.contains("--verbose") - - for target in context.package.targets { - guard getSwiftJavaConfigPath(target: target) != nil else { - log("[swift-java-command] Skipping jextract step: Missing swift-java.config for target '\(target.name)'") - continue - } - - do { - let extraArguments = arguments.filter { arg in - arg != "-v" && arg != "--verbose" - } - print("[swift-java-command] Extracting Java wrappers from target: '\(target.name)'...") - try performCommand(context: context, target: target, extraArguments: extraArguments) - } catch { - print("[swift-java-command] error: Failed to extract from target '\(target.name)': \(error)") - } - - print("[swift-java-command] Done.") - } - print("[swift-java-command] Generating sources: " + "done".green + ".") - } - - func prepareJExtractArguments(context: PluginContext, target: Target) throws -> [String] { - guard let sourceModule = target.sourceModule else { return [] } - - // Note: Target doesn't have a directoryURL counterpart to directory, - // so we cannot eliminate this deprecation warning. - let sourceDir = target.directory.string - - let configuration = try readConfiguration(sourceDir: "\(sourceDir)") - - var arguments: [String] = [ - /*subcommand=*/"jextract", - "--swift-module", sourceModule.name, - "--input-swift", sourceDir, - "--output-java", context.outputJavaDirectory.path(percentEncoded: false), - "--output-swift", context.outputSwiftDirectory.path(percentEncoded: false), - // TODO: "--build-cache-directory", ... - // Since plugins cannot depend on libraries we cannot detect what the output files will be, - // as it depends on the contents of the input files. Therefore we have to implement this as a prebuild plugin. - // We'll have to make up some caching inside the tool so we don't re-parse files which have not changed etc. - ] - if let package = configuration?.javaPackage, !package.isEmpty { - arguments += ["--java-package", package] - } - - return arguments - } - - /// Perform the command on a specific target. - func performCommand(context: PluginContext, target: Target, extraArguments: [String]) throws { - guard let sourceModule = target.sourceModule else { return } - - if self.buildInputs { - // Make sure the target can builds properly - log("Pre-building target '\(target.name)' before extracting sources...") - let targetBuildResult = try self.packageManager.build(.target(target.name), parameters: .init()) - - guard targetBuildResult.succeeded else { - print("[swift-java-command] Build of '\(target.name)' failed: \(targetBuildResult.logText)") - return - } - } - - let arguments = try prepareJExtractArguments(context: context, target: target) - - try runExtract(context: context, target: target, arguments: arguments + extraArguments) - - if self.buildOutputs { - // Building the *products* since we need to build the dylib that contains our newly generated sources, - // so just building the target again would not be enough. We build all products which we affected using - // our source generation, which usually would be just a product dylib with our library. - // - // In practice, we'll always want to build after generating; either here, - // or via some other task before we run any Java code, calling into Swift. - log("Post-extract building products with target '\(target.name)'...") - for product in context.package.products where product.targets.contains(where: { $0.id == target.id }) { - log("Post-extract building product '\(product.name)'...") - let buildResult = try self.packageManager.build(.product(product.name), parameters: .init()) - - if buildResult.succeeded { - log("Post-extract build: " + "done".green + ".") - } else { - log("Post-extract build: " + "done".red + "!") - } - } - } - } - - func runExtract(context: PluginContext, target: Target, arguments: [String]) throws { - let process = Process() - process.executableURL = try context.tool(named: "SwiftJavaTool").url - process.arguments = arguments - - do { - log("Execute: \(process.executableURL!.absoluteURL.relativePath) \(arguments.joined(separator: " "))") - - try process.run() - process.waitUntilExit() - - assert(process.terminationStatus == 0, "Process failed with exit code: \(process.terminationStatus)") - } catch { - print("[swift-java-command] Failed to extract Java sources for target: '\(target.name); Error: \(error)") - } - } - -} - -// Mini coloring helper, since we cannot have dependencies we keep it minimal here -extension String { - var red: String { - "\u{001B}[0;31m" + "\(self)" + "\u{001B}[0;0m" - } - var green: String { - "\u{001B}[0;32m" + "\(self)" + "\u{001B}[0;0m" - } -} - diff --git a/Plugins/JExtractSwiftCommandPlugin/_PluginsShared b/Plugins/JExtractSwiftCommandPlugin/_PluginsShared deleted file mode 120000 index de623a5e..00000000 --- a/Plugins/JExtractSwiftCommandPlugin/_PluginsShared +++ /dev/null @@ -1 +0,0 @@ -../PluginsShared \ No newline at end of file diff --git a/Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift b/Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift index 304f80fb..09c84474 100644 --- a/Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift +++ b/Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift @@ -108,14 +108,6 @@ struct JExtractSwiftBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { inputFiles: [ configFile ] + swiftFiles, outputFiles: outputSwiftFiles ) -// .prebuildCommand( -// displayName: "Generate Java wrappers for Swift types", -// executable: toolURL, -// arguments: arguments, -// // inputFiles: [ configFile ] + swiftFiles, -// // outputFiles: outputJavaFiles -// outputFilesDirectory: outputSwiftDirectory -// ) ] } } diff --git a/Samples/SwiftAndJavaJarSampleLib/build.gradle b/Samples/SwiftAndJavaJarSampleLib/build.gradle index 2ccc3e87..47171e03 100644 --- a/Samples/SwiftAndJavaJarSampleLib/build.gradle +++ b/Samples/SwiftAndJavaJarSampleLib/build.gradle @@ -49,21 +49,34 @@ dependencies { testImplementation("org.junit.jupiter:junit-jupiter") } -//// This is for development, when we edit the Swift swift-java project, the outputs of the generated sources may change. -//// Thus, we also need to watch and re-build the top level project. -//def compileSwiftJExtractPlugin = tasks.register("compileSwiftJava", Exec) { -// description = "Rebuild the swift-java root project (" + swiftBuildConfiguration() + ")" -// -// inputs.file(new File(rootDir, "Package.swift")) -// inputs.dir(new File(rootDir, "Sources")) -// outputs.dir(new File(rootDir, ".build")) -// -// workingDir = rootDir -// commandLine "swift" -// args("build", -// "-c", swiftBuildConfiguration(), -// "--product", "swift-java") -//} +def swiftProductsWithJExtractPlugin() { + def stdout = new ByteArrayOutputStream() + def stderr = new ByteArrayOutputStream() + + def result = exec { + commandLine 'swift', 'package', 'describe', '--type', 'json' + standardOutput = stdout + errorOutput = stderr + ignoreExitValue = true + } + + def jsonOutput = stdout.toString() + + if (result.exitValue == 0) { + def json = new JsonSlurper().parseText(jsonOutput) + def products = json.targets + .findAll { target -> + target.product_dependencies?.contains("JExtractSwiftPlugin") + } + .collectMany { target -> + target.product_memberships ?: [] + } + return products + } else { + logger.warn("Command failed: ${stderr.toString()}") + return [] + } +} def jextract = tasks.register("jextract", Exec) { description = "Generate Java wrappers for swift target" @@ -73,12 +86,14 @@ def jextract = tasks.register("jextract", Exec) { inputs.file(new File(rootDir, "Package.swift")) inputs.dir(new File(rootDir, "Sources")) + // If the package description changes, we should execute jextract again, maybe we added jextract to new targets inputs.file(new File(projectDir, "Package.swift")) - inputs.dir(new File(projectDir, "Sources")) - // TODO: we can use package describe --type json to figure out which targets depend on JExtractSwiftPlugin and will produce outputs - // Avoid adding this directory, but create the expected one specifically for all targets - // which WILL produce sources because they have the plugin + // monitor all targets/products which depend on the JExtract plugin + swiftProductsWithJExtractPlugin().each { + logger.info("[swift-java:jextract (Gradle)] Swift input target: ${it}") + inputs.dir(new File(layout.projectDirectory.asFile, "Sources/${it}".toString())) + } outputs.dir(layout.buildDirectory.dir("../.build/plugins/outputs/${layout.projectDirectory.asFile.getName().toLowerCase()}")) File baseSwiftPluginOutputsDir = layout.buildDirectory.dir("../.build/plugins/outputs/").get().asFile @@ -94,16 +109,16 @@ def jextract = tasks.register("jextract", Exec) { workingDir = layout.projectDirectory commandLine "swift" - args("run", - "swift-java", "jextract", - "--swift-module", "MySwiftLibrary", - "--java-module", "MyJavaLibrary", - // "--java-package", "com.example.myjava", - "--java-output", "${layout.buildDirectory.dir(".build/plugins/outputs/${layout.projectDirectory.asFile.getName().toLowerCase()}/JExtractSwiftPlugin/src/generated/java")}", - "--swift-output", "${layout.buildDirectory.dir(".build/plugins/outputs/${layout.projectDirectory.asFile.getName().toLowerCase()}/JExtractSwiftPlugin/Sources")}", -// "-v", -// "--log-level", "info" // TODO: pass log level from Gradle build - ) + args("build") // since Swift targets which need to be jextract-ed have the jextract build plugin, we just need to build + // If we wanted to execute a specific subcommand, we can like this: + // args("run",/* + // "swift-java", "jextract", + // "--swift-module", "MySwiftLibrary", + // // java.package is obtained from the swift-java.config in the swift module + // "--output-java", "${layout.buildDirectory.dir(".build/plugins/outputs/${layout.projectDirectory.asFile.getName().toLowerCase()}/JExtractSwiftPlugin/src/generated/java").get()}", + // "--output-swift", "${layout.buildDirectory.dir(".build/plugins/outputs/${layout.projectDirectory.asFile.getName().toLowerCase()}/JExtractSwiftPlugin/Sources").get()}", + // "--log-level", (logging.level <= LogLevel.INFO ? "debug" : */"info") + // ) } diff --git a/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh b/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh new file mode 100755 index 00000000..f21ed2f8 --- /dev/null +++ b/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e +set -x + +./gradlew jar + +# check if we can compile a plain Example file that uses the generated Java bindings that should be in the generated jar +javac -cp bin/default/build/libs/*jar Example.java + +# Can we run the example? +# - find libswiftCore.dylib +SWIFT_DYLIB_PATHS=$(find "$(swiftly use --print-location)" | grep dylib$ | grep libswiftCore | grep macos | xargs dirname) +# - find our library dylib +SWIFT_DYLIB_PATHS="${SWIFT_DYLIB_PATHS}:$(find . | grep libMySwiftLibrary.dylib$ | sort | head -n1 | xargs dirname)" +java -Djava.library.path="${SWIFT_DYLIB_PATHS}" -cp ".:bin/default/build/libs/*jar:../../SwiftKit/build/libs/*jar" Example \ No newline at end of file diff --git a/Samples/SwiftKitSampleApp/build.gradle b/Samples/SwiftKitSampleApp/build.gradle index bcd8af96..b5d4b848 100644 --- a/Samples/SwiftKitSampleApp/build.gradle +++ b/Samples/SwiftKitSampleApp/build.gradle @@ -34,24 +34,6 @@ java { } } - -// This is for development, when we edit the Swift swift-java project, the outputs of the generated sources may change. -// Thus, we also need to watch and re-build the top level project. -def compileSwiftJExtractPlugin = tasks.register("compileSwiftJExtractPlugin", Exec) { - description = "Rebuild the swift-java root project" - - inputs.file(new File(rootDir, "Package.swift")) - inputs.dir(new File(rootDir, "Sources")) - outputs.dir(new File(rootDir, ".build")) - - workingDir = rootDir - commandLine "swift" - args("build", - "--product", "SwiftKitSwift", - "--product", "JExtractSwiftPlugin", - "--product", "JExtractSwiftCommandPlugin") -} - def jextract = tasks.register("jextract", Exec) { description = "Builds swift sources, including swift-java source generation" dependsOn compileSwiftJExtractPlugin diff --git a/Sources/SwiftJavaTool/SwiftJava.swift b/Sources/SwiftJavaTool/SwiftJava.swift index 8a325f90..76dd48ad 100644 --- a/Sources/SwiftJavaTool/SwiftJava.swift +++ b/Sources/SwiftJavaTool/SwiftJava.swift @@ -27,7 +27,7 @@ import JavaKitShared /// Command-line utility to drive the export of Java classes into Swift types. @main -struct SwiftJava: AsyncParsableCommand { // FIXME: this is just a normal async command, no parsing happening here +struct SwiftJava: AsyncParsableCommand { static var _commandName: String { "swift-java" } static let configuration = CommandConfiguration( @@ -39,10 +39,6 @@ struct SwiftJava: AsyncParsableCommand { // FIXME: this is just a normal async c JExtractCommand.self ]) - var effectiveSwiftModule: String { - fatalError("SHOULD NOT BE USED") - } - mutating func run() async throws { guard CommandLine.arguments.count >= 2 else { // there's no "default" command, print USAGE when no arguments/parameters are passed. diff --git a/SwiftKit/src/main/java/org/swift/swiftkit/SwiftValueLayout.java b/SwiftKit/src/main/java/org/swift/swiftkit/SwiftValueLayout.java index dea52154..8d2d7ad2 100644 --- a/SwiftKit/src/main/java/org/swift/swiftkit/SwiftValueLayout.java +++ b/SwiftKit/src/main/java/org/swift/swiftkit/SwiftValueLayout.java @@ -41,8 +41,8 @@ public static long addressByteSize() { public static final ValueLayout.OfLong SWIFT_INT64 = ValueLayout.JAVA_LONG; public static final ValueLayout.OfFloat SWIFT_FLOAT = ValueLayout.JAVA_FLOAT; public static final ValueLayout.OfDouble SWIFT_DOUBLE = ValueLayout.JAVA_DOUBLE; - public static final AddressLayout SWIFT_POINTER = ValueLayout.ADDRESS - .withTargetLayout(MemoryLayout.sequenceLayout(Long.MAX_VALUE, JAVA_BYTE)); + public static final AddressLayout SWIFT_POINTER = ValueLayout.ADDRESS; + // .withTargetLayout(MemoryLayout.sequenceLayout(Long.MAX_VALUE, JAVA_BYTE)); public static final SequenceLayout SWIFT_BYTE_ARRAY = MemoryLayout.sequenceLayout(8, ValueLayout.JAVA_BYTE); /** From e71e07965e8359bf062ffbbcd3eef1cafb64d58d Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 23 Jun 2025 10:52:28 +0900 Subject: [PATCH 15/37] git-ignore sdkman filest --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b3dbac45..f77390a7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .swift-version +.sdkmanrc .DS_Store .build From ba159c98d367e6a9e3ffbb13ddfd169676a17222 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 23 Jun 2025 11:04:02 +0900 Subject: [PATCH 16/37] wip: bringing validation of sample to linux --- Samples/SwiftAndJavaJarSampleLib/build.gradle | 9 ++++++++- .../SwiftAndJavaJarSampleLib/ci-validate.sh | 19 +++++++++++++------ Samples/SwiftKitSampleApp/build.gradle | 10 ++++++++-- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/Samples/SwiftAndJavaJarSampleLib/build.gradle b/Samples/SwiftAndJavaJarSampleLib/build.gradle index 47171e03..77181ac5 100644 --- a/Samples/SwiftAndJavaJarSampleLib/build.gradle +++ b/Samples/SwiftAndJavaJarSampleLib/build.gradle @@ -78,8 +78,14 @@ def swiftProductsWithJExtractPlugin() { } } +def swiftCheckValid = tasks.register("swift-check-valid", Exec) { + commandLine "swift" + args("-version") +} + def jextract = tasks.register("jextract", Exec) { description = "Generate Java wrappers for swift target" + dependsOn swiftCheckValid // dependsOn compileSwiftJExtractPlugin // only because we depend on "live developing" the plugin while using this project to test it @@ -90,7 +96,8 @@ def jextract = tasks.register("jextract", Exec) { inputs.file(new File(projectDir, "Package.swift")) // monitor all targets/products which depend on the JExtract plugin - swiftProductsWithJExtractPlugin().each { + // swiftProductsWithJExtractPlugin().each { + ["MySwiftLibrary"].each { logger.info("[swift-java:jextract (Gradle)] Swift input target: ${it}") inputs.dir(new File(layout.projectDirectory.asFile, "Sources/${it}".toString())) } diff --git a/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh b/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh index f21ed2f8..b97a265c 100755 --- a/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh +++ b/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh @@ -8,9 +8,16 @@ set -x # check if we can compile a plain Example file that uses the generated Java bindings that should be in the generated jar javac -cp bin/default/build/libs/*jar Example.java -# Can we run the example? -# - find libswiftCore.dylib -SWIFT_DYLIB_PATHS=$(find "$(swiftly use --print-location)" | grep dylib$ | grep libswiftCore | grep macos | xargs dirname) -# - find our library dylib -SWIFT_DYLIB_PATHS="${SWIFT_DYLIB_PATHS}:$(find . | grep libMySwiftLibrary.dylib$ | sort | head -n1 | xargs dirname)" -java -Djava.library.path="${SWIFT_DYLIB_PATHS}" -cp ".:bin/default/build/libs/*jar:../../SwiftKit/build/libs/*jar" Example \ No newline at end of file + +if [ "$(uname -s)" = 'Linux' ] +then + exit 1 # not implemented yet +elif [ "$(uname -s)" = 'Darwin' ] +then + # Can we run the example? + # - find libswiftCore.dylib + SWIFT_DYLIB_PATHS=$(find "$(swiftly use --print-location)" | grep dylib$ | grep libswiftCore | grep macos | xargs dirname) + # - find our library dylib + SWIFT_DYLIB_PATHS="${SWIFT_DYLIB_PATHS}:$(find . | grep libMySwiftLibrary.dylib$ | sort | head -n1 | xargs dirname)" + java -Djava.library.path="${SWIFT_DYLIB_PATHS}" -cp ".:bin/default/build/libs/*jar:../../SwiftKit/build/libs/*jar" Example +fi \ No newline at end of file diff --git a/Samples/SwiftKitSampleApp/build.gradle b/Samples/SwiftKitSampleApp/build.gradle index b5d4b848..222ad7ff 100644 --- a/Samples/SwiftKitSampleApp/build.gradle +++ b/Samples/SwiftKitSampleApp/build.gradle @@ -30,13 +30,19 @@ repositories { java { toolchain { - languageVersion.set(JavaLanguageVersion.of(22)) + languageVersion.set(JavaLanguageVersion.of(23)) } } +def swiftCheckValid = tasks.register("swift-check-valid", Exec) { + commandLine "swift" + args("-version") +} + def jextract = tasks.register("jextract", Exec) { description = "Builds swift sources, including swift-java source generation" - dependsOn compileSwiftJExtractPlugin + dependsOn swiftCheckValid + // dependsOn compileSwiftJExtractPlugin // only because we depend on "live developing" the plugin while using this project to test it inputs.file(new File(rootDir, "Package.swift")) From 7c6a8123fd4dedacfbc4a4003e1356092feaf3e0 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 23 Jun 2025 12:03:10 +0900 Subject: [PATCH 17/37] JDK: Update install_jdk script such that we use Corretto JDK 24 This is primarily to ease up development envs and bring consistency / use the latest release etc. --- ...d-logic.java-common-conventions.gradle.kts | 2 +- Samples/SwiftAndJavaJarSampleLib/build.gradle | 2 +- SwiftKit/build.gradle | 2 +- docker/install_jdk.sh | 33 ++++--------------- 4 files changed, 9 insertions(+), 30 deletions(-) diff --git a/BuildLogic/src/main/kotlin/build-logic.java-common-conventions.gradle.kts b/BuildLogic/src/main/kotlin/build-logic.java-common-conventions.gradle.kts index ef88ce15..fb10bee0 100644 --- a/BuildLogic/src/main/kotlin/build-logic.java-common-conventions.gradle.kts +++ b/BuildLogic/src/main/kotlin/build-logic.java-common-conventions.gradle.kts @@ -23,7 +23,7 @@ plugins { java { toolchain { - languageVersion = JavaLanguageVersion.of(22) + languageVersion = JavaLanguageVersion.of(24) } } diff --git a/Samples/SwiftAndJavaJarSampleLib/build.gradle b/Samples/SwiftAndJavaJarSampleLib/build.gradle index 77181ac5..6a202558 100644 --- a/Samples/SwiftAndJavaJarSampleLib/build.gradle +++ b/Samples/SwiftAndJavaJarSampleLib/build.gradle @@ -38,7 +38,7 @@ repositories { java { toolchain { - languageVersion.set(JavaLanguageVersion.of(22)) + languageVersion.set(JavaLanguageVersion.of(24)) } } diff --git a/SwiftKit/build.gradle b/SwiftKit/build.gradle index e0896e66..8f3b6c51 100644 --- a/SwiftKit/build.gradle +++ b/SwiftKit/build.gradle @@ -25,7 +25,7 @@ repositories { java { toolchain { - languageVersion.set(JavaLanguageVersion.of(22)) + languageVersion.set(JavaLanguageVersion.of(24)) } } diff --git a/docker/install_jdk.sh b/docker/install_jdk.sh index 8aa295ce..a9514bac 100755 --- a/docker/install_jdk.sh +++ b/docker/install_jdk.sh @@ -14,7 +14,7 @@ ##===----------------------------------------------------------------------===## set -euo pipefail -# Supported JDKs: Corretto or OpenJDK +# Supported JDKs: Corretto if [ "$JDK_VENDOR" = "" ]; then declare -r JDK_VENDOR="Corretto" fi @@ -24,34 +24,13 @@ apt-get update && apt-get install -y wget echo "Download JDK for: $(uname -m)" -if [ "$JDK_VENDOR" = 'OpenJDK' ]; then +if [ "$JDK_VENDOR" = 'Corretto' ]; then if [ "$(uname -m)" = 'aarch64' ]; then - declare -r JDK_URL="https://download.java.net/java/GA/jdk23/3c5b90190c68498b986a97f276efd28a/37/GPL/openjdk-23_linux-aarch64_bin.tar.gz" - declare -r EXPECT_JDK_SHA="076dcf7078cdf941951587bf92733abacf489a6570f1df97ee35945ffebec5b7" + declare -r JDK_URL="https://corretto.aws/downloads/latest/amazon-corretto-24-aarch64-linux-jdk.tar.gz" + declare -r EXPECT_JDK_MD5="3b543f4e971350b73d0ab6d8174cc030" else - declare -r JDK_URL="https://download.java.net/java/GA/jdk23/3c5b90190c68498b986a97f276efd28a/37/GPL/$JDK_NAME" - declare -r EXPECT_JDK_SHA="08fea92724127c6fa0f2e5ea0b07ff4951ccb1e2f22db3c21eebbd7347152a67" - fi - - wget -q -O jdk.tar.gz "$JDK_URL" - - declare JDK_SHA # on separate lines due to: SC2155 (warning): Declare and assign separately to avoid masking return values. - JDK_SHA="$(sha256sum jdk.tar.gz | cut -d ' ' -f 1)" - if [ "$JDK_SHA" != "$EXPECT_JDK_SHA" ]; then - echo "Downloaded JDK SHA does not match expected!" - echo "Expected: $EXPECT_JDK_SHA" - echo " Was: $JDK_SHA" - exit 1; - else - echo "JDK SHA is correct."; - fi -elif [ "$JDK_VENDOR" = 'Corretto' ]; then - if [ "$(uname -m)" = 'aarch64' ]; then - declare -r JDK_URL="https://corretto.aws/downloads/latest/amazon-corretto-22-aarch64-linux-jdk.tar.gz" - declare -r EXPECT_JDK_MD5="1ebe5f5229bb18bc784a1e0f54d3fe39" - else - declare -r JDK_URL="https://corretto.aws/downloads/latest/amazon-corretto-22-x64-linux-jdk.tar.gz" - declare -r EXPECT_JDK_MD5="5bd7fe30eb063699a3b4db7a00455841" + declare -r JDK_URL="https://corretto.aws/downloads/latest/amazon-corretto-24-x64-linux-jdk.tar.gz" + declare -r EXPECT_JDK_MD5="130885ded3cbfc712fbe9f7dace45a52" fi wget -q -O jdk.tar.gz "$JDK_URL" From 8461aa053f4854e7ecde87750e9624299193204b Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 23 Jun 2025 12:16:54 +0900 Subject: [PATCH 18/37] enable complete validation of SwiftAndJavaJarSampleLib on linux --- Samples/SwiftAndJavaJarSampleLib/ci-validate.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh b/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh index b97a265c..5e2c6a80 100755 --- a/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh +++ b/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh @@ -8,16 +8,19 @@ set -x # check if we can compile a plain Example file that uses the generated Java bindings that should be in the generated jar javac -cp bin/default/build/libs/*jar Example.java - if [ "$(uname -s)" = 'Linux' ] then - exit 1 # not implemented yet + SWIFT_LIB_PATHS=$HOME/.local/share/swiftly/toolchains/6.2-snapshot-2025-06-17/usr/lib/swift/linux/ + SWIFT_LIB_PATHS="${SWIFT_LIB_PATHS}:$(find . | grep libMySwiftLibrary.so$ | sort | head -n1 | xargs dirname)" elif [ "$(uname -s)" = 'Darwin' ] then - # Can we run the example? # - find libswiftCore.dylib - SWIFT_DYLIB_PATHS=$(find "$(swiftly use --print-location)" | grep dylib$ | grep libswiftCore | grep macos | xargs dirname) + SWIFT_LIB_PATHS=$(find "$(swiftly use --print-location)" | grep dylib$ | grep libswiftCore | grep macos | xargs dirname) # - find our library dylib - SWIFT_DYLIB_PATHS="${SWIFT_DYLIB_PATHS}:$(find . | grep libMySwiftLibrary.dylib$ | sort | head -n1 | xargs dirname)" - java -Djava.library.path="${SWIFT_DYLIB_PATHS}" -cp ".:bin/default/build/libs/*jar:../../SwiftKit/build/libs/*jar" Example -fi \ No newline at end of file + SWIFT_LIB_PATHS="${SWIFT_LIB_PATHS}:$(find . | grep libMySwiftLibrary.dylib$ | sort | head -n1 | xargs dirname)" +fi + +# Can we run the example? +java --enable-native-access=ALL-UNNAMED \ + -Djava.library.path="${SWIFT_LIB_PATHS}" -cp ".:bin/default/build/libs/*:../../SwiftKit/build/libs/*" \ + Example From 16de9034303712b5950e42536bcc562fd4333973 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 23 Jun 2025 12:23:26 +0900 Subject: [PATCH 19/37] move installing swift to Swiftly, so we can pick 6.1 / 6.2 easily --- Samples/SwiftAndJavaJarSampleLib/LICENSE.txt | 202 +++++++++++++++++++ docker/Dockerfile | 7 +- docker/install_swiftly.sh | 28 +++ docker/install_untested_nightly_swift.sh | 43 ---- 4 files changed, 233 insertions(+), 47 deletions(-) create mode 100644 Samples/SwiftAndJavaJarSampleLib/LICENSE.txt create mode 100755 docker/install_swiftly.sh delete mode 100755 docker/install_untested_nightly_swift.sh diff --git a/Samples/SwiftAndJavaJarSampleLib/LICENSE.txt b/Samples/SwiftAndJavaJarSampleLib/LICENSE.txt new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/Samples/SwiftAndJavaJarSampleLib/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/docker/Dockerfile b/docker/Dockerfile index c68ccc3c..a7584489 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -24,7 +24,6 @@ RUN bash -xc 'JDK_VENDOR=Corretto ./install_jdk.sh' ENV JAVA_HOME="/usr/lib/jvm/default-jdk" ENV PATH="$PATH:/usr/lib/jvm/default-jdk/bin" -# Install "untested" nightly 'main' Swift -# TODO: Only do this if the released Swift is older than what we require -#COPY install_untested_nightly_swift.sh . -RUN #bash -xc './install_untested_nightly_swift.sh' +# Install Swift with Swiftly +COPY install_swiftly.sh . +RUN bash -xc './install_swiftly.sh' diff --git a/docker/install_swiftly.sh b/docker/install_swiftly.sh new file mode 100755 index 00000000..e5b42849 --- /dev/null +++ b/docker/install_swiftly.sh @@ -0,0 +1,28 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the Swift.org open source project +## +## Copyright (c) 2024 Apple Inc. and the Swift.org project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of Swift.org project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## +set -euo pipefail + +echo "Install Swiftly and toolchain for: $(uname -m)" + +apt-get install -y curl + +curl -O https://download.swift.org/swiftly/linux/swiftly-$(uname -m).tar.gz +tar zxf swiftly-$(uname -m).tar.gz +./swiftly init --assume-yes +. "${SWIFTLY_HOME_DIR:-$HOME/.local/share/swiftly}/env.sh" +hash -r + +swiftly install 6.1.2 --assume-yes --use +swift -version diff --git a/docker/install_untested_nightly_swift.sh b/docker/install_untested_nightly_swift.sh deleted file mode 100755 index 9cb920ab..00000000 --- a/docker/install_untested_nightly_swift.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the Swift.org open source project -## -## Copyright (c) 2024 Apple Inc. and the Swift.org project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of Swift.org project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## -set -euo pipefail - -echo "Download [nightly] [untested] Swift toolchain for: $(uname -m)" - -ARCH="$(arch)" -if [[ "$ARCH" = "i386" || "$ARCH" = "x86_64" ]]; then - SWIFT_UNTESTED_TOOLCHAIN_JOB_URL="https://ci.swift.org/job/oss-swift-package-ubuntu-22_04/lastSuccessfulBuild/consoleText" -else - SWIFT_UNTESTED_TOOLCHAIN_JOB_URL="https://ci.swift.org/job/oss-swift-package-ubuntu-22_04-aarch64/lastSuccessfulBuild/consoleText" -fi - -if [[ "$(grep "22.04" /etc/lsb-release)" = "" ]]; then - echo "This script specifically only supports Ubuntu 22.04 due to nightly toolchain availability" - exit 1 -fi - -UNTESTED_TOOLCHAIN_URL=$(curl -s $SWIFT_UNTESTED_TOOLCHAIN_JOB_URL | grep 'Toolchain: ' | sed 's/Toolchain: //g') -UNTESTED_TOOLCHAIN_FILENAME=$"toolchain.tar.gz" - -echo "Download toolchain: $UNTESTED_TOOLCHAIN_URL" - -cd / -curl -s "$UNTESTED_TOOLCHAIN_URL" > "$UNTESTED_TOOLCHAIN_FILENAME" - -swift -version - -echo "Extract toolchain: $UNTESTED_TOOLCHAIN_FILENAME" -tar xzf "$UNTESTED_TOOLCHAIN_FILENAME" -swift -version From 1c1f9b857d6cbfc8e7978257236f63cd247f5b1f Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 23 Jun 2025 12:41:54 +0900 Subject: [PATCH 20/37] install JDK 21 for gradle, and JDK 24 for runtime --- docker/install_jdk.sh | 115 ++++++++++++++++++++++++++++++------------ 1 file changed, 83 insertions(+), 32 deletions(-) diff --git a/docker/install_jdk.sh b/docker/install_jdk.sh index a9514bac..6ca7be18 100755 --- a/docker/install_jdk.sh +++ b/docker/install_jdk.sh @@ -14,49 +14,100 @@ ##===----------------------------------------------------------------------===## set -euo pipefail +# We need JDK 24 because that's the supported version with latest FFM +# However, we also need JDK 23 at most because Gradle does not support 24. + # Supported JDKs: Corretto if [ "$JDK_VENDOR" = "" ]; then declare -r JDK_VENDOR="Corretto" fi -echo "Installing $JDK_VENDOR JDK..." apt-get update && apt-get install -y wget echo "Download JDK for: $(uname -m)" -if [ "$JDK_VENDOR" = 'Corretto' ]; then - if [ "$(uname -m)" = 'aarch64' ]; then - declare -r JDK_URL="https://corretto.aws/downloads/latest/amazon-corretto-24-aarch64-linux-jdk.tar.gz" - declare -r EXPECT_JDK_MD5="3b543f4e971350b73d0ab6d8174cc030" - else - declare -r JDK_URL="https://corretto.aws/downloads/latest/amazon-corretto-24-x64-linux-jdk.tar.gz" - declare -r EXPECT_JDK_MD5="130885ded3cbfc712fbe9f7dace45a52" - fi - - wget -q -O jdk.tar.gz "$JDK_URL" - - declare JDK_MD5 # on separate lines due to: SC2155 (warning): Declare and assign separately to avoid masking return values. - JDK_MD5="$(md5sum jdk.tar.gz | cut -d ' ' -f 1)" - if [ "$JDK_MD5" != "$EXPECT_JDK_MD5" ]; then - echo "Downloaded JDK MD5 does not match expected!" - echo "Expected: $EXPECT_JDK_MD5" - echo " Was: $JDK_MD5" - exit 1; - else - echo "JDK MD5 is correct."; - fi -else - echo "Unsupported JDK vendor: '$JDK_VENDOR'" - exit 1 -fi +download_and_install_jdk() { + local jdk_version="$1" + local jdk_url="" + local expected_md5="" + + echo "Installing $JDK_VENDOR JDK (${jdk_version})..." + + if [ "$JDK_VENDOR" = 'Corretto' ]; then + if [ "$(uname -m)" = 'aarch64' ]; then + case "$jdk_version" in + "21") + jdk_url="https://corretto.aws/downloads/latest/amazon-corretto-21-aarch64-linux-jdk.tar.gz" + expected_md5="87e458029cf9976945dfa3a22af3f850" + ;; + "24") + jdk_url="https://corretto.aws/downloads/latest/amazon-corretto-24-aarch64-linux-jdk.tar.gz" + expected_md5="3b543f4e971350b73d0ab6d8174cc030" + ;; + *) + echo "Unsupported JDK version: '$jdk_version'" + exit 1 + ;; + esac + else + case "$jdk_version" in + "21") + jdk_url="https://corretto.aws/downloads/latest/amazon-corretto-21-x64-linux-jdk.tar.gz" + expected_md5="84368821f590bd58708d9e350534c7f8" + ;; + "24") + jdk_url="https://corretto.aws/downloads/latest/amazon-corretto-24-x64-linux-jdk.tar.gz" + expected_md5="130885ded3cbfc712fbe9f7dace45a52" + ;; + *) + echo "Unsupported JDK version: '$jdk_version'" + exit 1 + ;; + esac + fi + else + echo "Unsupported JDK vendor: '$JDK_VENDOR'" + exit 1 + fi + + # Download JDK + local jdk_filename="jdk_${jdk_version}.tar.gz" + wget -q -O "$jdk_filename" "$jdk_url" + + # Verify MD5 + local jdk_md5 + jdk_md5="$(md5sum "$jdk_filename" | cut -d ' ' -f 1)" + if [ "$jdk_md5" != "$expected_md5" ]; then + echo "Downloaded JDK $jdk_version MD5 does not match expected!" + echo "Expected: $expected_md5" + echo " Was: $jdk_md5" + exit 1 + else + echo "JDK $jdk_version MD5 is correct." + fi + + # Extract and install JDK + mkdir -p "/usr/lib/jvm/jdk-${jdk_version}" + mv "$jdk_filename" "/usr/lib/jvm/jdk-${jdk_version}/" + cd "/usr/lib/jvm/jdk-${jdk_version}/" || exit 1 + tar xzf "$jdk_filename" && rm "$jdk_filename" + + # Move extracted directory to a standard name + local extracted_dir + extracted_dir="$(find . -maxdepth 1 -type d -name "*jdk*" | head -n1)" + if [ -n "$extracted_dir" ]; then + mv "$extracted_dir"/* . + rm -rf "$extracted_dir" + fi -# Extract and verify the JDK installation + echo "JDK $jdk_version installed successfully in /usr/lib/jvm/jdk-${jdk_version}/" + cd +} -mkdir -p /usr/lib/jvm/ -mv jdk.tar.gz /usr/lib/jvm/ -cd /usr/lib/jvm/ -tar xzvf jdk.tar.gz && rm jdk.tar.gz -mv "$(find . -depth -maxdepth 1 -type d | head -n1)" default-jdk +# Usage: Install both JDK versions +download_and_install_jdk "21" +download_and_install_jdk "24" +ln -sf /usr/lib/jvm/jdk-21 /usr/lib/jvm/default-jdk echo "JAVA_HOME = /usr/lib/jvm/default-jdk" /usr/lib/jvm/default-jdk/bin/java -version \ No newline at end of file From 5c49306fbb89537bb72c97488d3f4c998f8ad2fb Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 23 Jun 2025 12:49:59 +0900 Subject: [PATCH 21/37] Reworking CI validation for samples to pass on linux --- .github/actions/prepare_env/action.yml | 2 +- .github/workflows/pull_request.yml | 24 ++++++++---------------- docker/Dockerfile | 9 +++++---- docker/install_jdk.sh | 19 ++++++++++--------- 4 files changed, 24 insertions(+), 30 deletions(-) diff --git a/.github/actions/prepare_env/action.yml b/.github/actions/prepare_env/action.yml index ebcc8aed..5142b535 100644 --- a/.github/actions/prepare_env/action.yml +++ b/.github/actions/prepare_env/action.yml @@ -15,7 +15,7 @@ runs: path: /usr/lib/jvm/default-jdk/ key: ${{ runner.os }}-jdk-${{ matrix.jdk_vendor }}-${{ hashFiles('/usr/lib/jvm/default-jdk/*') }} restore-keys: | - ${{ runner.os }}-jdk- + ${{ runner.os }}-jdk-${{ matrix.jdk_vendor }} - name: Install JDK if: steps.cache-jdk.outputs.cache-hit != 'true' run: "bash -xc 'JDK_VENDOR=${{ matrix.jdk_vendor }} ./docker/install_jdk.sh'" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 2df3c596..39fa7ee6 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -20,8 +20,7 @@ jobs: strategy: fail-fast: true matrix: - # swift_version: ['nightly-main'] - swift_version: ['6.0.2'] + swift_version: ['6.1.2'] os_version: ['jammy'] jdk_vendor: ['Corretto'] container: @@ -46,8 +45,7 @@ jobs: strategy: fail-fast: false matrix: - # swift_version: ['nightly-main'] - swift_version: ['6.0.2'] + swift_version: ['6.1.2'] os_version: ['jammy'] jdk_vendor: ['Corretto'] container: @@ -70,8 +68,7 @@ jobs: strategy: fail-fast: false matrix: - # swift_version: ['nightly-main'] - swift_version: ['6.0.2'] + swift_version: ['6.1.2'] os_version: ['jammy'] jdk_vendor: ['Corretto'] container: @@ -90,8 +87,7 @@ jobs: strategy: fail-fast: false matrix: - # swift_version: ['nightly-main'] - swift_version: ['6.0.2'] + swift_version: ['6.1.2'] os_version: ['jammy'] jdk_vendor: ['Corretto'] container: @@ -110,8 +106,7 @@ jobs: strategy: fail-fast: false matrix: - # swift_version: ['nightly-main'] - swift_version: ['6.0.2'] + swift_version: ['6.1.2'] os_version: ['jammy'] jdk_vendor: ['Corretto'] container: @@ -130,8 +125,7 @@ jobs: strategy: fail-fast: false matrix: - # swift_version: ['nightly-main'] - swift_version: ['6.0.2'] + swift_version: ['6.1.2'] os_version: ['jammy'] jdk_vendor: ['Corretto'] container: @@ -150,8 +144,7 @@ jobs: strategy: fail-fast: false matrix: - # swift_version: ['nightly-main'] - swift_version: ['6.0.2'] + swift_version: ['6.1.2'] os_version: ['jammy'] jdk_vendor: ['Corretto'] container: @@ -170,8 +163,7 @@ jobs: strategy: fail-fast: false matrix: - # swift_version: ['nightly-main'] - swift_version: ['6.0.2'] + swift_version: ['6.1.2'] os_version: ['jammy'] jdk_vendor: ['Corretto'] container: diff --git a/docker/Dockerfile b/docker/Dockerfile index a7584489..d5599841 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ ARG swift_version=nightly-main ARG ubuntu_version=jammy -ARG base_image=docker.io/swiftlang/swift:$swift_version-$ubuntu_version +ARG base_image=docker.io/swiftlang/swift:${swift_version}-${ubuntu_version} FROM $base_image # needed to do again after FROM due to docker limitation ARG swift_version @@ -18,12 +18,13 @@ ENV LC_ALL=en_US.UTF-8 ENV LANG=en_US.UTF-8 ENV LANGUAGE=en_US.UTF-8 -# JDK dependency COPY install_jdk.sh . +COPY install_swiftly.sh . + +# JDK dependency RUN bash -xc 'JDK_VENDOR=Corretto ./install_jdk.sh' ENV JAVA_HOME="/usr/lib/jvm/default-jdk" ENV PATH="$PATH:/usr/lib/jvm/default-jdk/bin" # Install Swift with Swiftly -COPY install_swiftly.sh . -RUN bash -xc './install_swiftly.sh' +# RUN bash -xc './install_swiftly.sh' diff --git a/docker/install_jdk.sh b/docker/install_jdk.sh index 6ca7be18..8d0ddda3 100755 --- a/docker/install_jdk.sh +++ b/docker/install_jdk.sh @@ -22,7 +22,7 @@ if [ "$JDK_VENDOR" = "" ]; then declare -r JDK_VENDOR="Corretto" fi -apt-get update && apt-get install -y wget +apt-get update && apt-get install -y wget tree echo "Download JDK for: $(uname -m)" @@ -53,7 +53,7 @@ download_and_install_jdk() { case "$jdk_version" in "21") jdk_url="https://corretto.aws/downloads/latest/amazon-corretto-21-x64-linux-jdk.tar.gz" - expected_md5="84368821f590bd58708d9e350534c7f8" + expected_md5="a123e7f50807c27de521bef7378d3377" ;; "24") jdk_url="https://corretto.aws/downloads/latest/amazon-corretto-24-x64-linux-jdk.tar.gz" @@ -94,20 +94,21 @@ download_and_install_jdk() { # Move extracted directory to a standard name local extracted_dir - extracted_dir="$(find . -maxdepth 1 -type d -name "*jdk*" | head -n1)" - if [ -n "$extracted_dir" ]; then - mv "$extracted_dir"/* . - rm -rf "$extracted_dir" - fi + extracted_dir="$(find . -maxdepth 1 -type d -name '*linux*' | head -n1)" + echo "move $extracted_dir to $(pwd)..." + mv "${extracted_dir}"/* . echo "JDK $jdk_version installed successfully in /usr/lib/jvm/jdk-${jdk_version}/" - cd + cd "$HOME" } # Usage: Install both JDK versions download_and_install_jdk "21" download_and_install_jdk "24" -ln -sf /usr/lib/jvm/jdk-21 /usr/lib/jvm/default-jdk +ls -la /usr/lib/jvm/ +cd /usr/lib/jvm/ +ln -s jdk-21 default-jdk +find . | grep java | grep bin echo "JAVA_HOME = /usr/lib/jvm/default-jdk" /usr/lib/jvm/default-jdk/bin/java -version \ No newline at end of file From f721854589ca08789199ee7830d0b66edfc99205 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 23 Jun 2025 15:46:25 +0900 Subject: [PATCH 22/37] adjust jdk version we need for runtime to match installed jdk --- Samples/SwiftKitSampleApp/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Samples/SwiftKitSampleApp/build.gradle b/Samples/SwiftKitSampleApp/build.gradle index 222ad7ff..d57efe29 100644 --- a/Samples/SwiftKitSampleApp/build.gradle +++ b/Samples/SwiftKitSampleApp/build.gradle @@ -30,7 +30,7 @@ repositories { java { toolchain { - languageVersion.set(JavaLanguageVersion.of(23)) + languageVersion.set(JavaLanguageVersion.of(24)) } } From 57865db70299507afb704f6c838f7b0176ccaa8e Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 23 Jun 2025 15:50:29 +0900 Subject: [PATCH 23/37] make sure we have JAVA_HOME/bin on the PATH in validation --- Samples/SwiftAndJavaJarSampleLib/ci-validate.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh b/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh index 5e2c6a80..10b9bcec 100755 --- a/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh +++ b/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh @@ -3,6 +3,8 @@ set -e set -x +export PATH="${PATH}:${JAVA_HOME}/bin" + ./gradlew jar # check if we can compile a plain Example file that uses the generated Java bindings that should be in the generated jar From 6d5e3dcaeac849398a26dfc9cdd88a37b8bd9351 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 23 Jun 2025 15:57:02 +0900 Subject: [PATCH 24/37] correct jdk caching, now that we have to cache two JDKs --- .github/actions/prepare_env/action.yml | 8 ++++---- Samples/SwiftAndJavaJarSampleLib/ci-validate.sh | 16 +++++++++------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/actions/prepare_env/action.yml b/.github/actions/prepare_env/action.yml index 5142b535..dfd0a435 100644 --- a/.github/actions/prepare_env/action.yml +++ b/.github/actions/prepare_env/action.yml @@ -7,15 +7,15 @@ runs: - name: Install System Dependencies run: apt-get -qq update && apt-get -qq install -y make curl wget libjemalloc2 libjemalloc-dev shell: bash - - name: Cache JDK + - name: Cache JDKs id: cache-jdk uses: actions/cache@v4 continue-on-error: true with: - path: /usr/lib/jvm/default-jdk/ - key: ${{ runner.os }}-jdk-${{ matrix.jdk_vendor }}-${{ hashFiles('/usr/lib/jvm/default-jdk/*') }} + path: /usr/lib/jvm/ + key: ${{ runner.os }}-jdk-${{ matrix.jdk_vendor }}-${{ hashFiles('/usr/lib/jvm/*') }} restore-keys: | - ${{ runner.os }}-jdk-${{ matrix.jdk_vendor }} + ${{ runner.os }}-jdk- - name: Install JDK if: steps.cache-jdk.outputs.cache-hit != 'true' run: "bash -xc 'JDK_VENDOR=${{ matrix.jdk_vendor }} ./docker/install_jdk.sh'" diff --git a/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh b/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh index 10b9bcec..aedd11f8 100755 --- a/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh +++ b/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh @@ -3,26 +3,28 @@ set -e set -x -export PATH="${PATH}:${JAVA_HOME}/bin" - ./gradlew jar +# we make sure to build and run with JDK 24 because the runtime needs latest JDK, unlike Gradle which needed 21. +export PATH="${PATH}:/usr/lib/jvm/jdk-24/bin" + # check if we can compile a plain Example file that uses the generated Java bindings that should be in the generated jar -javac -cp bin/default/build/libs/*jar Example.java +MYLIB_CLASSPATH="$(ls bin/default/build/libs/*.jar)" +javac -cp "${MYLIB_CLASSPATH}" Example.java if [ "$(uname -s)" = 'Linux' ] then - SWIFT_LIB_PATHS=$HOME/.local/share/swiftly/toolchains/6.2-snapshot-2025-06-17/usr/lib/swift/linux/ + SWIFT_LIB_PATHS=/usr/lib/swift/linux SWIFT_LIB_PATHS="${SWIFT_LIB_PATHS}:$(find . | grep libMySwiftLibrary.so$ | sort | head -n1 | xargs dirname)" elif [ "$(uname -s)" = 'Darwin' ] then - # - find libswiftCore.dylib SWIFT_LIB_PATHS=$(find "$(swiftly use --print-location)" | grep dylib$ | grep libswiftCore | grep macos | xargs dirname) - # - find our library dylib SWIFT_LIB_PATHS="${SWIFT_LIB_PATHS}:$(find . | grep libMySwiftLibrary.dylib$ | sort | head -n1 | xargs dirname)" fi # Can we run the example? +SWIFTKIT_CLASSPATH="$(ls ../../SwiftKit/build/libs/*.jar)" java --enable-native-access=ALL-UNNAMED \ - -Djava.library.path="${SWIFT_LIB_PATHS}" -cp ".:bin/default/build/libs/*:../../SwiftKit/build/libs/*" \ + -Djava.library.path="${SWIFT_LIB_PATHS}" \ + -cp ".:${MYLIB_CLASSPATH}:${SWIFTKIT_CLASSPATH}" \ Example From f8deab8259676806e5a7088168fd94c0dae1861d Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 23 Jun 2025 17:01:48 +0900 Subject: [PATCH 25/37] fixing classpaths and library paths... --- Package.swift | 2 +- .../JavaDependencySampleApp/ci-validate.sh | 2 +- .../SwiftAndJavaJarSampleLib/ci-validate.sh | 38 ++++++++++++++++--- Samples/SwiftKitSampleApp/Package.swift | 2 +- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/Package.swift b/Package.swift index a82ecce3..58ee1840 100644 --- a/Package.swift +++ b/Package.swift @@ -86,7 +86,7 @@ let javaIncludePath = "\(javaHome)/include" let package = Package( name: "SwiftJava", platforms: [ - .macOS(.v13) + .macOS(.v15) ], products: [ // ==== JavaKit (i.e. calling Java directly Swift utilities) diff --git a/Samples/JavaDependencySampleApp/ci-validate.sh b/Samples/JavaDependencySampleApp/ci-validate.sh index 2adce5a7..2758e6aa 100755 --- a/Samples/JavaDependencySampleApp/ci-validate.sh +++ b/Samples/JavaDependencySampleApp/ci-validate.sh @@ -8,7 +8,7 @@ swift run --disable-sandbox # explicitly invoke resolve without explicit path or dependency # the dependencies should be uses from the --swift-module -.build/plugins/tools/debug/SwiftJavaTool-tool resolve \ +swift run swift-java resolve \ Sources/JavaCommonsCSV/swift-java.config \ --swift-module JavaCommonsCSV \ --output-directory .build/plugins/outputs/javadependencysampleapp/JavaCommonsCSV/destination/SwiftJavaPlugin/ diff --git a/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh b/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh index aedd11f8..947ac444 100755 --- a/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh +++ b/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh @@ -5,26 +5,52 @@ set -x ./gradlew jar +export SWIFT_VERSION="$(swift -version | awk '/Swift version/ { print $3 }')" + # we make sure to build and run with JDK 24 because the runtime needs latest JDK, unlike Gradle which needed 21. -export PATH="${PATH}:/usr/lib/jvm/jdk-24/bin" +if [ "$(uname -s)" = 'Darwin' ] +then + export OS='osx' +elif [ "$(uname -s)" = 'Linux' ] +then + export OS='linux' + export PATH="${PATH}:/usr/lib/jvm/jdk-24/bin" # we need to make sure to use the latest JDK to actually compile/run the executable +fi # check if we can compile a plain Example file that uses the generated Java bindings that should be in the generated jar -MYLIB_CLASSPATH="$(ls bin/default/build/libs/*.jar)" -javac -cp "${MYLIB_CLASSPATH}" Example.java +# The classpath MUST end with a * if it contains jar files, and must not if it directly contains class files. +SWIFTKIT_CLASSPATH="$(pwd)/../../SwiftKit/build/libs/*" +MYLIB_CLASSPATH="$(pwd)/build/libs/*" +CLASSPATH="$(pwd)/:${SWIFTKIT_CLASSPATH}:${MYLIB_CLASSPATH}" +echo "CLASSPATH = ${CLASSPATH}" + +javac -cp "${CLASSPATH}" Example.java if [ "$(uname -s)" = 'Linux' ] then SWIFT_LIB_PATHS=/usr/lib/swift/linux SWIFT_LIB_PATHS="${SWIFT_LIB_PATHS}:$(find . | grep libMySwiftLibrary.so$ | sort | head -n1 | xargs dirname)" + + # if we are on linux, find the Swiftly or System-wide installed libraries dir + SWIFT_CORE_LIB=$(find "$HOME"/.local -name "libswiftCore.so" 2>/dev/null | grep "$SWIFT_VERSION" | head -n1) + if [ -n "$SWIFT_CORE_LIB" ]; then + SWIFT_LIB_PATHS="${SWIFT_LIB_PATHS}:$(dirname "$SWIFT_CORE_LIB")" + else + # maybe there is one installed system-wide in /usr/lib? + SWIFT_CORE_LIB2=$(find /usr/lib -name "libswiftCore.so" 2>/dev/null | grep "$SWIFT_VERSION" | head -n1) + if [ -n "$SWIFT_CORE_LIB2" ]; then + SWIFT_LIB_PATHS="${SWIFT_LIB_PATHS}:$(dirname "$SWIFT_CORE_LIB2")" + fi + fi elif [ "$(uname -s)" = 'Darwin' ] then SWIFT_LIB_PATHS=$(find "$(swiftly use --print-location)" | grep dylib$ | grep libswiftCore | grep macos | xargs dirname) - SWIFT_LIB_PATHS="${SWIFT_LIB_PATHS}:$(find . | grep libMySwiftLibrary.dylib$ | sort | head -n1 | xargs dirname)" + SWIFT_LIB_PATHS="${SWIFT_LIB_PATHS}:$(pwd)/$(find . | grep libMySwiftLibrary.dylib$ | sort | head -n1 | xargs dirname)" fi +echo "SWIFT_LIB_PATHS = ${SWIFT_LIB_PATHS}" # Can we run the example? -SWIFTKIT_CLASSPATH="$(ls ../../SwiftKit/build/libs/*.jar)" java --enable-native-access=ALL-UNNAMED \ -Djava.library.path="${SWIFT_LIB_PATHS}" \ - -cp ".:${MYLIB_CLASSPATH}:${SWIFTKIT_CLASSPATH}" \ + -cp "${CLASSPATH}" \ Example diff --git a/Samples/SwiftKitSampleApp/Package.swift b/Samples/SwiftKitSampleApp/Package.swift index 65de7d3a..34d8fcd3 100644 --- a/Samples/SwiftKitSampleApp/Package.swift +++ b/Samples/SwiftKitSampleApp/Package.swift @@ -43,7 +43,7 @@ let javaIncludePath = "\(javaHome)/include" let package = Package( name: "SwiftKitSampleApp", platforms: [ - .macOS(.v10_15) + .macOS(.v15) ], products: [ .library( From 5ba1365ad4326367368e4f4959686f144e5ec71a Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 23 Jun 2025 19:48:45 +0900 Subject: [PATCH 26/37] some shellcheck fixes --- Samples/SwiftAndJavaJarSampleLib/ci-validate.sh | 2 +- docker/install_swiftly.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh b/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh index 947ac444..1b4769e8 100755 --- a/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh +++ b/Samples/SwiftAndJavaJarSampleLib/ci-validate.sh @@ -5,7 +5,7 @@ set -x ./gradlew jar -export SWIFT_VERSION="$(swift -version | awk '/Swift version/ { print $3 }')" +SWIFT_VERSION="$(swift -version | awk '/Swift version/ { print $3 }')" # we make sure to build and run with JDK 24 because the runtime needs latest JDK, unlike Gradle which needed 21. if [ "$(uname -s)" = 'Darwin' ] diff --git a/docker/install_swiftly.sh b/docker/install_swiftly.sh index e5b42849..3cf4eda0 100755 --- a/docker/install_swiftly.sh +++ b/docker/install_swiftly.sh @@ -18,8 +18,8 @@ echo "Install Swiftly and toolchain for: $(uname -m)" apt-get install -y curl -curl -O https://download.swift.org/swiftly/linux/swiftly-$(uname -m).tar.gz -tar zxf swiftly-$(uname -m).tar.gz +curl -O "https://download.swift.org/swiftly/linux/swiftly-$(uname -m).tar.gz" +tar zxf "swiftly-$(uname -m).tar.gz" ./swiftly init --assume-yes . "${SWIFTLY_HOME_DIR:-$HOME/.local/share/swiftly}/env.sh" hash -r From 816c77cf73db9e830ddc90f54002954f208f4ebe Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 23 Jun 2025 20:26:32 +0900 Subject: [PATCH 27/37] swift-java: print the complete invocation args, to help debugging when validation failed --- Sources/SwiftJavaTool/SwiftJava.swift | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Sources/SwiftJavaTool/SwiftJava.swift b/Sources/SwiftJavaTool/SwiftJava.swift index 76dd48ad..07eacce4 100644 --- a/Sources/SwiftJavaTool/SwiftJava.swift +++ b/Sources/SwiftJavaTool/SwiftJava.swift @@ -39,6 +39,20 @@ struct SwiftJava: AsyncParsableCommand { JExtractCommand.self ]) + public static func main() async { + do { + var command = try parseAsRoot(nil) + if var asyncCommand = command as? AsyncParsableCommand { + try await asyncCommand.run() + } else { + try command.run() + } + } catch { + print("Invocation: \(CommandLine.arguments.joined(separator: " "))") + exit(withError: error) + } + } + mutating func run() async throws { guard CommandLine.arguments.count >= 2 else { // there's no "default" command, print USAGE when no arguments/parameters are passed. From f48bccaa41ee3affe64462b1a3b8e1d1cb771ae4 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 23 Jun 2025 20:34:25 +0900 Subject: [PATCH 28/37] fix shellcheck with swiftly --- docker/install_swiftly.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/install_swiftly.sh b/docker/install_swiftly.sh index 3cf4eda0..756373a9 100755 --- a/docker/install_swiftly.sh +++ b/docker/install_swiftly.sh @@ -21,7 +21,8 @@ apt-get install -y curl curl -O "https://download.swift.org/swiftly/linux/swiftly-$(uname -m).tar.gz" tar zxf "swiftly-$(uname -m).tar.gz" ./swiftly init --assume-yes -. "${SWIFTLY_HOME_DIR:-$HOME/.local/share/swiftly}/env.sh" +disable=SC1091 # we're ok to not validate the swiftly script +. "${SWIFTLY_HOME_DIR:-$HOME/.local/share/swiftly}/env.sh" && \ hash -r swiftly install 6.1.2 --assume-yes --use From 1ba1eb5cf50a55405274644daf18dc0197710742 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 23 Jun 2025 20:44:03 +0900 Subject: [PATCH 29/37] Make use of jextract build plugin from all gradle builds (TODO: make it a plugin) --- Samples/SwiftAndJavaJarSampleLib/build.gradle | 4 +- Samples/SwiftKitSampleApp/build.gradle | 56 ++++++++++++++++--- .../swiftkit/SwiftValueWitnessTable.java | 1 - 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/Samples/SwiftAndJavaJarSampleLib/build.gradle b/Samples/SwiftAndJavaJarSampleLib/build.gradle index 6a202558..cc4c79d4 100644 --- a/Samples/SwiftAndJavaJarSampleLib/build.gradle +++ b/Samples/SwiftAndJavaJarSampleLib/build.gradle @@ -86,7 +86,6 @@ def swiftCheckValid = tasks.register("swift-check-valid", Exec) { def jextract = tasks.register("jextract", Exec) { description = "Generate Java wrappers for swift target" dependsOn swiftCheckValid - // dependsOn compileSwiftJExtractPlugin // only because we depend on "live developing" the plugin while using this project to test it inputs.file(new File(rootDir, "Package.swift")) @@ -96,8 +95,7 @@ def jextract = tasks.register("jextract", Exec) { inputs.file(new File(projectDir, "Package.swift")) // monitor all targets/products which depend on the JExtract plugin - // swiftProductsWithJExtractPlugin().each { - ["MySwiftLibrary"].each { + swiftProductsWithJExtractPlugin().each { logger.info("[swift-java:jextract (Gradle)] Swift input target: ${it}") inputs.dir(new File(layout.projectDirectory.asFile, "Sources/${it}".toString())) } diff --git a/Samples/SwiftKitSampleApp/build.gradle b/Samples/SwiftKitSampleApp/build.gradle index d57efe29..647eaba9 100644 --- a/Samples/SwiftKitSampleApp/build.gradle +++ b/Samples/SwiftKitSampleApp/build.gradle @@ -12,9 +12,11 @@ // //===----------------------------------------------------------------------===// +import groovy.json.JsonSlurper import org.swift.swiftkit.gradle.BuildUtils import java.nio.file.* +import kotlinx.serialization.json.* plugins { id("build-logic.java-application-conventions") @@ -34,26 +36,57 @@ java { } } +def swiftProductsWithJExtractPlugin() { + def stdout = new ByteArrayOutputStream() + def stderr = new ByteArrayOutputStream() + + def result = exec { + commandLine 'swift', 'package', 'describe', '--type', 'json' + standardOutput = stdout + errorOutput = stderr + ignoreExitValue = true + } + + def jsonOutput = stdout.toString() + + if (result.exitValue == 0) { + def json = new JsonSlurper().parseText(jsonOutput) + def products = json.targets + .findAll { target -> + target.product_dependencies?.contains("JExtractSwiftPlugin") + } + .collectMany { target -> + target.product_memberships ?: [] + } + return products + } else { + logger.warn("Command failed: ${stderr.toString()}") + return [] + } +} + + def swiftCheckValid = tasks.register("swift-check-valid", Exec) { commandLine "swift" args("-version") } def jextract = tasks.register("jextract", Exec) { - description = "Builds swift sources, including swift-java source generation" + description = "Generate Java wrappers for swift target" dependsOn swiftCheckValid - // dependsOn compileSwiftJExtractPlugin // only because we depend on "live developing" the plugin while using this project to test it inputs.file(new File(rootDir, "Package.swift")) inputs.dir(new File(rootDir, "Sources")) + // If the package description changes, we should execute jextract again, maybe we added jextract to new targets inputs.file(new File(projectDir, "Package.swift")) - inputs.dir(new File(projectDir, "Sources")) - // TODO: we can use package describe --type json to figure out which targets depend on JExtractSwiftPlugin and will produce outputs - // Avoid adding this directory, but create the expected one specifically for all targets - // which WILL produce sources because they have the plugin + // monitor all targets/products which depend on the JExtract plugin + swiftProductsWithJExtractPlugin().each { + logger.info("[swift-java:jextract (Gradle)] Swift input target: ${it}") + inputs.dir(new File(layout.projectDirectory.asFile, "Sources/${it}".toString())) + } outputs.dir(layout.buildDirectory.dir("../.build/plugins/outputs/${layout.projectDirectory.asFile.getName().toLowerCase()}")) File baseSwiftPluginOutputsDir = layout.buildDirectory.dir("../.build/plugins/outputs/").get().asFile @@ -69,7 +102,16 @@ def jextract = tasks.register("jextract", Exec) { workingDir = layout.projectDirectory commandLine "swift" - args("run", "swift-java", "jextract", "-v", "--log-level", "info") // TODO: pass log level from Gradle build + args("build") // since Swift targets which need to be jextract-ed have the jextract build plugin, we just need to build + // If we wanted to execute a specific subcommand, we can like this: + // args("run",/* + // "swift-java", "jextract", + // "--swift-module", "MySwiftLibrary", + // // java.package is obtained from the swift-java.config in the swift module + // "--output-java", "${layout.buildDirectory.dir(".build/plugins/outputs/${layout.projectDirectory.asFile.getName().toLowerCase()}/JExtractSwiftPlugin/src/generated/java").get()}", + // "--output-swift", "${layout.buildDirectory.dir(".build/plugins/outputs/${layout.projectDirectory.asFile.getName().toLowerCase()}/JExtractSwiftPlugin/Sources").get()}", + // "--log-level", (logging.level <= LogLevel.INFO ? "debug" : */"info") + // ) } // Add the java-swift generated Java sources diff --git a/SwiftKit/src/main/java/org/swift/swiftkit/SwiftValueWitnessTable.java b/SwiftKit/src/main/java/org/swift/swiftkit/SwiftValueWitnessTable.java index 53680f5f..2a031b7e 100644 --- a/SwiftKit/src/main/java/org/swift/swiftkit/SwiftValueWitnessTable.java +++ b/SwiftKit/src/main/java/org/swift/swiftkit/SwiftValueWitnessTable.java @@ -71,7 +71,6 @@ public static MemorySegment fullTypeMetadata(MemorySegment typeMetadata) { public static MemorySegment valueWitnessTable(MemorySegment typeMetadata) { return fullTypeMetadata(typeMetadata) .get(SwiftValueLayout.SWIFT_POINTER, SwiftValueWitnessTable.fullTypeMetadata$vwt$offset); -// .get(ValueLayout.ADDRESS, SwiftValueWitnessTable.fullTypeMetadata$vwt$offset); } From 740af4d58d852c6bb06101eb6173aca7a9cbbc22 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 23 Jun 2025 21:02:05 +0900 Subject: [PATCH 30/37] bump samples to macOS 15 base for now --- Samples/JavaDependencySampleApp/Package.swift | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Samples/JavaDependencySampleApp/Package.swift b/Samples/JavaDependencySampleApp/Package.swift index b39e7b81..4fa33116 100644 --- a/Samples/JavaDependencySampleApp/Package.swift +++ b/Samples/JavaDependencySampleApp/Package.swift @@ -43,11 +43,7 @@ let javaIncludePath = "\(javaHome)/include" let package = Package( name: "JavaDependencySampleApp", platforms: [ - .macOS(.v13), - .iOS(.v13), - .tvOS(.v13), - .watchOS(.v6), - .macCatalyst(.v13), + .macOS(.v15), ], products: [ From 8ff4d8777ec3538cc7f86965f9b53658b3830b0d Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Mon, 23 Jun 2025 21:04:27 +0900 Subject: [PATCH 31/37] shellcheck disable checking swiftly script --- docker/install_swiftly.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/install_swiftly.sh b/docker/install_swiftly.sh index 756373a9..1284b047 100755 --- a/docker/install_swiftly.sh +++ b/docker/install_swiftly.sh @@ -21,7 +21,7 @@ apt-get install -y curl curl -O "https://download.swift.org/swiftly/linux/swiftly-$(uname -m).tar.gz" tar zxf "swiftly-$(uname -m).tar.gz" ./swiftly init --assume-yes -disable=SC1091 # we're ok to not validate the swiftly script +# disable=SC1091 # we're ok to not validate the swiftly script . "${SWIFTLY_HOME_DIR:-$HOME/.local/share/swiftly}/env.sh" && \ hash -r From f0f9cfa1cddde01e32f063bff368cf403ec2f43e Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Tue, 24 Jun 2025 12:33:57 +0900 Subject: [PATCH 32/37] fix shellcheck for swiftly --- docker/install_swiftly.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/install_swiftly.sh b/docker/install_swiftly.sh index 1284b047..4f4a458b 100755 --- a/docker/install_swiftly.sh +++ b/docker/install_swiftly.sh @@ -21,7 +21,8 @@ apt-get install -y curl curl -O "https://download.swift.org/swiftly/linux/swiftly-$(uname -m).tar.gz" tar zxf "swiftly-$(uname -m).tar.gz" ./swiftly init --assume-yes -# disable=SC1091 # we're ok to not validate the swiftly script +# we're ok to not validate the swiftly script: +# disable=SC1091 . "${SWIFTLY_HOME_DIR:-$HOME/.local/share/swiftly}/env.sh" && \ hash -r From a9471934539ed28b4a31075cacfc4ce9ec261643 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Tue, 24 Jun 2025 12:34:04 +0900 Subject: [PATCH 33/37] jextract: also emit empty files when --write-empty-files is passed This is important for swiftpm build plugins and the plugin will fail if we don't emit those empty files --- .../JExtractSwiftPlugin.swift | 20 ++--- ...ift2JavaGenerator+SwiftThunkPrinting.swift | 21 +++++- .../FFM/FFMSwift2JavaGenerator.swift | 24 ++++++ Sources/JExtractSwiftLib/Swift2Java.swift | 4 +- .../Swift2JavaTranslator.swift | 11 ++- .../Configuration.swift | 2 + .../Commands/JExtractCommand.swift | 6 +- .../swiftkit/SwiftRuntimeMetadataTest.java | 74 +++++++++---------- 8 files changed, 105 insertions(+), 57 deletions(-) diff --git a/Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift b/Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift index 09c84474..7bfb51cf 100644 --- a/Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift +++ b/Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift @@ -63,6 +63,9 @@ struct JExtractSwiftBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { "--input-swift", sourceDir, "--output-java", outputJavaDirectory.path(percentEncoded: false), "--output-swift", outputSwiftDirectory.path(percentEncoded: false), + // since SwiftPM requires all "expected" files do end up being written + // and we don't know which files will have actual thunks generated... we force jextract to write even empty files. + "--write-empty-files", // TODO: "--build-cache-directory", ... // Since plugins cannot depend on libraries we cannot detect what the output files will be, // as it depends on the contents of the input files. Therefore we have to implement this as a prebuild plugin. @@ -76,7 +79,7 @@ struct JExtractSwiftBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { $0.pathExtension == "swift" } - let outputSwiftFiles: [URL] = swiftFiles.compactMap { sourceFileURL in + var outputSwiftFiles: [URL] = swiftFiles.compactMap { sourceFileURL in guard sourceFileURL.isFileURL else { return nil as URL? } @@ -89,17 +92,14 @@ struct JExtractSwiftBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { .appending(path: String(sourceFilePath.dropFirst(sourceDir.count).dropLast(sourceFileURL.lastPathComponent.count + 1))) let inputFileName = sourceFileURL.deletingPathExtension().lastPathComponent - print(" inputFileName = \(inputFileName)") - let isModuleFile = inputFileName.contains("Library") // FIXME: this is a hack - print(" isModuleFile = \(isModuleFile)") - - return if isModuleFile { - outputURL.appending(path: "\(inputFileName)Module+SwiftJava.swift") - } else { - outputURL.appending(path: "\(inputFileName)+SwiftJava.swift") - } + return outputURL.appending(path: "\(inputFileName)+SwiftJava.swift") } + // Append the "module file" that contains any thunks for global func definitions + outputSwiftFiles += [ + outputSwiftDirectory.appending(path: "\(sourceModule.name)Module+SwiftJava.swift") + ] + return [ .buildCommand( displayName: "Generate Java wrappers for Swift types", diff --git a/Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator+SwiftThunkPrinting.swift b/Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator+SwiftThunkPrinting.swift index c1ad19b6..54ce0d72 100644 --- a/Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator+SwiftThunkPrinting.swift +++ b/Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator+SwiftThunkPrinting.swift @@ -21,6 +21,19 @@ extension FFMSwift2JavaGenerator { try writeSwiftThunkSources(printer: &printer) } + package func writeSwiftExpectedEmptySources() throws { + for expectedFileName in self.expectedOutputSwiftFiles { + log.trace("Write empty file: \(expectedFileName) ...") + + var printer = CodePrinter() + printer.print("// Empty file generated on purpose") + try printer.writeContents( + outputDirectory: self.swiftOutputDirectory, + javaPackagePath: nil, + filename: expectedFileName) + } + } + package func writeSwiftThunkSources(printer: inout CodePrinter) throws { let moduleFilenameBase = "\(self.swiftModuleName)Module+SwiftJava" let moduleFilename = "\(moduleFilenameBase).swift" @@ -32,9 +45,9 @@ extension FFMSwift2JavaGenerator { if let outputFile = try printer.writeContents( outputDirectory: self.swiftOutputDirectory, javaPackagePath: nil, - filename: moduleFilename) - { + filename: moduleFilename) { print("[swift-java] Generated: \(moduleFilenameBase.bold).swift (at \(outputFile))") + self.expectedOutputSwiftFiles.remove(moduleFilename) } } catch { log.warning("Failed to write to Swift thunks: \(moduleFilename)") @@ -53,9 +66,9 @@ extension FFMSwift2JavaGenerator { if let outputFile = try printer.writeContents( outputDirectory: self.swiftOutputDirectory, javaPackagePath: nil, - filename: filename) - { + filename: filename) { print("[swift-java] Generated: \(fileNameBase.bold).swift (at \(outputFile))") + self.expectedOutputSwiftFiles.remove(filename) } } catch { log.warning("Failed to write to Swift thunks: \(filename)") diff --git a/Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator.swift b/Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator.swift index 4a65f7cc..09fcf858 100644 --- a/Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator.swift +++ b/Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator.swift @@ -15,6 +15,7 @@ import JavaTypes import SwiftSyntax import SwiftSyntaxBuilder +import struct Foundation.URL package class FFMSwift2JavaGenerator: Swift2JavaGenerator { let log: Logger @@ -35,6 +36,9 @@ package class FFMSwift2JavaGenerator: Swift2JavaGenerator { /// Cached Java translation result. 'nil' indicates failed translation. var translatedDecls: [ImportedFunc: TranslatedFunctionDecl?] = [:] + /// Because we need to write empty files for SwiftPM, keep track which files we didn't write yet, + /// and write an empty file for those. + var expectedOutputSwiftFiles: Set package init( translator: Swift2JavaTranslator, @@ -50,6 +54,20 @@ package class FFMSwift2JavaGenerator: Swift2JavaGenerator { self.javaOutputDirectory = javaOutputDirectory self.symbolTable = translator.symbolTable self.swiftStdlibTypes = translator.swiftStdlibTypeDecls + + // If we are forced to write empty files, construct the expected outputs + if translator.config.writeEmptyFiles ?? false { + self.expectedOutputSwiftFiles = Set(translator.inputs.compactMap { (input) -> String? in + guard let filePathPart = input.filePath.split(separator: "/\(translator.swiftModuleName)/").last else { + return nil + } + + return String(filePathPart.replacing(".swift", with: "+SwiftJava.swift")) + }) + self.expectedOutputSwiftFiles.insert("\(translator.swiftModuleName)Module+SwiftJava.swift") + } else { + self.expectedOutputSwiftFiles = [] + } } func generate() throws { @@ -58,6 +76,12 @@ package class FFMSwift2JavaGenerator: Swift2JavaGenerator { try writeExportedJavaSources() print("[swift-java] Generated Java sources (package: '\(javaPackage)') in: \(javaOutputDirectory)/") + + let pendingFileCount = self.expectedOutputSwiftFiles.count + if pendingFileCount > 0 { + print("[swift-java] Write empty [\(pendingFileCount)] 'expected' files in: \(swiftOutputDirectory)/") + try writeSwiftExpectedEmptySources() + } } } diff --git a/Sources/JExtractSwiftLib/Swift2Java.swift b/Sources/JExtractSwiftLib/Swift2Java.swift index ac746c1d..dad25299 100644 --- a/Sources/JExtractSwiftLib/Swift2Java.swift +++ b/Sources/JExtractSwiftLib/Swift2Java.swift @@ -30,9 +30,7 @@ public struct SwiftToJava { fatalError("Missing '--swift-module' name.") } - let translator = Swift2JavaTranslator( - swiftModuleName: swiftModule - ) + let translator = Swift2JavaTranslator(config: config) translator.log.logLevel = config.logLevel ?? .info if config.javaPackage == nil || config.javaPackage!.isEmpty { diff --git a/Sources/JExtractSwiftLib/Swift2JavaTranslator.swift b/Sources/JExtractSwiftLib/Swift2JavaTranslator.swift index 857a053a..07d8cafb 100644 --- a/Sources/JExtractSwiftLib/Swift2JavaTranslator.swift +++ b/Sources/JExtractSwiftLib/Swift2JavaTranslator.swift @@ -16,6 +16,7 @@ import Foundation import JavaTypes import SwiftBasicFormat import SwiftParser +import JavaKitConfigurationShared import SwiftSyntax /// Takes swift interfaces and translates them into Java used to access those. @@ -24,6 +25,8 @@ public final class Swift2JavaTranslator { package var log = Logger(label: "translator", logLevel: .info) + let config: Configuration + // ==== Input struct Input { @@ -53,9 +56,13 @@ public final class Swift2JavaTranslator { } public init( - swiftModuleName: String + config: Configuration ) { - self.symbolTable = SwiftSymbolTable(parsedModuleName: swiftModuleName) + guard let swiftModule = config.swiftModule else { + fatalError("Missing 'swiftModule' name.") // FIXME: can we make it required in config? but we shared config for many cases + } + self.config = config + self.symbolTable = SwiftSymbolTable(parsedModuleName: swiftModule) // Create a mock of the Swift standard library. var parsedSwiftModule = SwiftParsedModuleSymbolTable(moduleName: "Swift") diff --git a/Sources/JavaKitConfigurationShared/Configuration.swift b/Sources/JavaKitConfigurationShared/Configuration.swift index 26b01702..876b577b 100644 --- a/Sources/JavaKitConfigurationShared/Configuration.swift +++ b/Sources/JavaKitConfigurationShared/Configuration.swift @@ -40,6 +40,8 @@ public struct Configuration: Codable { public var mode: JExtractGenerationMode? + public var writeEmptyFiles: Bool? // FIXME: default it to false, but that plays not nice with Codable + // ==== java 2 swift --------------------------------------------------------- /// The Java class path that should be passed along to the swift-java tool. diff --git a/Sources/SwiftJavaTool/Commands/JExtractCommand.swift b/Sources/SwiftJavaTool/Commands/JExtractCommand.swift index da845eb3..c7fe51a8 100644 --- a/Sources/SwiftJavaTool/Commands/JExtractCommand.swift +++ b/Sources/SwiftJavaTool/Commands/JExtractCommand.swift @@ -57,6 +57,9 @@ extension SwiftJava { @Option(help: "The directory where generated Java files should be written. Generally used with jextract mode.") var outputJava: String + + @Flag(help: "Some build systems require an output to be present when it was 'expected', even if empty. This is used by the JExtractSwiftPlugin build plugin, but otherwise should not be necessary.") + var writeEmptyFiles: Bool = false } } @@ -68,6 +71,7 @@ extension SwiftJava.JExtractCommand { config.swiftModule = self.effectiveSwiftModule config.outputJavaDirectory = outputJava config.outputSwiftDirectory = outputSwift + config.writeEmptyFiles = writeEmptyFiles if let inputSwift = commonOptions.inputSwift { config.inputSwiftDirectory = inputSwift @@ -76,7 +80,7 @@ extension SwiftJava.JExtractCommand { config.inputSwiftDirectory = "\(FileManager.default.currentDirectoryPath)/Sources/\(swiftModule)" } - print("[debug][swift-java] Running swift-java in mode: " + "\(self.mode)".bold) + print("[debug][swift-java] Running 'swift-java jextract' in mode: " + "\(self.mode)".bold) try jextractSwift(config: config) } diff --git a/SwiftKit/src/test/java/org/swift/swiftkit/SwiftRuntimeMetadataTest.java b/SwiftKit/src/test/java/org/swift/swiftkit/SwiftRuntimeMetadataTest.java index ffefe72e..3419405e 100644 --- a/SwiftKit/src/test/java/org/swift/swiftkit/SwiftRuntimeMetadataTest.java +++ b/SwiftKit/src/test/java/org/swift/swiftkit/SwiftRuntimeMetadataTest.java @@ -20,42 +20,42 @@ public class SwiftRuntimeMetadataTest { - @Test - public void integer_layout_metadata() { - SwiftAnyType swiftType = SwiftKit.getTypeByMangledNameInEnvironment("Si").get(); - - if (SwiftValueLayout.addressByteSize() == 4) { - // 32-bit platform - Assertions.assertEquals(8, SwiftValueWitnessTable.sizeOfSwiftType(swiftType.$memorySegment())); - Assertions.assertEquals(8, SwiftValueWitnessTable.strideOfSwiftType(swiftType.$memorySegment())); - Assertions.assertEquals(8, SwiftValueWitnessTable.alignmentOfSwiftType(swiftType.$memorySegment())); - Assertions.assertEquals("[8%[9:b1]x7](Swift.Int)", SwiftValueWitnessTable.layoutOfSwiftType(swiftType.$memorySegment()).toString()); - } else { - // 64-bit platform - Assertions.assertEquals(8, SwiftValueWitnessTable.sizeOfSwiftType(swiftType.$memorySegment())); - Assertions.assertEquals(8, SwiftValueWitnessTable.strideOfSwiftType(swiftType.$memorySegment())); - Assertions.assertEquals(8, SwiftValueWitnessTable.alignmentOfSwiftType(swiftType.$memorySegment())); - Assertions.assertEquals("[8%[8:b1]](Swift.Int)", SwiftValueWitnessTable.layoutOfSwiftType(swiftType.$memorySegment()).toString()); - } - } - - @Test - public void optional_integer_layout_metadata() { - SwiftAnyType swiftType = SwiftKit.getTypeByMangledNameInEnvironment("SiSg").get(); - - if (SwiftValueLayout.addressByteSize() == 4) { - // 64-bit platform - Assertions.assertEquals(9, SwiftValueWitnessTable.sizeOfSwiftType(swiftType.$memorySegment())); - Assertions.assertEquals(16, SwiftValueWitnessTable.strideOfSwiftType(swiftType.$memorySegment())); - Assertions.assertEquals(8, SwiftValueWitnessTable.alignmentOfSwiftType(swiftType.$memorySegment())); - Assertions.assertEquals("[8%[9:b1]x7](Swift.Optional)", SwiftValueWitnessTable.layoutOfSwiftType(swiftType.$memorySegment()).toString()); - } else { - // 64-bit platform - Assertions.assertEquals(9, SwiftValueWitnessTable.sizeOfSwiftType(swiftType.$memorySegment())); - Assertions.assertEquals(16, SwiftValueWitnessTable.strideOfSwiftType(swiftType.$memorySegment())); - Assertions.assertEquals(8, SwiftValueWitnessTable.alignmentOfSwiftType(swiftType.$memorySegment())); - Assertions.assertEquals("[8%[9:b1]x7](Swift.Optional)", SwiftValueWitnessTable.layoutOfSwiftType(swiftType.$memorySegment()).toString()); - } - } +// @Test +// public void integer_layout_metadata() { +// SwiftAnyType swiftType = SwiftKit.getTypeByMangledNameInEnvironment("Si").get(); +// +// if (SwiftValueLayout.addressByteSize() == 4) { +// // 32-bit platform +// Assertions.assertEquals(8, SwiftValueWitnessTable.sizeOfSwiftType(swiftType.$memorySegment())); +// Assertions.assertEquals(8, SwiftValueWitnessTable.strideOfSwiftType(swiftType.$memorySegment())); +// Assertions.assertEquals(8, SwiftValueWitnessTable.alignmentOfSwiftType(swiftType.$memorySegment())); +// Assertions.assertEquals("[8%[9:b1]x7](Swift.Int)", SwiftValueWitnessTable.layoutOfSwiftType(swiftType.$memorySegment()).toString()); +// } else { +// // 64-bit platform +// Assertions.assertEquals(8, SwiftValueWitnessTable.sizeOfSwiftType(swiftType.$memorySegment())); +// Assertions.assertEquals(8, SwiftValueWitnessTable.strideOfSwiftType(swiftType.$memorySegment())); +// Assertions.assertEquals(8, SwiftValueWitnessTable.alignmentOfSwiftType(swiftType.$memorySegment())); +// Assertions.assertEquals("[8%[8:b1]](Swift.Int)", SwiftValueWitnessTable.layoutOfSwiftType(swiftType.$memorySegment()).toString()); +// } +// } +// +// @Test +// public void optional_integer_layout_metadata() { +// SwiftAnyType swiftType = SwiftKit.getTypeByMangledNameInEnvironment("SiSg").get(); +// +// if (SwiftValueLayout.addressByteSize() == 4) { +// // 64-bit platform +// Assertions.assertEquals(9, SwiftValueWitnessTable.sizeOfSwiftType(swiftType.$memorySegment())); +// Assertions.assertEquals(16, SwiftValueWitnessTable.strideOfSwiftType(swiftType.$memorySegment())); +// Assertions.assertEquals(8, SwiftValueWitnessTable.alignmentOfSwiftType(swiftType.$memorySegment())); +// Assertions.assertEquals("[8%[9:b1]x7](Swift.Optional)", SwiftValueWitnessTable.layoutOfSwiftType(swiftType.$memorySegment()).toString()); +// } else { +// // 64-bit platform +// Assertions.assertEquals(9, SwiftValueWitnessTable.sizeOfSwiftType(swiftType.$memorySegment())); +// Assertions.assertEquals(16, SwiftValueWitnessTable.strideOfSwiftType(swiftType.$memorySegment())); +// Assertions.assertEquals(8, SwiftValueWitnessTable.alignmentOfSwiftType(swiftType.$memorySegment())); +// Assertions.assertEquals("[8%[9:b1]x7](Swift.Optional)", SwiftValueWitnessTable.layoutOfSwiftType(swiftType.$memorySegment()).toString()); +// } +// } } From 59aef187691d7bfe3eff48c0f842a3ec90debd1a Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Tue, 24 Jun 2025 13:37:42 +0900 Subject: [PATCH 34/37] fix: seems these workaround decls not necessary on 6.1 linux anymore? --- .../MySwiftLibrary/MySwiftLibrary.swift | 53 ------------------- 1 file changed, 53 deletions(-) diff --git a/Samples/SwiftAndJavaJarSampleLib/Sources/MySwiftLibrary/MySwiftLibrary.swift b/Samples/SwiftAndJavaJarSampleLib/Sources/MySwiftLibrary/MySwiftLibrary.swift index 39688f08..e900fdd0 100644 --- a/Samples/SwiftAndJavaJarSampleLib/Sources/MySwiftLibrary/MySwiftLibrary.swift +++ b/Samples/SwiftAndJavaJarSampleLib/Sources/MySwiftLibrary/MySwiftLibrary.swift @@ -38,51 +38,6 @@ public func globalTakeIntInt(i: Int, j: Int) { public func globalCallMeRunnable(run: () -> ()) { run() } -// -//public class MySwiftClass { -// -// public var len: Int -// public var cap: Int -// -// public init(len: Int, cap: Int) { -// self.len = len -// self.cap = cap -// -// p("\(MySwiftClass.self).len = \(self.len)") -// p("\(MySwiftClass.self).cap = \(self.cap)") -// let addr = unsafeBitCast(self, to: UInt64.self) -// p("initializer done, self = 0x\(String(addr, radix: 16, uppercase: true))") -// } -// -// deinit { -// let addr = unsafeBitCast(self, to: UInt64.self) -// p("Deinit, self = 0x\(String(addr, radix: 16, uppercase: true))") -// } -// -// public var counter: Int32 = 0 -// -// public func voidMethod() { -// p("") -// } -// -// public func takeIntMethod(i: Int) { -// p("i:\(i)") -// } -// -// public func echoIntMethod(i: Int) -> Int { -// p("i:\(i)") -// return i -// } -// -// public func makeIntMethod() -> Int { -// p("make int -> 12") -// return 12 -// } -// -// public func makeRandomIntMethod() -> Int { -// return Int.random(in: 1..<256) -// } -//} // ==== Internal helpers @@ -91,11 +46,3 @@ func p(_ msg: String, file: String = #fileID, line: UInt = #line, function: Stri fflush(stdout) } -#if os(Linux) -// FIXME: why do we need this workaround? -@_silgen_name("_objc_autoreleaseReturnValue") -public func _objc_autoreleaseReturnValue(a: Any) {} - -@_silgen_name("objc_autoreleaseReturnValue") -public func objc_autoreleaseReturnValue(a: Any) {} -#endif From f2a195fb82f2bd97a99596e80aac1eb7a504fd7c Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Tue, 24 Jun 2025 14:01:49 +0900 Subject: [PATCH 35/37] fix: adjust tests to take config, not just module name --- .../Asserts/LoweringAssertions.swift | 13 +++-- .../Asserts/TextAssertions.swift | 4 +- .../FuncCallbackImportTests.swift | 13 +++-- .../FunctionDescriptorImportTests.swift | 13 +++-- .../MethodImportTests.swift | 56 ++++++++++--------- 5 files changed, 53 insertions(+), 46 deletions(-) diff --git a/Tests/JExtractSwiftTests/Asserts/LoweringAssertions.swift b/Tests/JExtractSwiftTests/Asserts/LoweringAssertions.swift index f017bc49..05323630 100644 --- a/Tests/JExtractSwiftTests/Asserts/LoweringAssertions.swift +++ b/Tests/JExtractSwiftTests/Asserts/LoweringAssertions.swift @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// @_spi(Testing) import JExtractSwiftLib +import JavaKitConfigurationShared import SwiftSyntax import Testing @@ -31,9 +32,9 @@ func assertLoweredFunction( line: Int = #line, column: Int = #column ) throws { - let translator = Swift2JavaTranslator( - swiftModuleName: swiftModuleName - ) + var config = Configuration() + config.swiftModule = swiftModuleName + let translator = Swift2JavaTranslator(config: config) if let sourceFile { translator.add(filePath: "Fake.swift", text: sourceFile) @@ -117,9 +118,9 @@ func assertLoweredVariableAccessor( line: Int = #line, column: Int = #column ) throws { - let translator = Swift2JavaTranslator( - swiftModuleName: swiftModuleName - ) + var config = Configuration() + config.swiftModule = swiftModuleName + let translator = Swift2JavaTranslator(config: config) if let sourceFile { translator.add(filePath: "Fake.swift", text: sourceFile) diff --git a/Tests/JExtractSwiftTests/Asserts/TextAssertions.swift b/Tests/JExtractSwiftTests/Asserts/TextAssertions.swift index 61866525..3c4ad56a 100644 --- a/Tests/JExtractSwiftTests/Asserts/TextAssertions.swift +++ b/Tests/JExtractSwiftTests/Asserts/TextAssertions.swift @@ -35,7 +35,9 @@ func assertOutput( line: Int = #line, column: Int = #column ) throws { - let translator = Swift2JavaTranslator(swiftModuleName: swiftModuleName) + var config = Configuration() + config.swiftModule = swiftModuleName + let translator = Swift2JavaTranslator(config: config) try! translator.analyze(file: "/fake/Fake.swiftinterface", text: input) diff --git a/Tests/JExtractSwiftTests/FuncCallbackImportTests.swift b/Tests/JExtractSwiftTests/FuncCallbackImportTests.swift index c738914a..73014a8b 100644 --- a/Tests/JExtractSwiftTests/FuncCallbackImportTests.swift +++ b/Tests/JExtractSwiftTests/FuncCallbackImportTests.swift @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// import JExtractSwiftLib +import JavaKitConfigurationShared import Testing final class FuncCallbackImportTests { @@ -36,9 +37,9 @@ final class FuncCallbackImportTests { @Test("Import: public func callMe(callback: () -> Void)") func func_callMeFunc_callback() throws { - let st = Swift2JavaTranslator( - swiftModuleName: "__FakeModule" - ) + var config = Configuration() + config.swiftModule = "__FakeModule" + let st = Swift2JavaTranslator(config: config) st.log.logLevel = .error try st.analyze(file: "Fake.swift", text: Self.class_interfaceFile) @@ -125,9 +126,9 @@ final class FuncCallbackImportTests { @Test("Import: public func callMeMore(callback: (UnsafeRawPointer, Float) -> Int, fn: () -> ())") func func_callMeMoreFunc_callback() throws { - let st = Swift2JavaTranslator( - swiftModuleName: "__FakeModule" - ) + var config = Configuration() + config.swiftModule = "__FakeModule" + let st = Swift2JavaTranslator(config: config) st.log.logLevel = .error try st.analyze(file: "Fake.swift", text: Self.class_interfaceFile) diff --git a/Tests/JExtractSwiftTests/FunctionDescriptorImportTests.swift b/Tests/JExtractSwiftTests/FunctionDescriptorImportTests.swift index 2cac6218..7af3c706 100644 --- a/Tests/JExtractSwiftTests/FunctionDescriptorImportTests.swift +++ b/Tests/JExtractSwiftTests/FunctionDescriptorImportTests.swift @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// import JExtractSwiftLib +import JavaKitConfigurationShared import Testing @Suite @@ -233,9 +234,9 @@ extension FunctionDescriptorTests { logLevel: Logger.Level = .trace, body: (String) throws -> Void ) throws { - let st = Swift2JavaTranslator( - swiftModuleName: swiftModuleName - ) + var config = Configuration() + config.swiftModule = swiftModuleName + let st = Swift2JavaTranslator(config: config) st.log.logLevel = logLevel try st.analyze(file: "/fake/Sample.swiftinterface", text: interfaceFile) @@ -266,9 +267,9 @@ extension FunctionDescriptorTests { logLevel: Logger.Level = .trace, body: (String) throws -> Void ) throws { - let st = Swift2JavaTranslator( - swiftModuleName: swiftModuleName - ) + var config = Configuration() + config.swiftModule = swiftModuleName + let st = Swift2JavaTranslator(config: config) st.log.logLevel = logLevel try st.analyze(file: "/fake/Sample.swiftinterface", text: interfaceFile) diff --git a/Tests/JExtractSwiftTests/MethodImportTests.swift b/Tests/JExtractSwiftTests/MethodImportTests.swift index 2405e0ac..36610dcd 100644 --- a/Tests/JExtractSwiftTests/MethodImportTests.swift +++ b/Tests/JExtractSwiftTests/MethodImportTests.swift @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// import JExtractSwiftLib +import JavaKitConfigurationShared import Testing final class MethodImportTests { @@ -64,9 +65,9 @@ final class MethodImportTests { @Test("Import: public func helloWorld()") func method_helloWorld() throws { - let st = Swift2JavaTranslator( - swiftModuleName: "__FakeModule" - ) + var config = Configuration() + config.swiftModule = "__FakeModule" + let st = Swift2JavaTranslator(config: config) st.log.logLevel = .error try st.analyze(file: "Fake.swift", text: class_interfaceFile) @@ -103,9 +104,9 @@ final class MethodImportTests { @Test("Import: public func globalTakeInt(i: Int)") func func_globalTakeInt() throws { - let st = Swift2JavaTranslator( - swiftModuleName: "__FakeModule" - ) + var config = Configuration() + config.swiftModule = "__FakeModule" + let st = Swift2JavaTranslator(config: config) st.log.logLevel = .error try st.analyze(file: "Fake.swift", text: class_interfaceFile) @@ -144,9 +145,9 @@ final class MethodImportTests { @Test("Import: public func globalTakeIntLongString(i32: Int32, l: Int64, s: String)") func func_globalTakeIntLongString() throws { - let st = Swift2JavaTranslator( - swiftModuleName: "__FakeModule" - ) + var config = Configuration() + config.swiftModule = "__FakeModule" + let st = Swift2JavaTranslator(config: config) st.log.logLevel = .error try st.analyze(file: "Fake.swift", text: class_interfaceFile) @@ -188,9 +189,9 @@ final class MethodImportTests { @Test("Import: public func globalReturnClass() -> MySwiftClass") func func_globalReturnClass() throws { - let st = Swift2JavaTranslator( - swiftModuleName: "__FakeModule" - ) + var config = Configuration() + config.swiftModule = "__FakeModule" + let st = Swift2JavaTranslator(config: config) st.log.logLevel = .error try st.analyze(file: "Fake.swift", text: class_interfaceFile) @@ -232,9 +233,9 @@ final class MethodImportTests { @Test("Import: func swapRawBufferPointer(buffer: _)") func func_globalSwapRawBufferPointer() throws { - let st = Swift2JavaTranslator( - swiftModuleName: "__FakeModule" - ) + var config = Configuration() + config.swiftModule = "__FakeModule" + let st = Swift2JavaTranslator(config: config) st.log.logLevel = .error try st.analyze(file: "Fake.swift", text: class_interfaceFile) @@ -279,9 +280,9 @@ final class MethodImportTests { @Test func method_class_helloMemberFunction() throws { - let st = Swift2JavaTranslator( - swiftModuleName: "__FakeModule" - ) + var config = Configuration() + config.swiftModule = "__FakeModule" + let st = Swift2JavaTranslator(config: config) st.log.logLevel = .error try st.analyze(file: "Fake.swift", text: class_interfaceFile) @@ -321,9 +322,9 @@ final class MethodImportTests { @Test func method_class_makeInt() throws { - let st = Swift2JavaTranslator( - swiftModuleName: "__FakeModule" - ) + var config = Configuration() + config.swiftModule = "__FakeModule" + let st = Swift2JavaTranslator(config: config) st.log.logLevel = .info try st.analyze(file: "Fake.swift", text: class_interfaceFile) @@ -363,9 +364,9 @@ final class MethodImportTests { @Test func class_constructor() throws { - let st = Swift2JavaTranslator( - swiftModuleName: "__FakeModule" - ) + var config = Configuration() + config.swiftModule = "__FakeModule" + let st = Swift2JavaTranslator(config: config) st.log.logLevel = .info try st.analyze(file: "Fake.swift", text: class_interfaceFile) @@ -406,9 +407,10 @@ final class MethodImportTests { @Test func struct_constructor() throws { - let st = Swift2JavaTranslator( - swiftModuleName: "__FakeModule" - ) + var config = Configuration() + config.swiftModule = "__FakeModule" + let st = Swift2JavaTranslator(config: config) + st.log.logLevel = .info try st.analyze(file: "Fake.swift", text: class_interfaceFile) From 8794b666dcbfad2fea4fc1ddf228199297dc87d5 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Tue, 24 Jun 2025 14:17:58 +0900 Subject: [PATCH 36/37] remove runtime workarounds, seems not needed on linux anymore with 6.1 --- .../MySwiftLibrary/_RuntimeWorkarounds.swift | 22 ------------------- 1 file changed, 22 deletions(-) delete mode 100644 Samples/SwiftKitSampleApp/Sources/MySwiftLibrary/_RuntimeWorkarounds.swift diff --git a/Samples/SwiftKitSampleApp/Sources/MySwiftLibrary/_RuntimeWorkarounds.swift b/Samples/SwiftKitSampleApp/Sources/MySwiftLibrary/_RuntimeWorkarounds.swift deleted file mode 100644 index f3056973..00000000 --- a/Samples/SwiftKitSampleApp/Sources/MySwiftLibrary/_RuntimeWorkarounds.swift +++ /dev/null @@ -1,22 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift.org project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift.org project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -#if os(Linux) -// FIXME: why do we need this workaround? -@_silgen_name("_objc_autoreleaseReturnValue") -public func _objc_autoreleaseReturnValue(a: Any) {} - -@_silgen_name("objc_autoreleaseReturnValue") -public func objc_autoreleaseReturnValue(a: Any) {} -#endif From d44b7623e3f16ea4a0d3c4a0cb6050e314b1b8d4 Mon Sep 17 00:00:00 2001 From: Konrad 'ktoso' Malawski Date: Tue, 24 Jun 2025 15:56:31 +0900 Subject: [PATCH 37/37] we don't use swiftly in docker, remove the script --- .../FuncCallbackImportTests.swift | 6 ++-- docker/Dockerfile | 4 --- docker/install_swiftly.sh | 30 ------------------- 3 files changed, 3 insertions(+), 37 deletions(-) delete mode 100755 docker/install_swiftly.sh diff --git a/Tests/JExtractSwiftTests/FuncCallbackImportTests.swift b/Tests/JExtractSwiftTests/FuncCallbackImportTests.swift index 73014a8b..d81dad8c 100644 --- a/Tests/JExtractSwiftTests/FuncCallbackImportTests.swift +++ b/Tests/JExtractSwiftTests/FuncCallbackImportTests.swift @@ -241,9 +241,9 @@ final class FuncCallbackImportTests { @Test("Import: public func withBuffer(body: (UnsafeRawBufferPointer) -> Int)") func func_withBuffer_body() throws { - let st = Swift2JavaTranslator( - swiftModuleName: "__FakeModule" - ) + var config = Configuration() + config.swiftModule = "__FakeModule" + let st = Swift2JavaTranslator(config: config) st.log.logLevel = .error try st.analyze(file: "Fake.swift", text: Self.class_interfaceFile) diff --git a/docker/Dockerfile b/docker/Dockerfile index d5599841..6604d091 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,12 +19,8 @@ ENV LANG=en_US.UTF-8 ENV LANGUAGE=en_US.UTF-8 COPY install_jdk.sh . -COPY install_swiftly.sh . # JDK dependency RUN bash -xc 'JDK_VENDOR=Corretto ./install_jdk.sh' ENV JAVA_HOME="/usr/lib/jvm/default-jdk" ENV PATH="$PATH:/usr/lib/jvm/default-jdk/bin" - -# Install Swift with Swiftly -# RUN bash -xc './install_swiftly.sh' diff --git a/docker/install_swiftly.sh b/docker/install_swiftly.sh deleted file mode 100755 index 4f4a458b..00000000 --- a/docker/install_swiftly.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the Swift.org open source project -## -## Copyright (c) 2024 Apple Inc. and the Swift.org project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of Swift.org project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## -set -euo pipefail - -echo "Install Swiftly and toolchain for: $(uname -m)" - -apt-get install -y curl - -curl -O "https://download.swift.org/swiftly/linux/swiftly-$(uname -m).tar.gz" -tar zxf "swiftly-$(uname -m).tar.gz" -./swiftly init --assume-yes -# we're ok to not validate the swiftly script: -# disable=SC1091 -. "${SWIFTLY_HOME_DIR:-$HOME/.local/share/swiftly}/env.sh" && \ -hash -r - -swiftly install 6.1.2 --assume-yes --use -swift -version