diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index d289296d..2aaa845b 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -60,7 +60,7 @@ jobs: - name: Prepare CI Environment uses: ./.github/actions/prepare_env - name: Swift Build - run: "swift build --build-tests" + run: "swift build --build-tests --disable-sandbox" - name: Swift Test run: "swift test" diff --git a/.licenseignore b/.licenseignore index 5f93c9a6..8b9d0933 100644 --- a/.licenseignore +++ b/.licenseignore @@ -14,6 +14,7 @@ Package.resolved README.md SECURITY.md scripts/unacceptable-language.txt +.unacceptablelanguageignore docker/* **/*.docc/* **/.gitignore @@ -43,4 +44,5 @@ gradlew.bat **/DO_NOT_EDIT.txt Plugins/**/_PluginsShared Plugins/**/0_PLEASE_SYMLINK* -Plugins/PluginsShared/JavaKitConfigurationShared \ No newline at end of file +Plugins/PluginsShared/JavaKitConfigurationShared +Sources/_Subprocess/_nio_locks.swift \ No newline at end of file diff --git a/.unacceptablelanguageignore b/.unacceptablelanguageignore new file mode 100644 index 00000000..0310d1c3 --- /dev/null +++ b/.unacceptablelanguageignore @@ -0,0 +1,17 @@ +Sources/SwiftJavaBootstrapJavaTool/SwiftJavaBootstrapJavaTool.swift +Sources/_Subprocess/Platforms/Subprocess+Darwin.swift +Sources/_Subprocess/Platforms/Subprocess+Linux.swift +Sources/_Subprocess/Platforms/Subprocess+Unix.swift +Sources/_Subprocess/Platforms/Subprocess+Unix.swift +Sources/_Subprocess/Platforms/Subprocess+Unix.swift +Sources/_Subprocess/Platforms/Subprocess+Unix.swift +Sources/_Subprocess/Platforms/Subprocess+Unix.swift +Sources/_Subprocess/Platforms/Subprocess+Unix.swift +Sources/_Subprocess/Subprocess+Teardown.swift +Sources/_Subprocess/Subprocess+Teardown.swift +Sources/_Subprocess/Subprocess+Teardown.swift +Sources/_Subprocess/Subprocess+Teardown.swift +Sources/_Subprocess/Subprocess+Teardown.swift +Sources/_Subprocess/Subprocess+Teardown.swift +Sources/_Subprocess/Subprocess+Teardown.swift +Sources/_Subprocess/Subprocess.swift \ No newline at end of file diff --git a/JavaKit/build.gradle b/JavaKit/build.gradle index 8eed1c6e..70982e90 100644 --- a/JavaKit/build.gradle +++ b/JavaKit/build.gradle @@ -60,18 +60,13 @@ tasks.processResources { } } -//task fatJar(type: Jar) { -// archiveBaseName = 'java-kit-fat-jar' -// duplicatesStrategy = DuplicatesStrategy.EXCLUDE -// from { configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } } -// with jar -//} +// Task necessary to bootstrap and EXIT, this is to prevent hangs when used via SwiftPM plugin. +tasks.register('printRuntimeClasspath') { + dependsOn 'jar' -// Task necessary to bootstrap -task printRuntimeClasspath { - def runtimeClasspath = sourceSets.main.runtimeClasspath - inputs.files(runtimeClasspath) - doLast { - println("CLASSPATH:${runtimeClasspath.asPath}") - } + def runtimeClasspath = sourceSets.main.runtimeClasspath + inputs.files(runtimeClasspath) + doLast { + println("SWIFT_JAVA_CLASSPATH:${runtimeClasspath.asPath}") + } } diff --git a/JavaKit/src/main/java/org/swift/javakit/dependencies/DependencyResolver.java b/JavaKit/src/main/java/org/swift/javakit/dependencies/DependencyResolver.java index ff4085c9..0e801525 100644 --- a/JavaKit/src/main/java/org/swift/javakit/dependencies/DependencyResolver.java +++ b/JavaKit/src/main/java/org/swift/javakit/dependencies/DependencyResolver.java @@ -34,7 +34,7 @@ @SuppressWarnings("unused") public class DependencyResolver { - private static final String COMMAND_OUTPUT_LINE_PREFIX_CLASSPATH = "CLASSPATH:"; + private static final String COMMAND_OUTPUT_LINE_PREFIX_CLASSPATH = "SWIFT_JAVA_CLASSPATH:"; private static final String CLASSPATH_CACHE_FILENAME = "JavaKitDependencyResolver.swift-java.classpath"; public static String GRADLE_API_DEPENDENCY = "dev.gradleplugins:gradle-api:8.10.1"; @@ -236,11 +236,11 @@ private static void printBuildFiles(File projectDir, String[] dependencies) thro writer.println("}"); writer.println(""" - task printRuntimeClasspath { + tasks.register("printRuntimeClasspath") { def runtimeClasspath = sourceSets.main.runtimeClasspath inputs.files(runtimeClasspath) doLast { - println("CLASSPATH:${runtimeClasspath.asPath}") + println("SWIFT_JAVA_CLASSPATH:${runtimeClasspath.asPath}") } } """); diff --git a/Package.swift b/Package.swift index f6645115..8cda1aed 100644 --- a/Package.swift +++ b/Package.swift @@ -139,7 +139,7 @@ let package = Package( "JExtractSwiftCommandPlugin" ] ), - + // ==== Examples .library( @@ -152,6 +152,9 @@ let package = Package( dependencies: [ .package(url: "https://github.com/swiftlang/swift-syntax", from: "600.0.1"), .package(url: "https://github.com/apple/swift-argument-parser", from: "1.5.0"), + .package(url: "https://github.com/apple/swift-system", from: "1.4.0"), + + // Benchmarking .package(url: "https://github.com/ordo-one/package-benchmark", .upToNextMajor(from: "1.4.0")), ], targets: [ @@ -184,6 +187,11 @@ let package = Package( .swiftLanguageMode(.v5), .unsafeFlags(["-I\(javaIncludePath)", "-I\(javaPlatformIncludePath)"]), ] +// // FIXME: when the tool is run from plugin it hangs even if sandbox is disabled +// , +// plugins: [ +// "SwiftJavaBootstrapJavaPlugin", +// ] ), .target( @@ -211,7 +219,7 @@ let package = Package( .unsafeFlags( [ "-L\(javaHome)/lib" - ], + ], .when(platforms: [.windows])), .linkedLibrary("jvm"), ] @@ -378,6 +386,25 @@ let package = Package( ] ), + .executableTarget( + name: "SwiftJavaBootstrapJavaTool", + dependencies: [ + "JavaKitConfigurationShared", // for Configuration reading at runtime + "_Subprocess", + ], + swiftSettings: [ + .swiftLanguageMode(.v5) + ] + ), + + .plugin( + name: "SwiftJavaBootstrapJavaPlugin", + capability: .buildTool(), + dependencies: [ + "SwiftJavaBootstrapJavaTool" + ] + ), + .plugin( name: "SwiftJavaPlugin", capability: .buildTool(), @@ -442,6 +469,24 @@ let package = Package( .swiftLanguageMode(.v5), .unsafeFlags(["-I\(javaIncludePath)", "-I\(javaPlatformIncludePath)"]) ] + ), + + // Experimental Foundation Subprocess Copy + .target( + name: "_CShims", + swiftSettings: [ + .swiftLanguageMode(.v5) + ] + ), + .target( + name: "_Subprocess", + dependencies: [ + "_CShims", + .product(name: "SystemPackage", package: "swift-system"), + ], + swiftSettings: [ + .swiftLanguageMode(.v5) + ] ) ] ) diff --git a/Plugins/SwiftJavaBootstrapJavaPlugin/SwiftJavaBootstrapJavaPlugin.swift b/Plugins/SwiftJavaBootstrapJavaPlugin/SwiftJavaBootstrapJavaPlugin.swift new file mode 100644 index 00000000..7d614371 --- /dev/null +++ b/Plugins/SwiftJavaBootstrapJavaPlugin/SwiftJavaBootstrapJavaPlugin.swift @@ -0,0 +1,190 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation +import PackagePlugin + +fileprivate let SwiftJavaConfigFileName = "swift-java.config" + +@main +struct Java2SwiftBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin { + + var pluginName: String = "swift-java-bootstrap" + var verbose: Bool = getEnvironmentBool("SWIFT_JAVA_VERBOSE") + + func createBuildCommands(context: PluginContext, target: Target) throws -> [Command] { + log("Create build commands for target '\(target.name)'") + guard let sourceModule = target.sourceModule else { return [] } + + let executable = try context.tool(named: "SwiftJavaBootstrapJavaTool").url + var commands: [Command] = [] + + // Note: Target doesn't have a directoryURL counterpart to directory, + // so we cannot eliminate this deprecation warning. + let sourceDir = target.directory.string + + // The name of the configuration file JavaKit.config from the target for + // which we are generating Swift wrappers for Java classes. + let configFile = URL(filePath: sourceDir) + .appending(path: SwiftJavaConfigFileName) + let config = try readConfiguration(sourceDir: sourceDir) + + log("Config on path: \(configFile.path(percentEncoded: false))") + log("Config was: \(config)") + var javaDependencies = config.dependencies ?? [] + + /// Find the manifest files from other Java2Swift executions in any targets + /// this target depends on. + var dependentConfigFiles: [(String, URL)] = [] + func searchForConfigFiles(in target: any Target) { + // log("Search for config files in target: \(target.name)") + let dependencyURL = URL(filePath: target.directory.string) + + // Look for a config file within this target. + let dependencyConfigURL = dependencyURL + .appending(path: SwiftJavaConfigFileName) + let dependencyConfigString = dependencyConfigURL + .path(percentEncoded: false) + + if FileManager.default.fileExists(atPath: dependencyConfigString) { + dependentConfigFiles.append((target.name, dependencyConfigURL)) + } + } + + // Process direct dependencies of this target. + for dependency in target.dependencies { + switch dependency { + case .target(let target): + searchForConfigFiles(in: target) + + case .product(let product): + for target in product.targets { + searchForConfigFiles(in: target) + } + + @unknown default: + break + } + } + + // Process indirect target dependencies. + for dependency in target.recursiveTargetDependencies { + searchForConfigFiles(in: dependency) + } + + var arguments: [String] = [] + arguments += argumentsModuleName(sourceModule: sourceModule) + arguments += argumentsOutputDirectory(context: context) + + arguments += dependentConfigFiles.flatMap { moduleAndConfigFile in + let (moduleName, configFile) = moduleAndConfigFile + return [ + "--depends-on", + "\(moduleName)=\(configFile.path(percentEncoded: false))" + ] + } + arguments.append(configFile.path(percentEncoded: false)) + + let classes = config.classes ?? [:] + print("Classes to wrap: \(classes.map(\.key))") + + /// Determine the set of Swift files that will be emitted by the Java2Swift tool. + // TODO: this is not precise and won't work with more advanced Java files, e.g. lambdas etc. + let outputDirectoryGenerated = self.outputDirectory(context: context, generated: true) + let outputSwiftFiles = classes.map { (javaClassName, swiftName) in + let swiftNestedName = swiftName.replacingOccurrences(of: ".", with: "+") + return outputDirectoryGenerated.appending(path: "\(swiftNestedName).swift") + } + + arguments += [ + "--cache-directory", + context.pluginWorkDirectoryURL.path(percentEncoded: false) + ] + + // Find the Java .class files generated from prior plugins. + let compiledClassFiles = sourceModule.pluginGeneratedResources.filter { url in + url.pathExtension == "class" + } + + if let firstClassFile = compiledClassFiles.first { + // Keep stripping off parts of the path until we hit the "Java" part. + // That's where the class path starts. + var classpath = firstClassFile + while classpath.lastPathComponent != "Java" { + classpath.deleteLastPathComponent() + } + arguments += ["--classpath", classpath.path()] + } + + var fetchDependenciesOutputFiles: [URL] = [] + if let dependencies = config.dependencies, !dependencies.isEmpty { + let displayName = "Fetch (Java) dependencies for Swift target \(sourceModule.name)" + log("Prepared: \(displayName)") + + let arguments = [ + "--fetch", configFile.path(percentEncoded: false), + "--module-name", sourceModule.name, + "--output-directory", outputDirectory(context: context, generated: false).path(percentEncoded: false) + ] + + log("Command: \(executable) \(arguments.joined(separator: " "))") + + fetchDependenciesOutputFiles += [ + outputFilePath(context: context, generated: false, filename: "\(sourceModule.name).swift-java.classpath") + ] + + commands += [ + .buildCommand( + displayName: displayName, + executable: executable, + arguments: arguments, + inputFiles: [configFile], + outputFiles: fetchDependenciesOutputFiles + ) + ] + } else { + log("No dependencies to fetch for target \(sourceModule.name)") + } + + return commands + } +} + +extension Java2SwiftBuildToolPlugin { + func argumentsModuleName(sourceModule: Target) -> [String] { + return [ + "--module-name", sourceModule.name + ] + } + + func argumentsOutputDirectory(context: PluginContext, generated: Bool = true) -> [String] { + return [ + "--output-directory", + outputDirectory(context: context, generated: generated).path(percentEncoded: false) + ] + } + + func outputDirectory(context: PluginContext, generated: Bool = true) -> URL { + let dir = context.pluginWorkDirectoryURL + if generated { + return dir.appending(path: "generated") + } else { + return dir + } + } + + func outputFilePath(context: PluginContext, generated: Bool, filename: String) -> URL { + outputDirectory(context: context, generated: generated).appending(path: filename) + } +} diff --git a/Plugins/SwiftJavaBootstrapJavaPlugin/_PluginsShared b/Plugins/SwiftJavaBootstrapJavaPlugin/_PluginsShared new file mode 120000 index 00000000..de623a5e --- /dev/null +++ b/Plugins/SwiftJavaBootstrapJavaPlugin/_PluginsShared @@ -0,0 +1 @@ +../PluginsShared \ No newline at end of file diff --git a/Samples/JavaDependencySampleApp/ci-validate.sh b/Samples/JavaDependencySampleApp/ci-validate.sh index 86f83978..b21d19d4 100755 --- a/Samples/JavaDependencySampleApp/ci-validate.sh +++ b/Samples/JavaDependencySampleApp/ci-validate.sh @@ -3,8 +3,10 @@ set -e set -x -cd ../../JavaKit -./gradlew build +# TODO: this is a workaround for build plugins getting stuck running the bootstrap plugin +cd ../../ +swift build --product SwiftJavaBootstrapJavaTool +.build/debug/SwiftJavaBootstrapJavaTool --fetch Sources/JavaKitDependencyResolver/swift-java.config --module-name JavaKitDependencyResolver --output-directory .build/plugins/outputs/swift-java/JavaKitDependencyResolver/destination/SwiftJavaBootstrapJavaPlugin cd - swift run --disable-sandbox diff --git a/Sources/JavaKitConfigurationShared/Configuration.swift b/Sources/JavaKitConfigurationShared/Configuration.swift index 2078c544..d401c657 100644 --- a/Sources/JavaKitConfigurationShared/Configuration.swift +++ b/Sources/JavaKitConfigurationShared/Configuration.swift @@ -116,7 +116,7 @@ public func readConfiguration(configPath: URL, file: String = #fileID, line: UIn let configData = try Data(contentsOf: configPath) return try JSONDecoder().decode(Configuration.self, from: configData) } catch { - throw ConfigurationError(message: "Failed to parse SwiftJava configuration at '\(configPath)'!", error: error, + throw ConfigurationError(message: "Failed to parse SwiftJava configuration at '\(configPath.absoluteURL)'!", error: error, file: file, line: line) } } diff --git a/Sources/SwiftJavaBootstrapJavaTool/SwiftJavaBootstrapJavaTool.swift b/Sources/SwiftJavaBootstrapJavaTool/SwiftJavaBootstrapJavaTool.swift new file mode 100644 index 00000000..11f26a67 --- /dev/null +++ b/Sources/SwiftJavaBootstrapJavaTool/SwiftJavaBootstrapJavaTool.swift @@ -0,0 +1,137 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation +import Synchronization +import JavaKitConfigurationShared +import Dispatch +import _Subprocess + +@available(macOS 15.0, *) +@main +final class SwiftJavaBootstrapJavaTool { + + let SwiftJavaClasspathPrefix = "SWIFT_JAVA_CLASSPATH:" + + // We seem to have weird "hang" issues with Gradle launched from Process(), workaround it by existing once we get the classpath + let printRuntimeClasspathTaskName = "printRuntimeClasspath" + + let out = Synchronization.Mutex(Data()) + let err = Synchronization.Mutex(Data()) + + static func main() async { + await SwiftJavaBootstrapJavaTool().run() + } + + func run() async { + print("[debug][swift-java-bootstrap] RUN SwiftJavaBootstrapJavaTool: \(CommandLine.arguments.joined(separator: " "))") + + var args = CommandLine.arguments + _ = args.removeFirst() // executable + + assert(args.removeFirst() == "--fetch") + let configPath = args.removeFirst() + + assert(args.removeFirst() == "--module-name") + let moduleName = args.removeFirst() + + assert(args.removeFirst() == "--output-directory") + let outputDirectoryPath = args.removeFirst() + + let configPathURL = URL(fileURLWithPath: configPath) + print("[debug][swift-java-bootstrap] Load config: \(configPathURL.absoluteString)") + let config = try! readConfiguration(configPath: configPathURL) + + // We only support a single dependency right now. + let localGradleProjectDependencyName = (config.dependencies ?? []).filter { + $0.artifactID.hasPrefix(":") + }.map { + $0.artifactID + }.first! + + let process = try! await Subprocess.run( + .at("./gradlew"), + arguments: [ + "--no-daemon", + "--rerun-tasks", + // "--debug", + // "\(localGradleProjectDependencyName):jar", + "\(localGradleProjectDependencyName):\(printRuntimeClasspathTaskName)" + ] + ) + + let outString = String( + data: process.standardOutput, + encoding: .utf8 + ) + let errString = String( + data: process.standardError, + encoding: .utf8 + ) + + print("OUT ==== \(outString?.count) ::: \(outString ?? "")") + print("ERR ==== \(errString?.count) ::: \(errString ?? "")") + + let classpathOutput: String + if let found = outString?.split(separator: "\n").first(where: { $0.hasPrefix(self.SwiftJavaClasspathPrefix) }) { + classpathOutput = String(found) + } else if let found = errString?.split(separator: "\n").first(where: { $0.hasPrefix(self.SwiftJavaClasspathPrefix) }) { + classpathOutput = String(found) + } else { + let suggestDisablingSandbox = "It may be that the Sandbox has prevented dependency fetching, please re-run with '--disable-sandbox'." + fatalError("Gradle output had no SWIFT_JAVA_CLASSPATH! \(suggestDisablingSandbox). \n" + + "Output was:<<<\(outString ?? "")>>>; Err was:<<<\(errString ?? "")>>>") + } + + let classpathString = String(classpathOutput.dropFirst(self.SwiftJavaClasspathPrefix.count)) + + _ = try? FileManager.default.createDirectory( + at: URL(fileURLWithPath: outputDirectoryPath), + withIntermediateDirectories: true, + attributes: nil + ) + + let classpathOutputURL = + URL(fileURLWithPath: outputDirectoryPath) + .appendingPathComponent("\(moduleName).swift-java.classpath", isDirectory: false) + + try! classpathString.write(to: classpathOutputURL, atomically: true, encoding: .utf8) + + print("[swift-java-bootstrap] Done, written classpath to: \(classpathOutputURL)") + } + + func writeBuildGradle(directory: URL) { + // """ + // plugins { id 'java-library' } + // repositories { mavenCentral() } + // + // dependencies { + // implementation("dev.gradleplugins:gradle-api:8.10.1") + // } + // + // task \(printRuntimeClasspathTaskName) { + // def runtimeClasspath = sourceSets.main.runtimeClasspath + // inputs.files(runtimeClasspath) + // doLast { + // println("CLASSPATH:${runtimeClasspath.asPath}") + // } + // } + // """.write(to: URL(fileURLWithPath: tempDir.appendingPathComponent("build.gradle")).path(percentEncoded: false), atomically: true, encoding: .utf8) + // + // """ + // rootProject.name = "swift-java-resolve-temp-project" + // """.write(to: URL(fileURLWithPath: tempDir.appendingPathComponent("settings.gradle.kts")).path(percentEncoded: false), atomically: true, encoding: .utf8) + } + +} diff --git a/Sources/_CShims/include/_CShimsTargetConditionals.h b/Sources/_CShims/include/_CShimsTargetConditionals.h new file mode 100644 index 00000000..9e1d80cb --- /dev/null +++ b/Sources/_CShims/include/_CShimsTargetConditionals.h @@ -0,0 +1,52 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#ifndef _SHIMS_TARGET_CONDITIONALS_H +#define _SHIMS_TARGET_CONDITIONALS_H + +#if __has_include() +#include +#endif + +#if (defined(__APPLE__) && defined(__MACH__)) +#define TARGET_OS_MAC 1 +#else +#define TARGET_OS_MAC 0 +#endif + +#if defined(__linux__) +#define TARGET_OS_LINUX 1 +#else +#define TARGET_OS_LINUX 0 +#endif + +#if defined(__unix__) +#define TARGET_OS_BSD 1 +#else +#define TARGET_OS_BSD 0 +#endif + +#if defined(_WIN32) +#define TARGET_OS_WINDOWS 1 +#else +#define TARGET_OS_WINDOWS 0 +#endif + +#if defined(__wasi__) +#define TARGET_OS_WASI 1 +#else +#define TARGET_OS_WASI 0 +#endif + +#endif // _SHIMS_TARGET_CONDITIONALS_H diff --git a/Sources/_CShims/include/process_shims.h b/Sources/_CShims/include/process_shims.h new file mode 100644 index 00000000..563b517f --- /dev/null +++ b/Sources/_CShims/include/process_shims.h @@ -0,0 +1,75 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#ifndef process_shims_h +#define process_shims_h + +#include "_CShimsTargetConditionals.h" + +#if !TARGET_OS_WINDOWS +#include + +#if _POSIX_SPAWN +#include +#endif + +#if TARGET_OS_MAC +int _subprocess_spawn( + pid_t * _Nonnull pid, + const char * _Nonnull exec_path, + const posix_spawn_file_actions_t _Nullable * _Nonnull file_actions, + const posix_spawnattr_t _Nullable * _Nonnull spawn_attrs, + char * _Nullable const args[_Nonnull], + char * _Nullable const env[_Nullable], + uid_t * _Nullable uid, + gid_t * _Nullable gid, + int number_of_sgroups, const gid_t * _Nullable sgroups, + int create_session +); +#endif // TARGET_OS_MAC + +int _subprocess_fork_exec( + pid_t * _Nonnull pid, + const char * _Nonnull exec_path, + const char * _Nullable working_directory, + const int file_descriptors[_Nonnull], + char * _Nullable const args[_Nonnull], + char * _Nullable const env[_Nullable], + uid_t * _Nullable uid, + gid_t * _Nullable gid, + gid_t * _Nullable process_group_id, + int number_of_sgroups, const gid_t * _Nullable sgroups, + int create_session, + void (* _Nullable configurator)(void) +); + +int _was_process_exited(int status); +int _get_exit_code(int status); +int _was_process_signaled(int status); +int _get_signal_code(int status); +int _was_process_suspended(int status); + +#if TARGET_OS_LINUX +int _shims_snprintf( + char * _Nonnull str, + int len, + const char * _Nonnull format, + char * _Nonnull str1, + char * _Nonnull str2 +); +#endif + +#endif // !TARGET_OS_WINDOWS + +#endif /* process_shims_h */ diff --git a/Sources/_CShims/process_shims.c b/Sources/_CShims/process_shims.c new file mode 100644 index 00000000..fe96c675 --- /dev/null +++ b/Sources/_CShims/process_shims.c @@ -0,0 +1,340 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#include "include/_CShimsTargetConditionals.h" +#include "include/process_shims.h" + +#if !TARGET_OS_WINDOWS +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +int _was_process_exited(int status) { + return WIFEXITED(status); +} + +int _get_exit_code(int status) { + return WEXITSTATUS(status); +} + +int _was_process_signaled(int status) { + return WIFSIGNALED(status); +} + +int _get_signal_code(int status) { + return WTERMSIG(status); +} + +int _was_process_suspended(int status) { + return WIFSTOPPED(status); +} + +#if TARGET_OS_LINUX +#include + +int _shims_snprintf( + char * _Nonnull str, + int len, + const char * _Nonnull format, + char * _Nonnull str1, + char * _Nonnull str2 +) { + return snprintf(str, len, format, str1, str2); +} +#endif + +// MARK: - Darwin (posix_spawn) +#if TARGET_OS_MAC + +int _subprocess_spawn( + pid_t * _Nonnull pid, + const char * _Nonnull exec_path, + const posix_spawn_file_actions_t _Nullable * _Nonnull file_actions, + const posix_spawnattr_t _Nullable * _Nonnull spawn_attrs, + char * _Nullable const args[_Nonnull], + char * _Nullable const env[_Nullable], + uid_t * _Nullable uid, + gid_t * _Nullable gid, + int number_of_sgroups, const gid_t * _Nullable sgroups, + int create_session +) { + int require_pre_fork = uid != NULL || + gid != NULL || + number_of_sgroups > 0 || + create_session > 0; + + if (require_pre_fork != 0) { +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated" + pid_t childPid = vfork(); +#pragma GCC diagnostic pop + if (childPid != 0) { + *pid = childPid; + return childPid < 0 ? errno : 0; + } + + if (number_of_sgroups > 0 && sgroups != NULL) { + if (setgroups(number_of_sgroups, sgroups) != 0) { + return errno; + } + } + + if (uid != NULL) { + if (setuid(*uid) != 0) { + return errno; + } + } + + if (gid != NULL) { + if (setgid(*gid) != 0) { + return errno; + } + } + + if (create_session != 0) { + (void)setsid(); + } + } + + // Set POSIX_SPAWN_SETEXEC if we already forked + if (require_pre_fork) { + short flags = 0; + int rc = posix_spawnattr_getflags(spawn_attrs, &flags); + if (rc != 0) { + return rc; + } + + rc = posix_spawnattr_setflags( + (posix_spawnattr_t *)spawn_attrs, flags | POSIX_SPAWN_SETEXEC + ); + if (rc != 0) { + return rc; + } + } + + // Spawn + return posix_spawn(pid, exec_path, file_actions, spawn_attrs, args, env); +} + +#endif // TARGET_OS_MAC + +// MARK: - Linux (fork/exec + posix_spawn fallback) + +#if _POSIX_SPAWN +static int _subprocess_posix_spawn_fallback( + pid_t * _Nonnull pid, + const char * _Nonnull exec_path, + const char * _Nullable working_directory, + const int file_descriptors[_Nonnull], + char * _Nullable const args[_Nonnull], + char * _Nullable const env[_Nullable], + gid_t * _Nullable process_group_id +) { + // Setup stdin, stdout, and stderr + posix_spawn_file_actions_t file_actions; + + int rc = posix_spawn_file_actions_init(&file_actions); + if (rc != 0) { return rc; } + if (file_descriptors[0] >= 0) { + rc = posix_spawn_file_actions_adddup2( + &file_actions, file_descriptors[0], STDIN_FILENO + ); + if (rc != 0) { return rc; } + } + if (file_descriptors[2] >= 0) { + rc = posix_spawn_file_actions_adddup2( + &file_actions, file_descriptors[2], STDOUT_FILENO + ); + if (rc != 0) { return rc; } + } + if (file_descriptors[4] >= 0) { + rc = posix_spawn_file_actions_adddup2( + &file_actions, file_descriptors[4], STDERR_FILENO + ); + if (rc != 0) { return rc; } + } + + // Close parent side + if (file_descriptors[1] >= 0) { + rc = posix_spawn_file_actions_addclose(&file_actions, file_descriptors[1]); + if (rc != 0) { return rc; } + } + if (file_descriptors[3] >= 0) { + rc = posix_spawn_file_actions_addclose(&file_actions, file_descriptors[3]); + if (rc != 0) { return rc; } + } + if (file_descriptors[5] >= 0) { + rc = posix_spawn_file_actions_addclose(&file_actions, file_descriptors[5]); + if (rc != 0) { return rc; } + } + + // Setup spawnattr + posix_spawnattr_t spawn_attr; + rc = posix_spawnattr_init(&spawn_attr); + if (rc != 0) { return rc; } + // Masks + sigset_t no_signals; + sigset_t all_signals; + sigemptyset(&no_signals); + sigfillset(&all_signals); + rc = posix_spawnattr_setsigmask(&spawn_attr, &no_signals); + if (rc != 0) { return rc; } + rc = posix_spawnattr_setsigdefault(&spawn_attr, &all_signals); + if (rc != 0) { return rc; } + // Flags + short flags = POSIX_SPAWN_SETSIGMASK | POSIX_SPAWN_SETSIGDEF; + if (process_group_id != NULL) { + flags |= POSIX_SPAWN_SETPGROUP; + rc = posix_spawnattr_setpgroup(&spawn_attr, *process_group_id); + if (rc != 0) { return rc; } + } + rc = posix_spawnattr_setflags(&spawn_attr, flags); + + // Spawn! + rc = posix_spawn( + pid, exec_path, + &file_actions, &spawn_attr, + args, env + ); + posix_spawn_file_actions_destroy(&file_actions); + posix_spawnattr_destroy(&spawn_attr); + return rc; +} +#endif // _POSIX_SPAWN + +int _subprocess_fork_exec( + pid_t * _Nonnull pid, + const char * _Nonnull exec_path, + const char * _Nullable working_directory, + const int file_descriptors[_Nonnull], + char * _Nullable const args[_Nonnull], + char * _Nullable const env[_Nullable], + uid_t * _Nullable uid, + gid_t * _Nullable gid, + gid_t * _Nullable process_group_id, + int number_of_sgroups, const gid_t * _Nullable sgroups, + int create_session, + void (* _Nullable configurator)(void) +) { + int require_pre_fork = working_directory != NULL || + uid != NULL || + gid != NULL || + process_group_id != NULL || + (number_of_sgroups > 0 && sgroups != NULL) || + create_session || + configurator != NULL; + +#if _POSIX_SPAWN + // If posix_spawn is available on this platform and + // we do not require prefork, use posix_spawn if possible. + // + // (Glibc's posix_spawn does not support + // `POSIX_SPAWN_SETEXEC` therefore we have to keep + // using fork/exec if `require_pre_fork` is true. + if (require_pre_fork == 0) { + return _subprocess_posix_spawn_fallback( + pid, exec_path, + working_directory, + file_descriptors, + args, env, + process_group_id + ); + } +#endif + + pid_t child_pid = fork(); + if (child_pid != 0) { + *pid = child_pid; + return child_pid < 0 ? errno : 0; + } + + if (working_directory != NULL) { + if (chdir(working_directory) != 0) { + return errno; + } + } + + + if (uid != NULL) { + if (setuid(*uid) != 0) { + return errno; + } + } + + if (gid != NULL) { + if (setgid(*gid) != 0) { + return errno; + } + } + + if (number_of_sgroups > 0 && sgroups != NULL) { + if (setgroups(number_of_sgroups, sgroups) != 0) { + return errno; + } + } + + if (create_session != 0) { + (void)setsid(); + } + + if (process_group_id != NULL) { + (void)setpgid(0, *process_group_id); + } + + // Bind stdin, stdout, and stderr + int rc = 0; + if (file_descriptors[0] >= 0) { + rc = dup2(file_descriptors[0], STDIN_FILENO); + if (rc < 0) { return errno; } + } + if (file_descriptors[2] >= 0) { + rc = dup2(file_descriptors[2], STDOUT_FILENO); + if (rc < 0) { return errno; } + } + if (file_descriptors[4] >= 0) { + rc = dup2(file_descriptors[4], STDERR_FILENO); + if (rc < 0) { return errno; } + } + // Close parent side + if (file_descriptors[1] >= 0) { + rc = close(file_descriptors[1]); + } + if (file_descriptors[3] >= 0) { + rc = close(file_descriptors[3]); + } + if (file_descriptors[4] >= 0) { + rc = close(file_descriptors[5]); + } + if (rc != 0) { + return errno; + } + // Run custom configuratior + if (configurator != NULL) { + configurator(); + } + // Finally, exec + execve(exec_path, args, env); + // If we got here, something went wrong + return errno; +} + +#endif // !TARGET_OS_WINDOWS + diff --git a/Sources/_Subprocess/LockedState.swift b/Sources/_Subprocess/LockedState.swift new file mode 100644 index 00000000..e095668c --- /dev/null +++ b/Sources/_Subprocess/LockedState.swift @@ -0,0 +1,160 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if canImport(os) +internal import os +#if FOUNDATION_FRAMEWORK && canImport(C.os.lock) +internal import C.os.lock +#endif +#elseif canImport(Bionic) +import Bionic +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#elseif canImport(WinSDK) +import WinSDK +#endif + +package struct LockedState { + + // Internal implementation for a cheap lock to aid sharing code across platforms + private struct _Lock { +#if canImport(os) + typealias Primitive = os_unfair_lock +#elseif canImport(Bionic) || canImport(Glibc) || canImport(Musl) + typealias Primitive = pthread_mutex_t +#elseif canImport(WinSDK) + typealias Primitive = SRWLOCK +#elseif os(WASI) + // WASI is single-threaded, so we don't need a lock. + typealias Primitive = Void +#endif + + typealias PlatformLock = UnsafeMutablePointer + var _platformLock: PlatformLock + + fileprivate static func initialize(_ platformLock: PlatformLock) { +#if canImport(os) + platformLock.initialize(to: os_unfair_lock()) +#elseif canImport(Bionic) || canImport(Glibc) + pthread_mutex_init(platformLock, nil) +#elseif canImport(WinSDK) + InitializeSRWLock(platformLock) +#elseif os(WASI) + // no-op +#endif + } + + fileprivate static func deinitialize(_ platformLock: PlatformLock) { +#if canImport(Bionic) || canImport(Glibc) + pthread_mutex_destroy(platformLock) +#endif + platformLock.deinitialize(count: 1) + } + + static fileprivate func lock(_ platformLock: PlatformLock) { +#if canImport(os) + os_unfair_lock_lock(platformLock) +#elseif canImport(Bionic) || canImport(Glibc) + pthread_mutex_lock(platformLock) +#elseif canImport(WinSDK) + AcquireSRWLockExclusive(platformLock) +#elseif os(WASI) + // no-op +#endif + } + + static fileprivate func unlock(_ platformLock: PlatformLock) { +#if canImport(os) + os_unfair_lock_unlock(platformLock) +#elseif canImport(Bionic) || canImport(Glibc) + pthread_mutex_unlock(platformLock) +#elseif canImport(WinSDK) + ReleaseSRWLockExclusive(platformLock) +#elseif os(WASI) + // no-op +#endif + } + } + + private class _Buffer: ManagedBuffer { + deinit { + withUnsafeMutablePointerToElements { + _Lock.deinitialize($0) + } + } + } + + private let _buffer: ManagedBuffer + + package init(initialState: State) { + _buffer = _Buffer.create(minimumCapacity: 1, makingHeaderWith: { buf in + buf.withUnsafeMutablePointerToElements { + _Lock.initialize($0) + } + return initialState + }) + } + + package func withLock(_ body: @Sendable (inout State) throws -> T) rethrows -> T { + try withLockUnchecked(body) + } + + package func withLockUnchecked(_ body: (inout State) throws -> T) rethrows -> T { + try _buffer.withUnsafeMutablePointers { state, lock in + _Lock.lock(lock) + defer { _Lock.unlock(lock) } + return try body(&state.pointee) + } + } + + // Ensures the managed state outlives the locked scope. + package func withLockExtendingLifetimeOfState(_ body: @Sendable (inout State) throws -> T) rethrows -> T { + try _buffer.withUnsafeMutablePointers { state, lock in + _Lock.lock(lock) + return try withExtendedLifetime(state.pointee) { + defer { _Lock.unlock(lock) } + return try body(&state.pointee) + } + } + } +} + +extension LockedState where State == Void { + package init() { + self.init(initialState: ()) + } + + package func withLock(_ body: @Sendable () throws -> R) rethrows -> R { + return try withLock { _ in + try body() + } + } + + package func lock() { + _buffer.withUnsafeMutablePointerToElements { lock in + _Lock.lock(lock) + } + } + + package func unlock() { + _buffer.withUnsafeMutablePointerToElements { lock in + _Lock.unlock(lock) + } + } +} + +extension LockedState: @unchecked Sendable where State: Sendable {} + diff --git a/Sources/_Subprocess/Platforms/Subprocess+Darwin.swift b/Sources/_Subprocess/Platforms/Subprocess+Darwin.swift new file mode 100644 index 00000000..4ac2276e --- /dev/null +++ b/Sources/_Subprocess/Platforms/Subprocess+Darwin.swift @@ -0,0 +1,353 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if canImport(Darwin) + +#if canImport(FoundationEssentials) +import FoundationEssentials +#elseif canImport(Foundation) +import Foundation +#endif + +import Darwin +import Dispatch +import SystemPackage + +#if FOUNDATION_FRAMEWORK +@_implementationOnly import _FoundationCShims +#else +import _CShims +#endif + +// Darwin specific implementation +extension Subprocess.Configuration { + internal typealias StringOrRawBytes = Subprocess.StringOrRawBytes + + internal func spawn( + withInput input: Subprocess.ExecutionInput, + output: Subprocess.ExecutionOutput, + error: Subprocess.ExecutionOutput + ) throws -> Subprocess { + let (executablePath, + env, argv, + intendedWorkingDir, + uidPtr, gidPtr, supplementaryGroups + ) = try self.preSpawn() + defer { + for ptr in env { ptr?.deallocate() } + for ptr in argv { ptr?.deallocate() } + uidPtr?.deallocate() + gidPtr?.deallocate() + } + + // Setup file actions and spawn attributes + var fileActions: posix_spawn_file_actions_t? = nil + var spawnAttributes: posix_spawnattr_t? = nil + // Setup stdin, stdout, and stderr + posix_spawn_file_actions_init(&fileActions) + defer { + posix_spawn_file_actions_destroy(&fileActions) + } + // Input + var result: Int32 = -1 + if let inputRead = input.getReadFileDescriptor() { + result = posix_spawn_file_actions_adddup2(&fileActions, inputRead.rawValue, 0) + guard result == 0 else { + try self.cleanupAll(input: input, output: output, error: error) + throw POSIXError(.init(rawValue: result) ?? .ENODEV) + } + } + if let inputWrite = input.getWriteFileDescriptor() { + // Close parent side + result = posix_spawn_file_actions_addclose(&fileActions, inputWrite.rawValue) + guard result == 0 else { + try self.cleanupAll(input: input, output: output, error: error) + throw POSIXError(.init(rawValue: result) ?? .ENODEV) + } + } + // Output + if let outputWrite = output.getWriteFileDescriptor() { + result = posix_spawn_file_actions_adddup2(&fileActions, outputWrite.rawValue, 1) + guard result == 0 else { + try self.cleanupAll(input: input, output: output, error: error) + throw POSIXError(.init(rawValue: result) ?? .ENODEV) + } + } + if let outputRead = output.getReadFileDescriptor() { + // Close parent side + result = posix_spawn_file_actions_addclose(&fileActions, outputRead.rawValue) + guard result == 0 else { + try self.cleanupAll(input: input, output: output, error: error) + throw POSIXError(.init(rawValue: result) ?? .ENODEV) + } + } + // Error + if let errorWrite = error.getWriteFileDescriptor() { + result = posix_spawn_file_actions_adddup2(&fileActions, errorWrite.rawValue, 2) + guard result == 0 else { + try self.cleanupAll(input: input, output: output, error: error) + throw POSIXError(.init(rawValue: result) ?? .ENODEV) + } + } + if let errorRead = error.getReadFileDescriptor() { + // Close parent side + result = posix_spawn_file_actions_addclose(&fileActions, errorRead.rawValue) + guard result == 0 else { + try self.cleanupAll(input: input, output: output, error: error) + throw POSIXError(.init(rawValue: result) ?? .ENODEV) + } + } + // Setup spawnAttributes + posix_spawnattr_init(&spawnAttributes) + defer { + posix_spawnattr_destroy(&spawnAttributes) + } + var noSignals = sigset_t() + var allSignals = sigset_t() + sigemptyset(&noSignals) + sigfillset(&allSignals) + posix_spawnattr_setsigmask(&spawnAttributes, &noSignals) + posix_spawnattr_setsigdefault(&spawnAttributes, &allSignals) + // Configure spawnattr + var spawnAttributeError: Int32 = 0 + var flags: Int32 = POSIX_SPAWN_CLOEXEC_DEFAULT | + POSIX_SPAWN_SETSIGMASK | POSIX_SPAWN_SETSIGDEF + if let pgid = self.platformOptions.processGroupID { + flags |= POSIX_SPAWN_SETPGROUP + spawnAttributeError = posix_spawnattr_setpgroup(&spawnAttributes, pid_t(pgid)) + } + spawnAttributeError = posix_spawnattr_setflags(&spawnAttributes, Int16(flags)) + // Set QualityOfService + // spanattr_qos seems to only accept `QOS_CLASS_UTILITY` or `QOS_CLASS_BACKGROUND` + // and returns an error of `EINVAL` if anything else is provided + if spawnAttributeError == 0 && self.platformOptions.qualityOfService == .utility{ + spawnAttributeError = posix_spawnattr_set_qos_class_np(&spawnAttributes, QOS_CLASS_UTILITY) + } else if spawnAttributeError == 0 && self.platformOptions.qualityOfService == .background { + spawnAttributeError = posix_spawnattr_set_qos_class_np(&spawnAttributes, QOS_CLASS_BACKGROUND) + } + + // Setup cwd + var chdirError: Int32 = 0 + if intendedWorkingDir != .currentWorkingDirectory { + chdirError = intendedWorkingDir.withPlatformString { workDir in + return posix_spawn_file_actions_addchdir_np(&fileActions, workDir) + } + } + + // Error handling + if chdirError != 0 || spawnAttributeError != 0 { + try self.cleanupAll(input: input, output: output, error: error) + if spawnAttributeError != 0 { + throw POSIXError(.init(rawValue: result) ?? .ENODEV) + } + + if chdirError != 0 { + throw CocoaError(.fileNoSuchFile, userInfo: [ + .debugDescriptionErrorKey: "Cannot failed to change the working directory to \(intendedWorkingDir) with errno \(chdirError)" + ]) + } + } + // Run additional config + if let spawnConfig = self.platformOptions.preSpawnProcessConfigurator { + try spawnConfig(&spawnAttributes, &fileActions) + } + // Spawn + var pid: pid_t = 0 + let spawnError: CInt = executablePath.withCString { exePath in + return supplementaryGroups.withOptionalUnsafeBufferPointer { sgroups in + return _subprocess_spawn( + &pid, exePath, + &fileActions, &spawnAttributes, + argv, env, + uidPtr, gidPtr, + Int32(supplementaryGroups?.count ?? 0), sgroups?.baseAddress, + self.platformOptions.createSession ? 1 : 0 + ) + } + } + // Spawn error + if spawnError != 0 { + try self.cleanupAll(input: input, output: output, error: error) + throw POSIXError(.init(rawValue: spawnError) ?? .ENODEV) + } + return Subprocess( + processIdentifier: .init(value: pid), + executionInput: input, + executionOutput: output, + executionError: error + ) + } +} + +// Special keys used in Error's user dictionary +extension String { + static let debugDescriptionErrorKey = "NSDebugDescription" +} + +// MARK: - Platform Specific Options +extension Subprocess { + /// The collection of platform-specific settings + /// to configure the subprocess when running + public struct PlatformOptions: Sendable { + public var qualityOfService: QualityOfService = .default + /// Set user ID for the subprocess + public var userID: uid_t? = nil + /// Set the real and effective group ID and the saved + /// set-group-ID of the subprocess, equivalent to calling + /// `setgid()` on the child process. + /// Group ID is used to control permissions, particularly + /// for file access. + public var groupID: gid_t? = nil + /// Set list of supplementary group IDs for the subprocess + public var supplementaryGroups: [gid_t]? = nil + /// Set the process group for the subprocess, equivalent to + /// calling `setpgid()` on the child process. + /// Process group ID is used to group related processes for + /// controlling signals. + public var processGroupID: pid_t? = nil + /// Creates a session and sets the process group ID + /// i.e. Detach from the terminal. + public var createSession: Bool = false + /// A lightweight code requirement that you use to + /// evaluate the executable for a launching process. + public var launchRequirementData: Data? = nil + /// An ordered list of steps in order to tear down the child + /// process in case the parent task is cancelled before + /// the child proces terminates. + /// Always ends in sending a `.kill` signal at the end. + public var teardownSequence: [TeardownStep] = [] + /// A closure to configure platform-specific + /// spawning constructs. This closure enables direct + /// configuration or override of underlying platform-specific + /// spawn settings that `Subprocess` utilizes internally, + /// in cases where Subprocess does not provide higher-level + /// APIs for such modifications. + /// + /// On Darwin, Subprocess uses `posix_spawn()` as the + /// underlying spawning mechanism. This closure allows + /// modification of the `posix_spawnattr_t` spawn attribute + /// and file actions `posix_spawn_file_actions_t` before + /// they are sent to `posix_spawn()`. + public var preSpawnProcessConfigurator: ( + @Sendable ( + inout posix_spawnattr_t?, + inout posix_spawn_file_actions_t? + ) throws -> Void + )? = nil + + public init() {} + } +} + +extension Subprocess.PlatformOptions: Hashable { + public static func == (lhs: Subprocess.PlatformOptions, rhs: Subprocess.PlatformOptions) -> Bool { + // Since we can't compare closure equality, + // as long as preSpawnProcessConfigurator is set + // always returns false so that `PlatformOptions` + // with it set will never equal to each other + if lhs.preSpawnProcessConfigurator != nil || + rhs.preSpawnProcessConfigurator != nil { + return false + } + return lhs.qualityOfService == rhs.qualityOfService && + lhs.userID == rhs.userID && + lhs.groupID == rhs.groupID && + lhs.supplementaryGroups == rhs.supplementaryGroups && + lhs.processGroupID == rhs.processGroupID && + lhs.createSession == rhs.createSession && + lhs.launchRequirementData == rhs.launchRequirementData + } + + public func hash(into hasher: inout Hasher) { + hasher.combine(self.qualityOfService) + hasher.combine(self.userID) + hasher.combine(self.groupID) + hasher.combine(self.supplementaryGroups) + hasher.combine(self.processGroupID) + hasher.combine(self.createSession) + hasher.combine(self.launchRequirementData) + // Since we can't really hash closures, + // use an UUID such that as long as + // `preSpawnProcessConfigurator` is set, it will + // never equal to other PlatformOptions + if self.preSpawnProcessConfigurator != nil { + hasher.combine(UUID()) + } + } +} + +extension Subprocess.PlatformOptions : CustomStringConvertible, CustomDebugStringConvertible { + internal func description(withIndent indent: Int) -> String { + let indent = String(repeating: " ", count: indent * 4) + return """ +PlatformOptions( +\(indent) qualityOfService: \(self.qualityOfService), +\(indent) userID: \(String(describing: userID)), +\(indent) groupID: \(String(describing: groupID)), +\(indent) supplementaryGroups: \(String(describing: supplementaryGroups)), +\(indent) processGroupID: \(String(describing: processGroupID)), +\(indent) createSession: \(createSession), +\(indent) launchRequirementData: \(String(describing: launchRequirementData)), +\(indent) preSpawnProcessConfigurator: \(self.preSpawnProcessConfigurator == nil ? "not set" : "set") +\(indent)) +""" + } + + public var description: String { + return self.description(withIndent: 0) + } + + public var debugDescription: String { + return self.description(withIndent: 0) + } +} + +// MARK: - Process Monitoring +@Sendable +internal func monitorProcessTermination( + forProcessWithIdentifier pid: Subprocess.ProcessIdentifier +) async throws -> Subprocess.TerminationStatus { + return try await withCheckedThrowingContinuation { continuation in + let source = DispatchSource.makeProcessSource( + identifier: pid.value, + eventMask: [.exit], + queue: .global() + ) + source.setEventHandler { + source.cancel() + var siginfo = siginfo_t() + let rc = waitid(P_PID, id_t(pid.value), &siginfo, WEXITED) + guard rc == 0 else { + continuation.resume(throwing: POSIXError(.init(rawValue: errno) ?? .ENODEV)) + return + } + switch siginfo.si_code { + case .init(CLD_EXITED): + continuation.resume(returning: .exited(siginfo.si_status)) + return + case .init(CLD_KILLED), .init(CLD_DUMPED): + continuation.resume(returning: .unhandledException(siginfo.si_status)) + case .init(CLD_TRAPPED), .init(CLD_STOPPED), .init(CLD_CONTINUED), .init(CLD_NOOP): + // Ignore these signals because they are not related to + // process exiting + break + default: + fatalError("Unexpected exit status: \(siginfo.si_code)") + } + } + source.resume() + } +} + +#endif // canImport(Darwin) diff --git a/Sources/_Subprocess/Platforms/Subprocess+Linux.swift b/Sources/_Subprocess/Platforms/Subprocess+Linux.swift new file mode 100644 index 00000000..9debf2e3 --- /dev/null +++ b/Sources/_Subprocess/Platforms/Subprocess+Linux.swift @@ -0,0 +1,287 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if canImport(Glibc) + +import Glibc +import Dispatch +import SystemPackage +import FoundationEssentials +import _CShims + +// Linux specific implementations +extension Subprocess.Configuration { + internal typealias StringOrRawBytes = Subprocess.StringOrRawBytes + + internal func spawn( + withInput input: Subprocess.ExecutionInput, + output: Subprocess.ExecutionOutput, + error: Subprocess.ExecutionOutput + ) throws -> Subprocess { + _setupMonitorSignalHandler() + + let (executablePath, + env, argv, + intendedWorkingDir, + uidPtr, gidPtr, + supplementaryGroups + ) = try self.preSpawn() + var processGroupIDPtr: UnsafeMutablePointer? = nil + if let processGroupID = self.platformOptions.processGroupID { + processGroupIDPtr = .allocate(capacity: 1) + processGroupIDPtr?.pointee = gid_t(processGroupID) + } + defer { + for ptr in env { ptr?.deallocate() } + for ptr in argv { ptr?.deallocate() } + uidPtr?.deallocate() + gidPtr?.deallocate() + processGroupIDPtr?.deallocate() + } + + let fileDescriptors: [CInt] = [ + input.getReadFileDescriptor()?.rawValue ?? -1, + input.getWriteFileDescriptor()?.rawValue ?? -1, + output.getWriteFileDescriptor()?.rawValue ?? -1, + output.getReadFileDescriptor()?.rawValue ?? -1, + error.getWriteFileDescriptor()?.rawValue ?? -1, + error.getReadFileDescriptor()?.rawValue ?? -1 + ] + + var workingDirectory: String? + if intendedWorkingDir != FilePath.currentWorkingDirectory { + // Only pass in working directory if it's different + workingDirectory = intendedWorkingDir.string + } + // Spawn + var pid: pid_t = 0 + let spawnError: CInt = executablePath.withCString { exePath in + return workingDirectory.withOptionalCString { workingDir in + return supplementaryGroups.withOptionalUnsafeBufferPointer { sgroups in + return fileDescriptors.withUnsafeBufferPointer { fds in + return _subprocess_fork_exec( + &pid, exePath, workingDir, + fds.baseAddress!, + argv, env, + uidPtr, gidPtr, + processGroupIDPtr, + CInt(supplementaryGroups?.count ?? 0), sgroups?.baseAddress, + self.platformOptions.createSession ? 1 : 0, + self.platformOptions.preSpawnProcessConfigurator + ) + } + } + } + } + // Spawn error + if spawnError != 0 { + try self.cleanupAll(input: input, output: output, error: error) + throw POSIXError(.init(rawValue: spawnError) ?? .ENODEV) + } + return Subprocess( + processIdentifier: .init(value: pid), + executionInput: input, + executionOutput: output, + executionError: error + ) + } +} + +// MARK: - Platform Specific Options +extension Subprocess { + /// The collection of platform-specific settings + /// to configure the subprocess when running + public struct PlatformOptions: Sendable { + // Set user ID for the subprocess + public var userID: uid_t? = nil + /// Set the real and effective group ID and the saved + /// set-group-ID of the subprocess, equivalent to calling + /// `setgid()` on the child process. + /// Group ID is used to control permissions, particularly + /// for file access. + public var groupID: gid_t? = nil + // Set list of supplementary group IDs for the subprocess + public var supplementaryGroups: [gid_t]? = nil + /// Set the process group for the subprocess, equivalent to + /// calling `setpgid()` on the child process. + /// Process group ID is used to group related processes for + /// controlling signals. + public var processGroupID: pid_t? = nil + // Creates a session and sets the process group ID + // i.e. Detach from the terminal. + public var createSession: Bool = false + /// An ordered list of steps in order to tear down the child + /// process in case the parent task is cancelled before + /// the child proces terminates. + /// Always ends in sending a `.kill` signal at the end. + public var teardownSequence: [TeardownStep] = [] + /// A closure to configure platform-specific + /// spawning constructs. This closure enables direct + /// configuration or override of underlying platform-specific + /// spawn settings that `Subprocess` utilizes internally, + /// in cases where Subprocess does not provide higher-level + /// APIs for such modifications. + /// + /// On Linux, Subprocess uses `fork/exec` as the + /// underlying spawning mechanism. This closure is called + /// after `fork()` but before `exec()`. You may use it to + /// call any necessary process setup functions. + public var preSpawnProcessConfigurator: (@convention(c) @Sendable () -> Void)? = nil + + public init() {} + } +} + +extension Subprocess.PlatformOptions: Hashable { + public static func ==( + lhs: Subprocess.PlatformOptions, + rhs: Subprocess.PlatformOptions + ) -> Bool { + // Since we can't compare closure equality, + // as long as preSpawnProcessConfigurator is set + // always returns false so that `PlatformOptions` + // with it set will never equal to each other + if lhs.preSpawnProcessConfigurator != nil || + rhs.preSpawnProcessConfigurator != nil { + return false + } + return lhs.userID == rhs.userID && + lhs.groupID == rhs.groupID && + lhs.supplementaryGroups == rhs.supplementaryGroups && + lhs.processGroupID == rhs.processGroupID && + lhs.createSession == rhs.createSession + } + + public func hash(into hasher: inout Hasher) { + hasher.combine(userID) + hasher.combine(groupID) + hasher.combine(supplementaryGroups) + hasher.combine(processGroupID) + hasher.combine(createSession) + // Since we can't really hash closures, + // use an UUID such that as long as + // `preSpawnProcessConfigurator` is set, it will + // never equal to other PlatformOptions + if self.preSpawnProcessConfigurator != nil { + hasher.combine(UUID()) + } + } +} + +extension Subprocess.PlatformOptions : CustomStringConvertible, CustomDebugStringConvertible { + internal func description(withIndent indent: Int) -> String { + let indent = String(repeating: " ", count: indent * 4) + return """ +PlatformOptions( +\(indent) userID: \(String(describing: userID)), +\(indent) groupID: \(String(describing: groupID)), +\(indent) supplementaryGroups: \(String(describing: supplementaryGroups)), +\(indent) processGroupID: \(String(describing: processGroupID)), +\(indent) createSession: \(createSession), +\(indent) preSpawnProcessConfigurator: \(self.preSpawnProcessConfigurator == nil ? "not set" : "set") +\(indent)) +""" + } + + public var description: String { + return self.description(withIndent: 0) + } + + public var debugDescription: String { + return self.description(withIndent: 0) + } +} + +// Special keys used in Error's user dictionary +extension String { + static let debugDescriptionErrorKey = "DebugDescription" +} + +// MARK: - Process Monitoring +@Sendable +internal func monitorProcessTermination( + forProcessWithIdentifier pid: Subprocess.ProcessIdentifier +) async throws -> Subprocess.TerminationStatus { + return try await withCheckedThrowingContinuation { continuation in + _childProcessContinuations.withLock { continuations in + if let existing = continuations.removeValue(forKey: pid.value), + case .status(let existingStatus) = existing { + // We already have existing status to report + continuation.resume(returning: existingStatus) + } else { + // Save the continuation for handler + continuations[pid.value] = .continuation(continuation) + } + } + } +} + +private enum ContinuationOrStatus { + case continuation(CheckedContinuation) + case status(Subprocess.TerminationStatus) +} + +private let _childProcessContinuations: LockedState< + [pid_t: ContinuationOrStatus] +> = LockedState(initialState: [:]) + +private var signalSource: (any DispatchSourceSignal)? = nil +private let setup: () = { + signalSource = DispatchSource.makeSignalSource( + signal: SIGCHLD, + queue: .global() + ) + signalSource?.setEventHandler { + _childProcessContinuations.withLock { continuations in + while true { + var siginfo = siginfo_t() + guard waitid(P_ALL, id_t(0), &siginfo, WEXITED) == 0 else { + return + } + var status: Subprocess.TerminationStatus? = nil + switch siginfo.si_code { + case .init(CLD_EXITED): + status = .exited(siginfo._sifields._sigchld.si_status) + case .init(CLD_KILLED), .init(CLD_DUMPED): + status = .unhandledException(siginfo._sifields._sigchld.si_status) + case .init(CLD_TRAPPED), .init(CLD_STOPPED), .init(CLD_CONTINUED): + // Ignore these signals because they are not related to + // process exiting + break + default: + fatalError("Unexpected exit status: \(siginfo.si_code)") + } + if let status = status { + let pid = siginfo._sifields._sigchld.si_pid + if let existing = continuations.removeValue(forKey: pid), + case .continuation(let c) = existing { + c.resume(returning: status) + } else { + // We don't have continuation yet, just state status + continuations[pid] = .status(status) + } + } + } + } + } + signalSource?.resume() +}() + +private func _setupMonitorSignalHandler() { + // Only executed once + setup +} + +#endif // canImport(Glibc) + diff --git a/Sources/_Subprocess/Platforms/Subprocess+Unix.swift b/Sources/_Subprocess/Platforms/Subprocess+Unix.swift new file mode 100644 index 00000000..bd110301 --- /dev/null +++ b/Sources/_Subprocess/Platforms/Subprocess+Unix.swift @@ -0,0 +1,458 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if canImport(Darwin) || canImport(Glibc) + +#if canImport(FoundationEssentials) +import FoundationEssentials +#elseif canImport(Foundation) +import Foundation +#endif + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +#if FOUNDATION_FRAMEWORK +@_implementationOnly import _FoundationCShims +#else +import _CShims +#endif + +import Dispatch +import SystemPackage + +// MARK: - Signals +extension Subprocess { + /// Signals are standardized messages sent to a running program + /// to trigger specific behavior, such as quitting or error handling. + public struct Signal : Hashable, Sendable { + /// The underlying platform specific value for the signal + public let rawValue: Int32 + + private init(rawValue: Int32) { + self.rawValue = rawValue + } + + /// The `.interrupt` signal is sent to a process by its + /// controlling terminal when a user wishes to interrupt + /// the process. + public static var interrupt: Self { .init(rawValue: SIGINT) } + /// The `.terminate` signal is sent to a process to request its + /// termination. Unlike the `.kill` signal, it can be caught + /// and interpreted or ignored by the process. This allows + /// the process to perform nice termination releasing resources + /// and saving state if appropriate. `.interrupt` is nearly + /// identical to `.terminate`. + public static var terminate: Self { .init(rawValue: SIGTERM) } + /// The `.suspend` signal instructs the operating system + /// to stop a process for later resumption. + public static var suspend: Self { .init(rawValue: SIGSTOP) } + /// The `resume` signal instructs the operating system to + /// continue (restart) a process previously paused by the + /// `.suspend` signal. + public static var resume: Self { .init(rawValue: SIGCONT) } + /// The `.kill` signal is sent to a process to cause it to + /// terminate immediately (kill). In contrast to `.terminate` + /// and `.interrupt`, this signal cannot be caught or ignored, + /// and the receiving process cannot perform any + /// clean-up upon receiving this signal. + public static var kill: Self { .init(rawValue: SIGKILL) } + /// The `.terminalClosed` signal is sent to a process when + /// its controlling terminal is closed. In modern systems, + /// this signal usually means that the controlling pseudo + /// or virtual terminal has been closed. + public static var terminalClosed: Self { .init(rawValue: SIGHUP) } + /// The `.quit` signal is sent to a process by its controlling + /// terminal when the user requests that the process quit + /// and perform a core dump. + public static var quit: Self { .init(rawValue: SIGQUIT) } + /// The `.userDefinedOne` signal is sent to a process to indicate + /// user-defined conditions. + public static var userDefinedOne: Self { .init(rawValue: SIGUSR1) } + /// The `.userDefinedTwo` signal is sent to a process to indicate + /// user-defined conditions. + public static var userDefinedTwo: Self { .init(rawValue: SIGUSR2) } + /// The `.alarm` signal is sent to a process when the corresponding + /// time limit is reached. + public static var alarm: Self { .init(rawValue: SIGALRM) } + /// The `.windowSizeChange` signal is sent to a process when + /// its controlling terminal changes its size (a window change). + public static var windowSizeChange: Self { .init(rawValue: SIGWINCH) } + } + + /// Send the given signal to the child process. + /// - Parameters: + /// - signal: The signal to send. + /// - shouldSendToProcessGroup: Whether this signal should be sent to + /// the entire process group. + public func send(_ signal: Signal, toProcessGroup shouldSendToProcessGroup: Bool) throws { + let pid = shouldSendToProcessGroup ? -(self.processIdentifier.value) : self.processIdentifier.value + guard kill(pid, signal.rawValue) == 0 else { + throw POSIXError(.init(rawValue: errno)!) + } + } + + internal func tryTerminate() -> Error? { + do { + try self.send(.kill, toProcessGroup: true) + } catch { + guard let posixError: POSIXError = error as? POSIXError else { + return error + } + // Ignore ESRCH (no such process) + if posixError.code != .ESRCH { + return error + } + } + return nil + } +} + +// MARK: - Environment Resolution +extension Subprocess.Environment { + internal static let pathEnvironmentVariableName = "PATH" + + internal func pathValue() -> String? { + switch self.config { + case .inherit(let overrides): + // If PATH value exists in overrides, use it + if let value = overrides[.string(Self.pathEnvironmentVariableName)] { + return value.stringValue + } + // Fall back to current process + return ProcessInfo.processInfo.environment[Self.pathEnvironmentVariableName] + case .custom(let fullEnvironment): + if let value = fullEnvironment[.string(Self.pathEnvironmentVariableName)] { + return value.stringValue + } + return nil + } + } + + // This method follows the standard "create" rule: `env` needs to be + // manually deallocated + internal func createEnv() -> [UnsafeMutablePointer?] { + func createFullCString( + fromKey keyContainer: Subprocess.StringOrRawBytes, + value valueContainer: Subprocess.StringOrRawBytes + ) -> UnsafeMutablePointer { + let rawByteKey: UnsafeMutablePointer = keyContainer.createRawBytes() + let rawByteValue: UnsafeMutablePointer = valueContainer.createRawBytes() + defer { + rawByteKey.deallocate() + rawByteValue.deallocate() + } + /// length = `key` + `=` + `value` + `\null` + let totalLength = keyContainer.count + 1 + valueContainer.count + 1 + let fullString: UnsafeMutablePointer = .allocate(capacity: totalLength) + #if canImport(Darwin) + _ = snprintf(ptr: fullString, totalLength, "%s=%s", rawByteKey, rawByteValue) + #else + _ = _shims_snprintf(fullString, CInt(totalLength), "%s=%s", rawByteKey, rawByteValue) + #endif + return fullString + } + + var env: [UnsafeMutablePointer?] = [] + switch self.config { + case .inherit(let updates): + var current = ProcessInfo.processInfo.environment + for (keyContainer, valueContainer) in updates { + if let stringKey = keyContainer.stringValue { + // Remove the value from current to override it + current.removeValue(forKey: stringKey) + } + // Fast path + if case .string(let stringKey) = keyContainer, + case .string(let stringValue) = valueContainer { + let fullString = "\(stringKey)=\(stringValue)" + env.append(strdup(fullString)) + continue + } + + env.append(createFullCString(fromKey: keyContainer, value: valueContainer)) + } + // Add the rest of `current` to env + for (key, value) in current { + let fullString = "\(key)=\(value)" + env.append(strdup(fullString)) + } + case .custom(let customValues): + for (keyContainer, valueContainer) in customValues { + // Fast path + if case .string(let stringKey) = keyContainer, + case .string(let stringValue) = valueContainer { + let fullString = "\(stringKey)=\(stringValue)" + env.append(strdup(fullString)) + continue + } + env.append(createFullCString(fromKey: keyContainer, value: valueContainer)) + } + } + env.append(nil) + return env + } +} + +// MARK: Args Creation +extension Subprocess.Arguments { + // This method follows the standard "create" rule: `args` needs to be + // manually deallocated + internal func createArgs(withExecutablePath executablePath: String) -> [UnsafeMutablePointer?] { + var argv: [UnsafeMutablePointer?] = self.storage.map { $0.createRawBytes() } + // argv[0] = executable path + if let override = self.executablePathOverride { + argv.insert(override.createRawBytes(), at: 0) + } else { + argv.insert(strdup(executablePath), at: 0) + } + argv.append(nil) + return argv + } +} + +// MARK: - ProcessIdentifier +extension Subprocess { + /// A platform independent identifier for a subprocess. + public struct ProcessIdentifier: Sendable, Hashable, Codable { + /// The platform specific process identifier value + public let value: pid_t + + public init(value: pid_t) { + self.value = value + } + } +} + +extension Subprocess.ProcessIdentifier : CustomStringConvertible, CustomDebugStringConvertible { + public var description: String { "\(self.value)" } + + public var debugDescription: String { "\(self.value)" } +} + +// MARK: - Executable Searching +extension Subprocess.Executable { + internal static var defaultSearchPaths: Set { + return Set([ + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + "/usr/local/bin" + ]) + } + + internal func resolveExecutablePath(withPathValue pathValue: String?) -> String? { + switch self.storage { + case .executable(let executableName): + // If the executableName in is already a full path, return it directly + if Subprocess.Configuration.pathAccessible(executableName, mode: X_OK) { + return executableName + } + // Get $PATH from environment + let searchPaths: Set + if let pathValue = pathValue { + let localSearchPaths = pathValue.split(separator: ":").map { String($0) } + searchPaths = Set(localSearchPaths).union(Self.defaultSearchPaths) + } else { + searchPaths = Self.defaultSearchPaths + } + + for path in searchPaths { + let fullPath = "\(path)/\(executableName)" + let fileExists = Subprocess.Configuration.pathAccessible(fullPath, mode: X_OK) + if fileExists { + return fullPath + } + } + case .path(let executablePath): + // Use path directly + return executablePath.string + } + return nil + } +} + +// MARK: - Configuration +extension Subprocess.Configuration { + internal func preSpawn() throws -> ( + executablePath: String, + env: [UnsafeMutablePointer?], + argv: [UnsafeMutablePointer?], + intendedWorkingDir: FilePath, + uidPtr: UnsafeMutablePointer?, + gidPtr: UnsafeMutablePointer?, + supplementaryGroups: [gid_t]? + ) { + // Prepare environment + let env = self.environment.createEnv() + // Prepare executable path + guard let executablePath = self.executable.resolveExecutablePath( + withPathValue: self.environment.pathValue()) else { + for ptr in env { ptr?.deallocate() } + throw CocoaError(.executableNotLoadable, userInfo: [ + .debugDescriptionErrorKey : "\(self.executable.description) is not an executable" + ]) + } + // Prepare arguments + let argv: [UnsafeMutablePointer?] = self.arguments.createArgs(withExecutablePath: executablePath) + // Prepare workingDir + let intendedWorkingDir = self.workingDirectory + guard Self.pathAccessible(intendedWorkingDir.string, mode: F_OK) else { + for ptr in env { ptr?.deallocate() } + for ptr in argv { ptr?.deallocate() } + throw CocoaError(.fileNoSuchFile, userInfo: [ + .debugDescriptionErrorKey : "Failed to set working directory to \(intendedWorkingDir)" + ]) + } + + var uidPtr: UnsafeMutablePointer? = nil + if let userID = self.platformOptions.userID { + uidPtr = .allocate(capacity: 1) + uidPtr?.pointee = userID + } + var gidPtr: UnsafeMutablePointer? = nil + if let groupID = self.platformOptions.groupID { + gidPtr = .allocate(capacity: 1) + gidPtr?.pointee = groupID + } + var supplementaryGroups: [gid_t]? + if let groupsValue = self.platformOptions.supplementaryGroups { + supplementaryGroups = groupsValue + } + return ( + executablePath: executablePath, + env: env, argv: argv, + intendedWorkingDir: intendedWorkingDir, + uidPtr: uidPtr, gidPtr: gidPtr, + supplementaryGroups: supplementaryGroups + ) + } + + internal static func pathAccessible(_ path: String, mode: Int32) -> Bool { + return path.withCString { + return access($0, mode) == 0 + } + } +} + +// MARK: - FileDescriptor extensions +extension FileDescriptor { + internal static func openDevNull( + withAcessMode mode: FileDescriptor.AccessMode + ) throws -> FileDescriptor { + let devnull: FileDescriptor = try .open("/dev/null", mode) + return devnull + } + + internal var platformDescriptor: Subprocess.PlatformFileDescriptor { + return self + } + + internal func readChunk(upToLength maxLength: Int) async throws -> Data? { + return try await withCheckedThrowingContinuation { continuation in + DispatchIO.read( + fromFileDescriptor: self.rawValue, + maxLength: maxLength, + runningHandlerOn: .global() + ) { data, error in + if error != 0 { + continuation.resume(throwing: POSIXError(.init(rawValue: error) ?? .ENODEV)) + return + } + if data.isEmpty { + continuation.resume(returning: nil) + } else { + continuation.resume(returning: Data(data)) + } + } + } + } + + internal func readUntilEOF(upToLength maxLength: Int) async throws -> Data { + return try await withCheckedThrowingContinuation { continuation in + let dispatchIO = DispatchIO( + type: .stream, + fileDescriptor: self.rawValue, + queue: .global() + ) { error in + if error != 0 { + continuation.resume(throwing: POSIXError(.init(rawValue: error) ?? .ENODEV)) + } + } + var buffer: Data = Data() + dispatchIO.read( + offset: 0, + length: maxLength, + queue: .global() + ) { done, data, error in + guard error == 0 else { + continuation.resume(throwing: POSIXError(.init(rawValue: error) ?? .ENODEV)) + return + } + if let data = data { + buffer += Data(data) + } + if done { + dispatchIO.close() + continuation.resume(returning: buffer) + } + } + } + } + + internal func write(_ data: S) async throws where S.Element == UInt8 { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) -> Void in + let dispatchData: DispatchData = Array(data).withUnsafeBytes { + return DispatchData(bytes: $0) + } + DispatchIO.write( + toFileDescriptor: self.rawValue, + data: dispatchData, + runningHandlerOn: .global() + ) { _, error in + guard error == 0 else { + continuation.resume( + throwing: POSIXError( + .init(rawValue: error) ?? .ENODEV) + ) + return + } + continuation.resume() + } + } + } +} + +extension Subprocess { + internal typealias PlatformFileDescriptor = FileDescriptor +} + +// MARK: - Read Buffer Size +extension Subprocess { + @inline(__always) + internal static var readBufferSize: Int { +#if canImport(Darwin) + return 16384 +#else + // FIXME: Use Platform.pageSize here + return 4096 +#endif // canImport(Darwin) + } +} + +#endif // canImport(Darwin) || canImport(Glibc) diff --git a/Sources/_Subprocess/Platforms/Subprocess+Windows.swift b/Sources/_Subprocess/Platforms/Subprocess+Windows.swift new file mode 100644 index 00000000..978cb139 --- /dev/null +++ b/Sources/_Subprocess/Platforms/Subprocess+Windows.swift @@ -0,0 +1,1212 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if canImport(WinSDK) + +import WinSDK +import Dispatch +import SystemPackage +import FoundationEssentials + +// Windows specific implementation +extension Subprocess.Configuration { + internal func spawn( + withInput input: Subprocess.ExecutionInput, + output: Subprocess.ExecutionOutput, + error: Subprocess.ExecutionOutput + ) throws -> Subprocess { + // Spawn differently depending on whether + // we need to spawn as a user + if let userCredentials = self.platformOptions.userCredentials { + return try self.spawnAsUser( + withInput: input, + output: output, + error: error, + userCredentials: userCredentials + ) + } else { + return try self.spawnDirect( + withInput: input, + output: output, + error: error + ) + } + } + + internal func spawnDirect( + withInput input: Subprocess.ExecutionInput, + output: Subprocess.ExecutionOutput, + error: Subprocess.ExecutionOutput + ) throws -> Subprocess { + let ( + applicationName, + commandAndArgs, + environment, + intendedWorkingDir + ) = try self.preSpawn() + var startupInfo = try self.generateStartupInfo( + withInput: input, + output: output, + error: error + ) + var processInfo: PROCESS_INFORMATION = PROCESS_INFORMATION() + var createProcessFlags = self.generateCreateProcessFlag() + // Give calling process a chance to modify flag and startup info + if let configurator = self.platformOptions.preSpawnProcessConfigurator { + try configurator(&createProcessFlags, &startupInfo) + } + // Spawn! + try applicationName.withOptionalNTPathRepresentation { applicationNameW in + try commandAndArgs.withCString( + encodedAs: UTF16.self + ) { commandAndArgsW in + try environment.withCString( + encodedAs: UTF16.self + ) { environmentW in + try intendedWorkingDir.withNTPathRepresentation { intendedWorkingDirW in + let created = CreateProcessW( + applicationNameW, + UnsafeMutablePointer(mutating: commandAndArgsW), + nil, // lpProcessAttributes + nil, // lpThreadAttributes + true, // bInheritHandles + createProcessFlags, + UnsafeMutableRawPointer(mutating: environmentW), + intendedWorkingDirW, + &startupInfo, + &processInfo + ) + guard created else { + let windowsError = GetLastError() + try self.cleanupAll( + input: input, + output: output, + error: error + ) + throw CocoaError.windowsError( + underlying: windowsError, + errorCode: .fileWriteUnknown + ) + } + } + } + } + } + // We don't need the handle objects, so close it right away + guard CloseHandle(processInfo.hThread) else { + let windowsError = GetLastError() + try self.cleanupAll( + input: input, + output: output, + error: error + ) + throw CocoaError.windowsError( + underlying: windowsError, + errorCode: .fileReadUnknown + ) + } + guard CloseHandle(processInfo.hProcess) else { + let windowsError = GetLastError() + try self.cleanupAll( + input: input, + output: output, + error: error + ) + throw CocoaError.windowsError( + underlying: windowsError, + errorCode: .fileReadUnknown + ) + } + let pid = Subprocess.ProcessIdentifier( + processID: processInfo.dwProcessId, + threadID: processInfo.dwThreadId + ) + return Subprocess( + processIdentifier: pid, + executionInput: input, + executionOutput: output, + executionError: error, + consoleBehavior: self.platformOptions.consoleBehavior + ) + } + + internal func spawnAsUser( + withInput input: Subprocess.ExecutionInput, + output: Subprocess.ExecutionOutput, + error: Subprocess.ExecutionOutput, + userCredentials: Subprocess.PlatformOptions.UserCredentials + ) throws -> Subprocess { + let ( + applicationName, + commandAndArgs, + environment, + intendedWorkingDir + ) = try self.preSpawn() + var startupInfo = try self.generateStartupInfo( + withInput: input, + output: output, + error: error + ) + var processInfo: PROCESS_INFORMATION = PROCESS_INFORMATION() + var createProcessFlags = self.generateCreateProcessFlag() + // Give calling process a chance to modify flag and startup info + if let configurator = self.platformOptions.preSpawnProcessConfigurator { + try configurator(&createProcessFlags, &startupInfo) + } + // Spawn (featuring pyamid!) + try userCredentials.username.withCString( + encodedAs: UTF16.self + ) { usernameW in + try userCredentials.password.withCString( + encodedAs: UTF16.self + ) { passwordW in + try userCredentials.domain.withOptionalCString( + encodedAs: UTF16.self + ) { domainW in + try applicationName.withOptionalNTPathRepresentation { applicationNameW in + try commandAndArgs.withCString( + encodedAs: UTF16.self + ) { commandAndArgsW in + try environment.withCString( + encodedAs: UTF16.self + ) { environmentW in + try intendedWorkingDir.withNTPathRepresentation { intendedWorkingDirW in + let created = CreateProcessWithLogonW( + usernameW, + domainW, + passwordW, + DWORD(LOGON_WITH_PROFILE), + applicationNameW, + UnsafeMutablePointer(mutating: commandAndArgsW), + createProcessFlags, + UnsafeMutableRawPointer(mutating: environmentW), + intendedWorkingDirW, + &startupInfo, + &processInfo + ) + guard created else { + let windowsError = GetLastError() + try self.cleanupAll( + input: input, + output: output, + error: error + ) + throw CocoaError.windowsError( + underlying: windowsError, + errorCode: .fileWriteUnknown + ) + } + } + } + } + } + } + } + } + // We don't need the handle objects, so close it right away + guard CloseHandle(processInfo.hThread) else { + let windowsError = GetLastError() + try self.cleanupAll( + input: input, + output: output, + error: error + ) + throw CocoaError.windowsError( + underlying: windowsError, + errorCode: .fileReadUnknown + ) + } + guard CloseHandle(processInfo.hProcess) else { + let windowsError = GetLastError() + try self.cleanupAll( + input: input, + output: output, + error: error + ) + throw CocoaError.windowsError( + underlying: windowsError, + errorCode: .fileReadUnknown + ) + } + let pid = Subprocess.ProcessIdentifier( + processID: processInfo.dwProcessId, + threadID: processInfo.dwThreadId + ) + return Subprocess( + processIdentifier: pid, + executionInput: input, + executionOutput: output, + executionError: error, + consoleBehavior: self.platformOptions.consoleBehavior + ) + } +} + +// MARK: - Platform Specific Options +extension Subprocess { + /// The collection of platform-specific settings + /// to configure the subprocess when running + public struct PlatformOptions: Sendable { + /// A `UserCredentials` to use spawning the subprocess + /// as a different user + public struct UserCredentials: Sendable, Hashable { + // The name of the user. This is the name + // of the user account to run as. + public var username: String + // The clear-text password for the account. + public var password: String + // The name of the domain or server whose account database + // contains the account. + public var domain: String? + } + + /// `ConsoleBehavior` defines how should the console appear + /// when spawning a new process + public struct ConsoleBehavior: Sendable, Hashable { + internal enum Storage: Sendable, Hashable { + case createNew + case detatch + case inherit + } + + internal let storage: Storage + + private init(_ storage: Storage) { + self.storage = storage + } + + /// The subprocess has a new console, instead of + /// inheriting its parent's console (the default). + public static let createNew: Self = .init(.createNew) + /// For console processes, the new process does not + /// inherit its parent's console (the default). + /// The new process can call the `AllocConsole` + /// function at a later time to create a console. + public static let detatch: Self = .init(.detatch) + /// The subprocess inherits its parent's console. + public static let inherit: Self = .init(.inherit) + } + + /// `ConsoleBehavior` defines how should the window appear + /// when spawning a new process + public struct WindowStyle: Sendable, Hashable { + internal enum Storage: Sendable, Hashable { + case normal + case hidden + case maximized + case minimized + } + + internal let storage: Storage + + internal var platformStyle: WORD { + switch self.storage { + case .hidden: return WORD(SW_HIDE) + case .maximized: return WORD(SW_SHOWMAXIMIZED) + case .minimized: return WORD(SW_SHOWMINIMIZED) + default: return WORD(SW_SHOWNORMAL) + } + } + + private init(_ storage: Storage) { + self.storage = storage + } + + /// Activates and displays a window of normal size + public static let normal: Self = .init(.normal) + /// Does not activate a new window + public static let hidden: Self = .init(.hidden) + /// Activates the window and displays it as a maximized window. + public static let maximized: Self = .init(.maximized) + /// Activates the window and displays it as a minimized window. + public static let minimized: Self = .init(.minimized) + } + + /// Sets user credentials when starting the process as another user + public var userCredentials: UserCredentials? = nil + /// The console behavior of the new process, + /// default to inheriting the console from parent process + public var consoleBehavior: ConsoleBehavior = .inherit + /// Window style to use when the process is started + public var windowStyle: WindowStyle = .normal + /// Whether to create a new process group for the new + /// process. The process group includes all processes + /// that are descendants of this root process. + /// The process identifier of the new process group + /// is the same as the process identifier. + public var createProcessGroup: Bool = false + /// A closure to configure platform-specific + /// spawning constructs. This closure enables direct + /// configuration or override of underlying platform-specific + /// spawn settings that `Subprocess` utilizes internally, + /// in cases where Subprocess does not provide higher-level + /// APIs for such modifications. + /// + /// On Windows, Subprocess uses `CreateProcessW()` as the + /// underlying spawning mechanism. This closure allows + /// modification of the `dwCreationFlags` creation flag + /// and startup info `STARTUPINFOW` before + /// they are sent to `CreateProcessW()`. + public var preSpawnProcessConfigurator: ( + @Sendable ( + inout DWORD, + inout STARTUPINFOW + ) throws -> Void + )? = nil + + public init() {} + } +} + +extension Subprocess.PlatformOptions: Hashable { + public static func == ( + lhs: Subprocess.PlatformOptions, + rhs: Subprocess.PlatformOptions + ) -> Bool { + // Since we can't compare closure equality, + // as long as preSpawnProcessConfigurator is set + // always returns false so that `PlatformOptions` + // with it set will never equal to each other + if lhs.preSpawnProcessConfigurator != nil || rhs.preSpawnProcessConfigurator != nil { + return false + } + return lhs.userCredentials == rhs.userCredentials && lhs.consoleBehavior == rhs.consoleBehavior && lhs.windowStyle == rhs.windowStyle && + lhs.createProcessGroup == rhs.createProcessGroup + } + + public func hash(into hasher: inout Hasher) { + hasher.combine(userCredentials) + hasher.combine(consoleBehavior) + hasher.combine(windowStyle) + hasher.combine(createProcessGroup) + // Since we can't really hash closures, + // use an UUID such that as long as + // `preSpawnProcessConfigurator` is set, it will + // never equal to other PlatformOptions + if self.preSpawnProcessConfigurator != nil { + hasher.combine(UUID()) + } + } +} + +extension Subprocess.PlatformOptions : CustomStringConvertible, CustomDebugStringConvertible { + internal func description(withIndent indent: Int) -> String { + let indent = String(repeating: " ", count: indent * 4) + return """ +PlatformOptions( +\(indent) userCredentials: \(String(describing: self.userCredentials)), +\(indent) consoleBehavior: \(String(describing: self.consoleBehavior)), +\(indent) windowStyle: \(String(describing: self.windowStyle)), +\(indent) createProcessGroup: \(self.createProcessGroup), +\(indent) preSpawnProcessConfigurator: \(self.preSpawnProcessConfigurator == nil ? "not set" : "set") +\(indent)) +""" + } + + public var description: String { + return self.description(withIndent: 0) + } + + public var debugDescription: String { + return self.description(withIndent: 0) + } +} + +// MARK: - Process Monitoring +@Sendable +internal func monitorProcessTermination( + forProcessWithIdentifier pid: Subprocess.ProcessIdentifier +) async throws -> Subprocess.TerminationStatus { + // Once the continuation resumes, it will need to unregister the wait, so + // yield the wait handle back to the calling scope. + var waitHandle: HANDLE? + defer { + if let waitHandle { + _ = UnregisterWait(waitHandle) + } + } + guard let processHandle = OpenProcess( + DWORD(PROCESS_QUERY_INFORMATION | SYNCHRONIZE), + false, + pid.processID + ) else { + return .exited(1) + } + + try? await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + // Set up a callback that immediately resumes the continuation and does no + // other work. + let context = Unmanaged.passRetained(continuation as AnyObject).toOpaque() + let callback: WAITORTIMERCALLBACK = { context, _ in + let continuation = Unmanaged.fromOpaque(context!).takeRetainedValue() as! CheckedContinuation + continuation.resume() + } + + // We only want the callback to fire once (and not be rescheduled.) Waiting + // may take an arbitrarily long time, so let the thread pool know that too. + let flags = ULONG(WT_EXECUTEONLYONCE | WT_EXECUTELONGFUNCTION) + guard RegisterWaitForSingleObject( + &waitHandle, processHandle, callback, context, INFINITE, flags + ) else { + continuation.resume(throwing: CocoaError.windowsError( + underlying: GetLastError(), + errorCode: .fileWriteUnknown) + ) + return + } + } + + var status: DWORD = 0 + guard GetExitCodeProcess(processHandle, &status) else { + // The child process terminated but we couldn't get its status back. + // Assume generic failure. + return .exited(1) + } + let exitCodeValue = CInt(bitPattern: .init(status)) + if exitCodeValue >= 0 { + return .exited(status) + } else { + return .unhandledException(status) + } +} + +// MARK: - Subprocess Control +extension Subprocess { + /// Terminate the current subprocess with the given exit code + /// - Parameter exitCode: The exit code to use for the subprocess. + public func terminate(withExitCode exitCode: DWORD) throws { + guard let processHandle = OpenProcess( + // PROCESS_ALL_ACCESS + DWORD(STANDARD_RIGHTS_REQUIRED | SYNCHRONIZE | 0xFFFF), + false, + self.processIdentifier.processID + ) else { + throw CocoaError.windowsError( + underlying: GetLastError(), + errorCode: .fileWriteUnknown + ) + } + defer { + CloseHandle(processHandle) + } + guard TerminateProcess(processHandle, exitCode) else { + throw CocoaError.windowsError( + underlying: GetLastError(), + errorCode: .fileWriteUnknown + ) + } + } + + /// Suspend the current subprocess + public func suspend() throws { + guard let processHandle = OpenProcess( + // PROCESS_ALL_ACCESS + DWORD(STANDARD_RIGHTS_REQUIRED | SYNCHRONIZE | 0xFFFF), + false, + self.processIdentifier.processID + ) else { + throw CocoaError.windowsError( + underlying: GetLastError(), + errorCode: .fileWriteUnknown + ) + } + defer { + CloseHandle(processHandle) + } + + let NTSuspendProcess: Optional<(@convention(c) (HANDLE) -> LONG)> = + unsafeBitCast( + GetProcAddress( + GetModuleHandleA("ntdll.dll"), + "NtSuspendProcess" + ), + to: Optional<(@convention(c) (HANDLE) -> LONG)>.self + ) + guard let NTSuspendProcess = NTSuspendProcess else { + throw CocoaError(.executableNotLoadable) + } + guard NTSuspendProcess(processHandle) >= 0 else { + throw CocoaError.windowsError( + underlying: GetLastError(), + errorCode: .fileWriteUnknown + ) + } + } + + /// Resume the current subprocess after suspension + public func resume() throws { + guard let processHandle = OpenProcess( + // PROCESS_ALL_ACCESS + DWORD(STANDARD_RIGHTS_REQUIRED | SYNCHRONIZE | 0xFFFF), + false, + self.processIdentifier.processID + ) else { + throw CocoaError.windowsError( + underlying: GetLastError(), + errorCode: .fileWriteUnknown + ) + } + defer { + CloseHandle(processHandle) + } + + let NTResumeProcess: Optional<(@convention(c) (HANDLE) -> LONG)> = + unsafeBitCast( + GetProcAddress( + GetModuleHandleA("ntdll.dll"), + "NtResumeProcess" + ), + to: Optional<(@convention(c) (HANDLE) -> LONG)>.self + ) + guard let NTResumeProcess = NTResumeProcess else { + throw CocoaError(.executableNotLoadable) + } + guard NTResumeProcess(processHandle) >= 0 else { + throw CocoaError.windowsError( + underlying: GetLastError(), + errorCode: .fileWriteUnknown + ) + } + } + + internal func tryTerminate() -> Error? { + do { + try self.terminate(withExitCode: 0) + } catch { + return error + } + return nil + } +} + +// MARK: - Executable Searching +extension Subprocess.Executable { + // Technically not needed for CreateProcess since + // it takes process name. It's here to support + // Executable.resolveExecutablePath + internal func resolveExecutablePath(withPathValue pathValue: String?) -> String? { + switch self.storage { + case .executable(let executableName): + return executableName.withCString( + encodedAs: UTF16.self + ) { exeName -> String? in + return pathValue.withOptionalCString( + encodedAs: UTF16.self + ) { path -> String? in + let pathLenth = SearchPathW( + path, + exeName, + nil, 0, nil, nil + ) + guard pathLenth > 0 else { + return nil + } + return withUnsafeTemporaryAllocation( + of: WCHAR.self, capacity: Int(pathLenth) + 1 + ) { + _ = SearchPathW( + path, + exeName, nil, + pathLenth + 1, + $0.baseAddress, nil + ) + return String(decodingCString: $0.baseAddress!, as: UTF16.self) + } + } + } + case .path(let executablePath): + // Use path directly + return executablePath.string + } + } +} + +// MARK: - Environment Resolution +extension Subprocess.Environment { + internal static let pathEnvironmentVariableName = "Path" + + internal func pathValue() -> String? { + switch self.config { + case .inherit(let overrides): + // If PATH value exists in overrides, use it + if let value = overrides[.string(Self.pathEnvironmentVariableName)] { + return value.stringValue + } + // Fall back to current process + return ProcessInfo.processInfo.environment[Self.pathEnvironmentVariableName] + case .custom(let fullEnvironment): + if let value = fullEnvironment[.string(Self.pathEnvironmentVariableName)] { + return value.stringValue + } + return nil + } + } +} + +// MARK: - ProcessIdentifier +extension Subprocess { + /// A platform independent identifier for a subprocess. + public struct ProcessIdentifier: Sendable, Hashable, Codable { + /// Windows specifc process identifier value + public let processID: DWORD + /// Windows specific thread identifier associated with process + public let threadID: DWORD + + internal init( + processID: DWORD, + threadID: DWORD + ) { + self.processID = processID + self.threadID = threadID + } + } +} + +extension Subprocess.ProcessIdentifier: CustomStringConvertible, CustomDebugStringConvertible { + public var description: String { + return "(processID: \(self.processID), threadID: \(self.threadID))" + } + + public var debugDescription: String { + return description + } +} + +// MARK: - Private Utils +extension Subprocess.Configuration { + private func preSpawn() throws -> ( + applicationName: String?, + commandAndArgs: String, + environment: String, + intendedWorkingDir: String + ) { + // Prepare environment + var env: [String : String] = [:] + switch self.environment.config { + case .custom(let customValues): + // Use the custom values directly + for customKey in customValues.keys { + guard case .string(let stringKey) = customKey, + let valueContainer = customValues[customKey], + case .string(let stringValue) = valueContainer else { + fatalError("Windows does not support non unicode String as environments") + } + env.updateValue(stringValue, forKey: stringKey) + } + case .inherit(let updateValues): + // Combine current environment + env = ProcessInfo.processInfo.environment + for updatingKey in updateValues.keys { + // Override the current environment values + guard case .string(let stringKey) = updatingKey, + let valueContainer = updateValues[updatingKey], + case .string(let stringValue) = valueContainer else { + fatalError("Windows does not support non unicode String as environments") + } + env.updateValue(stringValue, forKey: stringKey) + } + } + // On Windows, the PATH is required in order to locate dlls needed by + // the process so we should also pass that to the child + let pathVariableName = Subprocess.Environment.pathEnvironmentVariableName + if env[pathVariableName] == nil, + let parentPath = ProcessInfo.processInfo.environment[pathVariableName] { + env[pathVariableName] = parentPath + } + // The environment string must be terminated by a double + // null-terminator. Otherwise, CreateProcess will fail with + // INVALID_PARMETER. + let environmentString = env.map { + $0.key + "=" + $0.value + }.joined(separator: "\0") + "\0\0" + + // Prepare arguments + let ( + applicationName, + commandAndArgs + ) = try self.generateWindowsCommandAndAgruments() + // Validate workingDir + guard Self.pathAccessible(self.workingDirectory.string) else { + throw CocoaError(.fileNoSuchFile, userInfo: [ + .debugDescriptionErrorKey : "Failed to set working directory to \(self.workingDirectory)" + ]) + } + return ( + applicationName: applicationName, + commandAndArgs: commandAndArgs, + environment: environmentString, + intendedWorkingDir: self.workingDirectory.string + ) + } + + private func generateCreateProcessFlag() -> DWORD { + var flags = CREATE_UNICODE_ENVIRONMENT + switch self.platformOptions.consoleBehavior.storage { + case .createNew: + flags |= CREATE_NEW_CONSOLE + case .detatch: + flags |= DETACHED_PROCESS + case .inherit: + break + } + if self.platformOptions.createProcessGroup { + flags |= CREATE_NEW_PROCESS_GROUP + } + return DWORD(flags) + } + + private func generateStartupInfo( + withInput input: Subprocess.ExecutionInput, + output: Subprocess.ExecutionOutput, + error: Subprocess.ExecutionOutput + ) throws -> STARTUPINFOW { + var info: STARTUPINFOW = STARTUPINFOW() + info.cb = DWORD(MemoryLayout.size) + info.dwFlags |= DWORD(STARTF_USESTDHANDLES) + + if self.platformOptions.windowStyle.storage != .normal { + info.wShowWindow = self.platformOptions.windowStyle.platformStyle + info.dwFlags |= DWORD(STARTF_USESHOWWINDOW) + } + // Bind IOs + // Input + if let inputRead = input.getReadFileDescriptor() { + info.hStdInput = inputRead.platformDescriptor + } + if let inputWrite = input.getWriteFileDescriptor() { + // Set parent side to be uninhertable + SetHandleInformation( + inputWrite.platformDescriptor, + DWORD(HANDLE_FLAG_INHERIT), + 0 + ) + } + // Output + if let outputWrite = output.getWriteFileDescriptor() { + info.hStdOutput = outputWrite.platformDescriptor + } + if let outputRead = output.getReadFileDescriptor() { + // Set parent side to be uninhertable + SetHandleInformation( + outputRead.platformDescriptor, + DWORD(HANDLE_FLAG_INHERIT), + 0 + ) + } + // Error + if let errorWrite = error.getWriteFileDescriptor() { + info.hStdError = errorWrite.platformDescriptor + } + if let errorRead = error.getReadFileDescriptor() { + // Set parent side to be uninhertable + SetHandleInformation( + errorRead.platformDescriptor, + DWORD(HANDLE_FLAG_INHERIT), + 0 + ) + } + return info + } + + private func generateWindowsCommandAndAgruments() throws -> ( + applicationName: String?, + commandAndArgs: String + ) { + // CreateProcess accepts partial names + let executableNameOrPath: String + switch self.executable.storage { + case .path(let path): + executableNameOrPath = path.string + case .executable(let name): + // Technically CreateProcessW accepts just the name + // of the executable, therefore we don't need to + // actually resolve the path. However, to maintain + // the same behavior as other platforms, still check + // here to make sure the executable actually exists + guard self.executable.resolveExecutablePath( + withPathValue: self.environment.pathValue() + ) != nil else { + throw CocoaError(.executableNotLoadable, userInfo: [ + .debugDescriptionErrorKey : "\(self.executable.description) is not an executable" + ]) + } + executableNameOrPath = name + } + var args = self.arguments.storage.map { + guard case .string(let stringValue) = $0 else { + // We should never get here since the API + // is guarded off + fatalError("Windows does not support non unicode String as arguments") + } + return stringValue + } + // The first parameter of CreateProcessW, `lpApplicationName` + // is optional. If it's nil, CreateProcessW uses argument[0] + // as the execuatble name. + // We should only set lpApplicationName if it's different from + // argument[0] (i.e. executablePathOverride) + var applicationName: String? = nil + if case .string(let overrideName) = self.arguments.executablePathOverride { + // Use the override as argument0 and set applicationName + args.insert(overrideName, at: 0) + applicationName = executableNameOrPath + } else { + // Set argument[0] to be executableNameOrPath + args.insert(executableNameOrPath, at: 0) + } + return ( + applicationName: applicationName, + commandAndArgs: self.quoteWindowsCommandLine(args) + ) + } + + // Taken from SCF + private func quoteWindowsCommandLine(_ commandLine: [String]) -> String { + func quoteWindowsCommandArg(arg: String) -> String { + // Windows escaping, adapted from Daniel Colascione's "Everyone quotes + // command line arguments the wrong way" - Microsoft Developer Blog + if !arg.contains(where: {" \t\n\"".contains($0)}) { + return arg + } + + // To escape the command line, we surround the argument with quotes. However + // the complication comes due to how the Windows command line parser treats + // backslashes (\) and quotes (") + // + // - \ is normally treated as a literal backslash + // - e.g. foo\bar\baz => foo\bar\baz + // - However, the sequence \" is treated as a literal " + // - e.g. foo\"bar => foo"bar + // + // But then what if we are given a path that ends with a \? Surrounding + // foo\bar\ with " would be "foo\bar\" which would be an unterminated string + + // since it ends on a literal quote. To allow this case the parser treats: + // + // - \\" as \ followed by the " metachar + // - \\\" as \ followed by a literal " + // - In general: + // - 2n \ followed by " => n \ followed by the " metachar + // - 2n+1 \ followed by " => n \ followed by a literal " + var quoted = "\"" + var unquoted = arg.unicodeScalars + + while !unquoted.isEmpty { + guard let firstNonBackslash = unquoted.firstIndex(where: { $0 != "\\" }) else { + // String ends with a backslash e.g. foo\bar\, escape all the backslashes + // then add the metachar " below + let backslashCount = unquoted.count + quoted.append(String(repeating: "\\", count: backslashCount * 2)) + break + } + let backslashCount = unquoted.distance(from: unquoted.startIndex, to: firstNonBackslash) + if (unquoted[firstNonBackslash] == "\"") { + // This is a string of \ followed by a " e.g. foo\"bar. Escape the + // backslashes and the quote + quoted.append(String(repeating: "\\", count: backslashCount * 2 + 1)) + quoted.append(String(unquoted[firstNonBackslash])) + } else { + // These are just literal backslashes + quoted.append(String(repeating: "\\", count: backslashCount)) + quoted.append(String(unquoted[firstNonBackslash])) + } + // Drop the backslashes and the following character + unquoted.removeFirst(backslashCount + 1) + } + quoted.append("\"") + return quoted + } + return commandLine.map(quoteWindowsCommandArg).joined(separator: " ") + } + + private static func pathAccessible(_ path: String) -> Bool { + return path.withCString(encodedAs: UTF16.self) { + let attrs = GetFileAttributesW($0) + return attrs != INVALID_FILE_ATTRIBUTES + } + } +} + +// MARK: - PlatformFileDescriptor Type +extension Subprocess { + internal typealias PlatformFileDescriptor = HANDLE +} + +// MARK: - Read Buffer Size +extension Subprocess { + @inline(__always) + internal static var readBufferSize: Int { + // FIXME: Use Platform.pageSize here + var sysInfo: SYSTEM_INFO = SYSTEM_INFO() + GetSystemInfo(&sysInfo) + return Int(sysInfo.dwPageSize) + } +} + +// MARK: - Pipe Support +extension FileDescriptor { + internal static func pipe() throws -> ( + readEnd: FileDescriptor, + writeEnd: FileDescriptor + ) { + var saAttributes: SECURITY_ATTRIBUTES = SECURITY_ATTRIBUTES() + saAttributes.nLength = DWORD(MemoryLayout.size) + saAttributes.bInheritHandle = true + saAttributes.lpSecurityDescriptor = nil + + var readHandle: HANDLE? = nil + var writeHandle: HANDLE? = nil + guard CreatePipe(&readHandle, &writeHandle, &saAttributes, 0), + readHandle != INVALID_HANDLE_VALUE, + writeHandle != INVALID_HANDLE_VALUE, + let readHandle: HANDLE = readHandle, + let writeHandle: HANDLE = writeHandle else { + throw CocoaError.windowsError( + underlying: GetLastError(), + errorCode: .fileReadUnknown + ) + } + let readFd = _open_osfhandle( + intptr_t(bitPattern: readHandle), + FileDescriptor.AccessMode.readOnly.rawValue + ) + let writeFd = _open_osfhandle( + intptr_t(bitPattern: writeHandle), + FileDescriptor.AccessMode.writeOnly.rawValue + ) + + return ( + readEnd: FileDescriptor(rawValue: readFd), + writeEnd: FileDescriptor(rawValue: writeFd) + ) + } + + internal static func openDevNull( + withAcessMode mode: FileDescriptor.AccessMode + ) throws -> FileDescriptor { + return try "NUL".withPlatformString { + let handle = CreateFileW( + $0, + DWORD(GENERIC_WRITE), + DWORD(FILE_SHARE_WRITE), + nil, + DWORD(OPEN_EXISTING), + DWORD(FILE_ATTRIBUTE_NORMAL), + nil + ) + guard let handle = handle, + handle != INVALID_HANDLE_VALUE else { + throw CocoaError.windowsError( + underlying: GetLastError(), + errorCode: .fileReadUnknown + ) + } + let devnull = _open_osfhandle( + intptr_t(bitPattern: handle), + mode.rawValue + ) + return FileDescriptor(rawValue: devnull) + } + } + + var platformDescriptor: Subprocess.PlatformFileDescriptor { + return HANDLE(bitPattern: _get_osfhandle(self.rawValue))! + } + + internal func read(upToLength maxLength: Int) async throws -> Data { + // TODO: Figure out a better way to asynchornously read + return try await withCheckedThrowingContinuation { continuation in + DispatchQueue.global(qos: .userInitiated).async { + var totalBytesRead: Int = 0 + var lastError: DWORD? = nil + let values = Array( + unsafeUninitializedCapacity: maxLength + ) { buffer, initializedCount in + while true { + guard let baseAddress = buffer.baseAddress else { + initializedCount = 0 + break + } + let bufferPtr = baseAddress.advanced(by: totalBytesRead) + var bytesRead: DWORD = 0 + let readSucceed = ReadFile( + self.platformDescriptor, + UnsafeMutableRawPointer(mutating: bufferPtr), + DWORD(maxLength - totalBytesRead), + &bytesRead, + nil + ) + if !readSucceed { + // Windows throws ERROR_BROKEN_PIPE when the pipe is closed + let error = GetLastError() + if error == ERROR_BROKEN_PIPE { + // We are done reading + initializedCount = totalBytesRead + } else { + // We got some error + lastError = error + initializedCount = 0 + } + break + } else { + // We succesfully read the current round + totalBytesRead += Int(bytesRead) + } + + if totalBytesRead >= maxLength { + initializedCount = min(maxLength, totalBytesRead) + break + } + } + } + if let lastError = lastError { + continuation.resume(throwing: CocoaError.windowsError( + underlying: lastError, + errorCode: .fileReadUnknown) + ) + } else { + continuation.resume(returning: Data(values)) + } + } + } + } + + internal func write(_ data: S) async throws where S.Element == UInt8 { + // TODO: Figure out a better way to asynchornously write + try await withCheckedThrowingContinuation { ( + continuation: CheckedContinuation + ) -> Void in + DispatchQueue.global(qos: .userInitiated).async { + let buffer = Array(data) + buffer.withUnsafeBytes { ptr in + var writtenBytes: DWORD = 0 + let writeSucceed = WriteFile( + self.platformDescriptor, + ptr.baseAddress, + DWORD(buffer.count), + &writtenBytes, + nil + ) + if !writeSucceed { + continuation.resume(throwing: CocoaError.windowsError( + underlying: GetLastError(), + errorCode: .fileWriteUnknown) + ) + } else { + continuation.resume() + } + } + } + } + } +} + +extension String { + static let debugDescriptionErrorKey = "DebugDescription" +} + +// MARK: - CocoaError + Win32 +internal let NSUnderlyingErrorKey = "NSUnderlyingError" + +extension CocoaError { + static func windowsError(underlying: DWORD, errorCode: Code) -> CocoaError { + let userInfo = [ + NSUnderlyingErrorKey : Win32Error(underlying) + ] + return CocoaError(errorCode, userInfo: userInfo) + } +} + +private extension Optional where Wrapped == String { + func withOptionalCString( + encodedAs targetEncoding: Encoding.Type, + _ body: (UnsafePointer?) throws -> Result + ) rethrows -> Result where Encoding : _UnicodeEncoding { + switch self { + case .none: + return try body(nil) + case .some(let value): + return try value.withCString(encodedAs: targetEncoding, body) + } + } + + func withOptionalNTPathRepresentation( + _ body: (UnsafePointer?) throws -> Result + ) throws -> Result { + switch self { + case .none: + return try body(nil) + case .some(let value): + return try value.withNTPathRepresentation(body) + } + } +} + +// MARK: - Remove these when merging back to SwiftFoundation +extension String { + internal func withNTPathRepresentation( + _ body: (UnsafePointer) throws -> Result + ) throws -> Result { + guard !isEmpty else { + throw CocoaError(.fileReadInvalidFileName) + } + + var iter = self.utf8.makeIterator() + let bLeadingSlash = if [._slash, ._backslash].contains(iter.next()), iter.next()?.isLetter ?? false, iter.next() == ._colon { true } else { false } + + // Strip the leading `/` on a RFC8089 path (`/[drive-letter]:/...` ). A + // leading slash indicates a rooted path on the drive for the current + // working directory. + return try Substring(self.utf8.dropFirst(bLeadingSlash ? 1 : 0)).withCString(encodedAs: UTF16.self) { pwszPath in + // 1. Normalize the path first. + let dwLength: DWORD = GetFullPathNameW(pwszPath, 0, nil, nil) + return try withUnsafeTemporaryAllocation(of: WCHAR.self, capacity: Int(dwLength)) { + guard GetFullPathNameW(pwszPath, DWORD($0.count), $0.baseAddress, nil) > 0 else { + throw CocoaError.windowsError( + underlying: GetLastError(), + errorCode: .fileReadUnknown + ) + } + + // 2. Perform the operation on the normalized path. + return try body($0.baseAddress!) + } + } + } +} + +struct Win32Error: Error { + public typealias Code = DWORD + public let code: Code + + public static var errorDomain: String { + return "NSWin32ErrorDomain" + } + + public init(_ code: Code) { + self.code = code + } +} + +internal extension UInt8 { + static var _slash: UInt8 { UInt8(ascii: "/") } + static var _backslash: UInt8 { UInt8(ascii: "\\") } + static var _colon: UInt8 { UInt8(ascii: ":") } + + var isLetter: Bool? { + return (0x41 ... 0x5a) ~= self || (0x61 ... 0x7a) ~= self + } +} + +#endif // canImport(WinSDK) diff --git a/Sources/_Subprocess/Subprocess+API.swift b/Sources/_Subprocess/Subprocess+API.swift new file mode 100644 index 00000000..f9c8b1ec --- /dev/null +++ b/Sources/_Subprocess/Subprocess+API.swift @@ -0,0 +1,465 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import SystemPackage +#if canImport(FoundationEssentials) +import FoundationEssentials +#elseif canImport(Foundation) +import Foundation +#endif + +extension Subprocess { + /// Run a executable with given parameters and capture its + /// standard output and standard error. + /// - Parameters: + /// - executable: The executable to run. + /// - arguments: The arguments to pass to the executable. + /// - environment: The environment to use for the process. + /// - workingDirectory: The working directory to use for the subprocess. + /// - platformOptions: The platform specific options to use + /// when running the executable. + /// - input: The input to send to the executable. + /// - output: The method to use for collecting the standard output. + /// - error: The method to use for collecting the standard error. + /// - Returns: `CollectedResult` which contains process identifier, + /// termination status, captured standard output and standard error. + public static func run( + _ executable: Executable, + arguments: Arguments = [], + environment: Environment = .inherit, + workingDirectory: FilePath? = nil, + platformOptions: PlatformOptions = PlatformOptions(), + input: InputMethod = .noInput, + output: CollectedOutputMethod = .collect(), + error: CollectedOutputMethod = .collect() + ) async throws -> CollectedResult { + let result = try await self.run( + executable, + arguments: arguments, + environment: environment, + workingDirectory: workingDirectory, + platformOptions: platformOptions, + input: input, + output: .init(method: output.method), + error: .init(method: error.method) + ) { subprocess in + let (standardOutput, standardError) = try await subprocess.captureIOs() + return ( + processIdentifier: subprocess.processIdentifier, + standardOutput: standardOutput, + standardError: standardError + ) + } + return CollectedResult( + processIdentifier: result.value.processIdentifier, + terminationStatus: result.terminationStatus, + standardOutput: result.value.standardOutput, + standardError: result.value.standardError + ) + } + + /// Run a executable with given parameters and capture its + /// standard output and standard error. + /// - Parameters: + /// - executable: The executable to run. + /// - arguments: The arguments to pass to the executable. + /// - environment: The environment to use for the process. + /// - workingDirectory: The working directory to use for the subprocess. + /// - platformOptions: The platform specific options to use + /// when running the executable. + /// - input: The input to send to the executable. + /// - output: The method to use for collecting the standard output. + /// - error: The method to use for collecting the standard error. + /// - Returns: `CollectedResult` which contains process identifier, + /// termination status, captured standard output and standard error. + public static func run( + _ executable: Executable, + arguments: Arguments = [], + environment: Environment = .inherit, + workingDirectory: FilePath? = nil, + platformOptions: PlatformOptions = PlatformOptions(), + input: some Sequence, + output: CollectedOutputMethod = .collect(), + error: CollectedOutputMethod = .collect() + ) async throws -> CollectedResult { + let result = try await self.run( + executable, + arguments: arguments, + environment: environment, + workingDirectory: workingDirectory, + platformOptions: platformOptions, + output: .init(method: output.method), + error: .init(method: output.method) + ) { subprocess, writer in + return try await withThrowingTaskGroup(of: CapturedIOs?.self) { group in + group.addTask { + try await writer.write(input) + try await writer.finish() + return nil + } + group.addTask { + return try await subprocess.captureIOs() + } + var capturedIOs: CapturedIOs! + while let result = try await group.next() { + if result != nil { + capturedIOs = result + } + } + return ( + processIdentifier: subprocess.processIdentifier, + standardOutput: capturedIOs.standardOutput, + standardError: capturedIOs.standardError + ) + } + } + return CollectedResult( + processIdentifier: result.value.processIdentifier, + terminationStatus: result.terminationStatus, + standardOutput: result.value.standardOutput, + standardError: result.value.standardError + ) + } + + /// Run a executable with given parameters and capture its + /// standard output and standard error. + /// - Parameters: + /// - executable: The executable to run. + /// - arguments: The arguments to pass to the executable. + /// - environment: The environment to use for the process. + /// - workingDirectory: The working directory to use for the subprocess. + /// - platformOptions: The platform specific options to use + /// when running the executable. + /// - input: The input to send to the executable. + /// - output: The method to use for collecting the standard output. + /// - error: The method to use for collecting the standard error. + /// - Returns: `CollectedResult` which contains process identifier, + /// termination status, captured standard output and standard error. + public static func run( + _ executable: Executable, + arguments: Arguments = [], + environment: Environment = .inherit, + workingDirectory: FilePath? = nil, + platformOptions: PlatformOptions = PlatformOptions(), + input: S, + output: CollectedOutputMethod = .collect(), + error: CollectedOutputMethod = .collect() + ) async throws -> CollectedResult where S.Element == UInt8 { + let result = try await self.run( + executable, + arguments: arguments, + environment: environment, + workingDirectory: workingDirectory, + platformOptions: platformOptions, + output: .init(method: output.method), + error: .init(method: output.method) + ) { subprocess, writer in + return try await withThrowingTaskGroup(of: CapturedIOs?.self) { group in + group.addTask { + try await writer.write(input) + try await writer.finish() + return nil + } + group.addTask { + return try await subprocess.captureIOs() + } + var capturedIOs: CapturedIOs! + while let result = try await group.next() { + capturedIOs = result + } + return ( + processIdentifier: subprocess.processIdentifier, + standardOutput: capturedIOs.standardOutput, + standardError: capturedIOs.standardError + ) + } + } + return CollectedResult( + processIdentifier: result.value.processIdentifier, + terminationStatus: result.terminationStatus, + standardOutput: result.value.standardOutput, + standardError: result.value.standardError + ) + } +} + +// MARK: Custom Execution Body +extension Subprocess { + /// Run a executable with given parameters and a custom closure + /// to manage the running subprocess' lifetime and its IOs. + /// - Parameters: + /// - executable: The executable to run. + /// - arguments: The arguments to pass to the executable. + /// - environment: The environment in which to run the executable. + /// - workingDirectory: The working directory in which to run the executable. + /// - platformOptions: The platform specific options to use + /// when running the executable. + /// - input: The input to send to the executable. + /// - output: The method to use for redirecting the standard output. + /// - error: The method to use for redirecting the standard error. + /// - body: The custom execution body to manually control the running process + /// - Returns a ExecutableResult type containing the return value + /// of the closure. + public static func run( + _ executable: Executable, + arguments: Arguments = [], + environment: Environment = .inherit, + workingDirectory: FilePath? = nil, + platformOptions: PlatformOptions = PlatformOptions(), + input: InputMethod = .noInput, + output: RedirectedOutputMethod = .redirectToSequence, + error: RedirectedOutputMethod = .redirectToSequence, + _ body: (sending @escaping (Subprocess) async throws -> R) + ) async throws -> ExecutionResult { + return try await Configuration( + executable: executable, + arguments: arguments, + environment: environment, + workingDirectory: workingDirectory, + platformOptions: platformOptions + ) + .run(input: input, output: output, error: error, body) + } + + /// Run a executable with given parameters and a custom closure + /// to manage the running subprocess' lifetime and its IOs. + /// - Parameters: + /// - executable: The executable to run. + /// - arguments: The arguments to pass to the executable. + /// - environment: The environment in which to run the executable. + /// - workingDirectory: The working directory in which to run the executable. + /// - platformOptions: The platform specific options to use + /// when running the executable. + /// - input: The input to send to the executable. + /// - output: The method to use for redirecting the standard output. + /// - error: The method to use for redirecting the standard error. + /// - body: The custom execution body to manually control the running process + /// - Returns a `ExecutableResult` type containing the return value + /// of the closure. + public static func run( + _ executable: Executable, + arguments: Arguments = [], + environment: Environment = .inherit, + workingDirectory: FilePath? = nil, + platformOptions: PlatformOptions = PlatformOptions(), + input: some Sequence, + output: RedirectedOutputMethod = .redirectToSequence, + error: RedirectedOutputMethod = .redirectToSequence, + _ body: (sending @escaping (Subprocess) async throws -> R) + ) async throws -> ExecutionResult { + return try await Configuration( + executable: executable, + arguments: arguments, + environment: environment, + workingDirectory: workingDirectory, + platformOptions: platformOptions + ) + .run(output: output, error: error) { execution, writer in + return try await withThrowingTaskGroup(of: R?.self) { group in + group.addTask { + try await writer.write(input) + try await writer.finish() + return nil + } + group.addTask { + return try await body(execution) + } + var result: R! + while let next = try await group.next() { + result = next + } + return result + } + } + } + + /// Run a executable with given parameters and a custom closure + /// to manage the running subprocess' lifetime and its IOs. + /// - Parameters: + /// - executable: The executable to run. + /// - arguments: The arguments to pass to the executable. + /// - environment: The environment in which to run the executable. + /// - workingDirectory: The working directory in which to run the executable. + /// - platformOptions: The platform specific options to use + /// when running the executable. + /// - input: The input to send to the executable. + /// - output: The method to use for redirecting the standard output. + /// - error: The method to use for redirecting the standard error. + /// - body: The custom execution body to manually control the running process + /// - Returns a `ExecutableResult` type containing the return value + /// of the closure. + public static func run( + _ executable: Executable, + arguments: Arguments = [], + environment: Environment = .inherit, + workingDirectory: FilePath? = nil, + platformOptions: PlatformOptions = PlatformOptions(), + input: S, + output: RedirectedOutputMethod = .redirectToSequence, + error: RedirectedOutputMethod = .redirectToSequence, + _ body: (sending @escaping (Subprocess) async throws -> R) + ) async throws -> ExecutionResult where S.Element == UInt8 { + return try await Configuration( + executable: executable, + arguments: arguments, + environment: environment, + workingDirectory: workingDirectory, + platformOptions: platformOptions + ) + .run(output: output, error: error) { execution, writer in + return try await withThrowingTaskGroup(of: R?.self) { group in + group.addTask { + try await writer.write(input) + try await writer.finish() + return nil + } + group.addTask { + return try await body(execution) + } + var result: R! + while let next = try await group.next() { + result = next + } + return result + } + } + } + + /// Run a executable with given parameters and a custom closure + /// to manage the running subprocess' lifetime and write to its + /// standard input via `StandardInputWriter` + /// - Parameters: + /// - executable: The executable to run. + /// - arguments: The arguments to pass to the executable. + /// - environment: The environment in which to run the executable. + /// - workingDirectory: The working directory in which to run the executable. + /// - platformOptions: The platform specific options to use + /// when running the executable. + /// - output: The method to use for redirecting the standard output. + /// - error: The method to use for redirecting the standard error. + /// - body: The custom execution body to manually control the running process + /// - Returns a ExecutableResult type containing the return value + /// of the closure. + public static func run( + _ executable: Executable, + arguments: Arguments = [], + environment: Environment = .inherit, + workingDirectory: FilePath? = nil, + platformOptions: PlatformOptions = PlatformOptions(), + output: RedirectedOutputMethod = .redirectToSequence, + error: RedirectedOutputMethod = .redirectToSequence, + _ body: (sending @escaping (Subprocess, StandardInputWriter) async throws -> R) + ) async throws -> ExecutionResult { + return try await Configuration( + executable: executable, + arguments: arguments, + environment: environment, + workingDirectory: workingDirectory, + platformOptions: platformOptions + ) + .run(output: output, error: error, body) + } +} + +// MARK: - Configuration Based +extension Subprocess { + /// Run a executable with given parameters specified by a + /// `Subprocess.Configuration` + /// - Parameters: + /// - configuration: The `Subprocess` configuration to run. + /// - output: The method to use for redirecting the standard output. + /// - error: The method to use for redirecting the standard error. + /// - body: The custom configuration body to manually control + /// the running process and write to its standard input. + /// - Returns a ExecutableResult type containing the return value + /// of the closure. + public static func run( + _ configuration: Configuration, + output: RedirectedOutputMethod = .redirectToSequence, + error: RedirectedOutputMethod = .redirectToSequence, + _ body: (sending @escaping (Subprocess, StandardInputWriter) async throws -> R) + ) async throws -> ExecutionResult { + return try await configuration.run(output: output, error: error, body) + } +} + +// MARK: - Detached +extension Subprocess { + /// Run a executable with given parameters and return its process + /// identifier immediately without monitoring the state of the + /// subprocess nor waiting until it exits. + /// + /// This method is useful for launching subprocesses that outlive their + /// parents (for example, daemons and trampolines). + /// + /// - Parameters: + /// - executable: The executable to run. + /// - arguments: The arguments to pass to the executable. + /// - environment: The environment to use for the process. + /// - workingDirectory: The working directory for the process. + /// - platformOptions: The platform specific options to use for the process. + /// - input: A file descriptor to bind to the subprocess' standard input. + /// - output: A file descriptor to bind to the subprocess' standard output. + /// - error: A file descriptor to bind to the subprocess' standard error. + /// - Returns: the process identifier for the subprocess. + public static func runDetached( + _ executable: Executable, + arguments: Arguments = [], + environment: Environment = .inherit, + workingDirectory: FilePath? = nil, + platformOptions: PlatformOptions = PlatformOptions(), + input: FileDescriptor? = nil, + output: FileDescriptor? = nil, + error: FileDescriptor? = nil + ) throws -> ProcessIdentifier { + // Create input + let executionInput: ExecutionInput + let executionOutput: ExecutionOutput + let executionError: ExecutionOutput + if let inputFd = input { + executionInput = .init(storage: .fileDescriptor(inputFd, false)) + } else { + let devnull: FileDescriptor = try .openDevNull(withAcessMode: .readOnly) + executionInput = .init(storage: .noInput(devnull)) + } + if let outputFd = output { + executionOutput = .init(storage: .fileDescriptor(outputFd, false)) + } else { + let devnull: FileDescriptor = try .openDevNull(withAcessMode: .writeOnly) + executionOutput = .init(storage: .discarded(devnull)) + } + if let errorFd = error { + executionError = .init( + storage: .fileDescriptor(errorFd, false) + ) + } else { + let devnull: FileDescriptor = try .openDevNull(withAcessMode: .writeOnly) + executionError = .init(storage: .discarded(devnull)) + } + // Spawn! + let config: Configuration = Configuration( + executable: executable, + arguments: arguments, + environment: environment, + workingDirectory: workingDirectory, + platformOptions: platformOptions + ) + return try config.spawn( + withInput: executionInput, + output: executionOutput, + error: executionError + ).processIdentifier + } +} + diff --git a/Sources/_Subprocess/Subprocess+AsyncDataSequence.swift b/Sources/_Subprocess/Subprocess+AsyncDataSequence.swift new file mode 100644 index 00000000..a3a3e393 --- /dev/null +++ b/Sources/_Subprocess/Subprocess+AsyncDataSequence.swift @@ -0,0 +1,86 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import SystemPackage +import Dispatch + +#if canImport(FoundationEssentials) +import FoundationEssentials +#elseif canImport(Foundation) +import Foundation +#endif + +extension Subprocess { + public struct AsyncDataSequence: AsyncSequence, Sendable, _AsyncSequence { + public typealias Error = any Swift.Error + + public typealias Element = Data + + @_nonSendable + public struct Iterator: AsyncIteratorProtocol { + public typealias Element = Data + + private let fileDescriptor: FileDescriptor + private var buffer: [UInt8] + private var currentPosition: Int + private var finished: Bool + + internal init(fileDescriptor: FileDescriptor) { + self.fileDescriptor = fileDescriptor + self.buffer = [] + self.currentPosition = 0 + self.finished = false + } + + public mutating func next() async throws -> Data? { + let data = try await self.fileDescriptor.readChunk( + upToLength: Subprocess.readBufferSize + ) + if data == nil { + // We finished reading. Close the file descriptor now + try self.fileDescriptor.close() + return nil + } + return data + } + } + + private let fileDescriptor: FileDescriptor + + init(fileDescriptor: FileDescriptor) { + self.fileDescriptor = fileDescriptor + } + + public func makeAsyncIterator() -> Iterator { + return Iterator(fileDescriptor: self.fileDescriptor) + } + } +} + +extension RangeReplaceableCollection { + /// Creates a new instance of a collection containing the elements of an asynchronous sequence. + /// + /// - Parameter source: The asynchronous sequence of elements for the new collection. + @inlinable + public init(_ source: Source) async rethrows where Source.Element == Element { + self.init() + for try await item in source { + append(item) + } + } +} + +public protocol _AsyncSequence: AsyncSequence { + associatedtype Error +} diff --git a/Sources/_Subprocess/Subprocess+Configuration.swift b/Sources/_Subprocess/Subprocess+Configuration.swift new file mode 100644 index 00000000..0b785790 --- /dev/null +++ b/Sources/_Subprocess/Subprocess+Configuration.swift @@ -0,0 +1,769 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import SystemPackage + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(WinSDK) +import WinSDK +#endif + +#if canImport(FoundationEssentials) +import FoundationEssentials +#elseif canImport(Foundation) +import Foundation +#endif + +extension Subprocess { + /// A collection of configurations parameters to use when + /// spawning a subprocess. + public struct Configuration: Sendable, Hashable { + + internal enum RunState: Sendable { + case workBody(Result) + case monitorChildProcess(TerminationStatus) + } + + /// The executable to run. + public var executable: Executable + /// The arguments to pass to the executable. + public var arguments: Arguments + /// The environment to use when running the executable. + public var environment: Environment + /// The working directory to use when running the executable. + public var workingDirectory: FilePath + /// The platform specifc options to use when + /// running the subprocess. + public var platformOptions: PlatformOptions + + public init( + executable: Executable, + arguments: Arguments = [], + environment: Environment = .inherit, + workingDirectory: FilePath? = nil, + platformOptions: PlatformOptions = PlatformOptions() + ) { + self.executable = executable + self.arguments = arguments + self.environment = environment + self.workingDirectory = workingDirectory ?? .currentWorkingDirectory + self.platformOptions = platformOptions + } + + /// Close each input individually, and throw the first error if there's multiple errors thrown + @Sendable + private func cleanup( + process: Subprocess, + childSide: Bool, parentSide: Bool, + attemptToTerminateSubProcess: Bool + ) async throws { + guard childSide || parentSide || attemptToTerminateSubProcess else { + return + } + + // Attempt to teardown the subprocess + if attemptToTerminateSubProcess { + await process.teardown( + using: self.platformOptions.teardownSequence + ) + } + + let inputCloseFunc: () throws -> Void + let outputCloseFunc: () throws -> Void + let errorCloseFunc: () throws -> Void + if childSide && parentSide { + // Close all + inputCloseFunc = process.executionInput.closeAll + outputCloseFunc = process.executionOutput.closeAll + errorCloseFunc = process.executionError.closeAll + } else if childSide { + // Close child only + inputCloseFunc = process.executionInput.closeChildSide + outputCloseFunc = process.executionOutput.closeChildSide + errorCloseFunc = process.executionError.closeChildSide + } else { + // Close parent only + inputCloseFunc = process.executionInput.closeParentSide + outputCloseFunc = process.executionOutput.closeParentSide + errorCloseFunc = process.executionError.closeParentSide + } + + var inputError: Error? + var outputError: Error? + var errorError: Error? // lol + do { + try inputCloseFunc() + } catch { + inputError = error + } + + do { + try outputCloseFunc() + } catch { + outputError = error + } + + do { + try errorCloseFunc() + } catch { + errorError = error // lolol + } + + if let inputError = inputError { + throw inputError + } + + if let outputError = outputError { + throw outputError + } + + if let errorError = errorError { + throw errorError + } + } + + /// Close each input individually, and throw the first error if there's multiple errors thrown + @Sendable + internal func cleanupAll( + input: ExecutionInput, + output: ExecutionOutput, + error: ExecutionOutput + ) throws { + var inputError: Error? + var outputError: Error? + var errorError: Error? + + do { + try input.closeAll() + } catch { + inputError = error + } + + do { + try output.closeAll() + } catch { + outputError = error + } + + do { + try error.closeAll() + } catch { + errorError = error + } + + if let inputError = inputError { + throw inputError + } + if let outputError = outputError { + throw outputError + } + if let errorError = errorError { + throw errorError + } + } + + internal func run( + output: RedirectedOutputMethod, + error: RedirectedOutputMethod, + _ body: sending @escaping (Subprocess, StandardInputWriter) async throws -> R + ) async throws -> ExecutionResult { + let (readFd, writeFd) = try FileDescriptor.pipe() + let executionInput: ExecutionInput = .init(storage: .customWrite(readFd, writeFd)) + let executionOutput: ExecutionOutput = try output.createExecutionOutput() + let executionError: ExecutionOutput = try error.createExecutionOutput() + let process: Subprocess = try self.spawn( + withInput: executionInput, + output: executionOutput, + error: executionError) + // After spawn, cleanup child side fds + try await self.cleanup( + process: process, + childSide: true, + parentSide: false, + attemptToTerminateSubProcess: false + ) + return try await withAsyncTaskCancellationHandler { + return try await withThrowingTaskGroup(of: RunState.self) { group in + group.addTask { + let status = try await monitorProcessTermination( + forProcessWithIdentifier: process.processIdentifier) + return .monitorChildProcess(status) + } + group.addTask { + do { + let result = try await body(process, .init(input: executionInput)) + try await self.cleanup( + process: process, + childSide: false, + parentSide: true, + attemptToTerminateSubProcess: false + ) + return .workBody(result) + } catch { + // Cleanup everything + try await self.cleanup( + process: process, + childSide: false, + parentSide: true, + attemptToTerminateSubProcess: false + ) + throw error + } + } + + var result: R! + var terminationStatus: TerminationStatus! + while let state = try await group.next() { + switch state { + case .monitorChildProcess(let status): + // We don't really care about termination status here + terminationStatus = status + case .workBody(let workResult): + result = workResult + } + } + return ExecutionResult(terminationStatus: terminationStatus, value: result) + } + } onCancel: { + // Attempt to terminate the child process + // Since the task has already been cancelled, + // this is the best we can do + try? await self.cleanup( + process: process, + childSide: true, + parentSide: true, + attemptToTerminateSubProcess: true + ) + } + } + + internal func run( + input: InputMethod, + output: RedirectedOutputMethod, + error: RedirectedOutputMethod, + _ body: (sending @escaping (Subprocess) async throws -> R) + ) async throws -> ExecutionResult { + let executionInput = try input.createExecutionInput() + let executionOutput = try output.createExecutionOutput() + let executionError = try error.createExecutionOutput() + let process = try self.spawn( + withInput: executionInput, + output: executionOutput, + error: executionError) + // After spawn, clean up child side + try await self.cleanup( + process: process, + childSide: true, + parentSide: false, + attemptToTerminateSubProcess: false + ) + return try await withAsyncTaskCancellationHandler { + return try await withThrowingTaskGroup(of: RunState.self) { group in + group.addTask { + let status = try await monitorProcessTermination( + forProcessWithIdentifier: process.processIdentifier) + return .monitorChildProcess(status) + } + group.addTask { + do { + let result = try await body(process) + try await self.cleanup( + process: process, + childSide: false, + parentSide: true, + attemptToTerminateSubProcess: false + ) + return .workBody(result) + } catch { + try await self.cleanup( + process: process, + childSide: false, + parentSide: true, + attemptToTerminateSubProcess: false + ) + throw error + } + } + + var result: R! + var terminationStatus: TerminationStatus! + while let state = try await group.next() { + switch state { + case .monitorChildProcess(let status): + terminationStatus = status + case .workBody(let workResult): + result = workResult + } + } + return ExecutionResult(terminationStatus: terminationStatus, value: result) + } + } onCancel: { + // Attempt to terminate the child process + // Since the task has already been cancelled, + // this is the best we can do + try? await self.cleanup( + process: process, + childSide: true, + parentSide: true, + attemptToTerminateSubProcess: true + ) + } + } + } +} + +extension Subprocess.Configuration : CustomStringConvertible, CustomDebugStringConvertible { + public var description: String { + return """ +Subprocess.Configuration( + executable: \(self.executable.description), + arguments: \(self.arguments.description), + environment: \(self.environment.description), + workingDirectory: \(self.workingDirectory), + platformOptions: \(self.platformOptions.description(withIndent: 1)) +) +""" + } + + public var debugDescription: String { + return """ +Subprocess.Configuration( + executable: \(self.executable.debugDescription), + arguments: \(self.arguments.debugDescription), + environment: \(self.environment.debugDescription), + workingDirectory: \(self.workingDirectory), + platformOptions: \(self.platformOptions.description(withIndent: 1)) +) +""" + } +} + +// MARK: - Executable +extension Subprocess { + /// `Subprocess.Executable` defines how should the executable + /// be looked up for execution. + public struct Executable: Sendable, Hashable { + internal enum Configuration: Sendable, Hashable { + case executable(String) + case path(FilePath) + } + + internal let storage: Configuration + + private init(_config: Configuration) { + self.storage = _config + } + + /// Locate the executable by its name. + /// `Subprocess` will use `PATH` value to + /// determine the full path to the executable. + public static func named(_ executableName: String) -> Self { + return .init(_config: .executable(executableName)) + } + /// Locate the executable by its full path. + /// `Subprocess` will use this path directly. + public static func at(_ filePath: FilePath) -> Self { + return .init(_config: .path(filePath)) + } + /// Returns the full executable path given the environment value. + public func resolveExecutablePath(in environment: Environment) -> FilePath? { + if let path = self.resolveExecutablePath(withPathValue: environment.pathValue()) { + return FilePath(path) + } + return nil + } + } +} + +extension Subprocess.Executable : CustomStringConvertible, CustomDebugStringConvertible { + public var description: String { + switch storage { + case .executable(let executableName): + return executableName + case .path(let filePath): + return filePath.string + } + } + + public var debugDescription: String { + switch storage { + case .executable(let string): + return "executable(\(string))" + case .path(let filePath): + return "path(\(filePath.string))" + } + } +} + +// MARK: - Arguments +extension Subprocess { + /// A collection of arguments to pass to the subprocess. + public struct Arguments: Sendable, ExpressibleByArrayLiteral, Hashable { + public typealias ArrayLiteralElement = String + + internal let storage: [StringOrRawBytes] + internal let executablePathOverride: StringOrRawBytes? + + /// Create an Arguments object using the given literal values + public init(arrayLiteral elements: String...) { + self.storage = elements.map { .string($0) } + self.executablePathOverride = nil + } + /// Create an Arguments object using the given array + public init(_ array: [String]) { + self.storage = array.map { .string($0) } + self.executablePathOverride = nil + } + +#if !os(Windows) // Windows does NOT support arg0 override + /// Create an `Argument` object using the given values, but + /// override the first Argument value to `executablePathOverride`. + /// If `executablePathOverride` is nil, + /// `Arguments` will automatically use the executable path + /// as the first argument. + /// - Parameters: + /// - executablePathOverride: the value to override the first argument. + /// - remainingValues: the rest of the argument value + public init(executablePathOverride: String?, remainingValues: [String]) { + self.storage = remainingValues.map { .string($0) } + if let executablePathOverride = executablePathOverride { + self.executablePathOverride = .string(executablePathOverride) + } else { + self.executablePathOverride = nil + } + } + + /// Create an `Argument` object using the given values, but + /// override the first Argument value to `executablePathOverride`. + /// If `executablePathOverride` is nil, + /// `Arguments` will automatically use the executable path + /// as the first argument. + /// - Parameters: + /// - executablePathOverride: the value to override the first argument. + /// - remainingValues: the rest of the argument value + public init(executablePathOverride: Data?, remainingValues: [Data]) { + self.storage = remainingValues.map { .rawBytes($0.toArray()) } + if let override = executablePathOverride { + self.executablePathOverride = .rawBytes(override.toArray()) + } else { + self.executablePathOverride = nil + } + } + + public init(_ array: [Data]) { + self.storage = array.map { .rawBytes($0.toArray()) } + self.executablePathOverride = nil + } +#endif + } +} + +extension Subprocess.Arguments : CustomStringConvertible, CustomDebugStringConvertible { + public var description: String { + var result: [String] = self.storage.map(\.description) + + if let override = self.executablePathOverride { + result.insert("override\(override.description)", at: 0) + } + return result.description + } + + public var debugDescription: String { return self.description } +} + +// MARK: - Environment +extension Subprocess { + /// A set of environment variables to use when executing the subprocess. + public struct Environment: Sendable, Hashable { + internal enum Configuration: Sendable, Hashable { + case inherit([StringOrRawBytes : StringOrRawBytes]) + case custom([StringOrRawBytes : StringOrRawBytes]) + } + + internal let config: Configuration + + init(config: Configuration) { + self.config = config + } + /// Child process should inherit the same environment + /// values from its parent process. + public static var inherit: Self { + return .init(config: .inherit([:])) + } + /// Override the provided `newValue` in the existing `Environment` + public func updating(_ newValue: [String : String]) -> Self { + return .init(config: .inherit(newValue.wrapToStringOrRawBytes())) + } + /// Use custom environment variables + public static func custom(_ newValue: [String : String]) -> Self { + return .init(config: .custom(newValue.wrapToStringOrRawBytes())) + } + +#if !os(Windows) + /// Override the provided `newValue` in the existing `Environment` + public func updating(_ newValue: [Data : Data]) -> Self { + return .init(config: .inherit(newValue.wrapToStringOrRawBytes())) + } + /// Use custom environment variables + public static func custom(_ newValue: [Data : Data]) -> Self { + return .init(config: .custom(newValue.wrapToStringOrRawBytes())) + } +#endif + } +} + +extension Subprocess.Environment : CustomStringConvertible, CustomDebugStringConvertible { + public var description: String { + switch self.config { + case .custom(let customDictionary): + return customDictionary.dictionaryDescription + case .inherit(let updateValue): + return "Inherting current environment with updates: \(updateValue.dictionaryDescription)" + } + } + + public var debugDescription: String { + return self.description + } +} + +fileprivate extension Dictionary where Key == String, Value == String { + func wrapToStringOrRawBytes() -> [Subprocess.StringOrRawBytes : Subprocess.StringOrRawBytes] { + var result = Dictionary< + Subprocess.StringOrRawBytes, + Subprocess.StringOrRawBytes + >(minimumCapacity: self.count) + for (key, value) in self { + result[.string(key)] = .string(value) + } + return result + } +} + +fileprivate extension Dictionary where Key == Data, Value == Data { + func wrapToStringOrRawBytes() -> [Subprocess.StringOrRawBytes : Subprocess.StringOrRawBytes] { + var result = Dictionary< + Subprocess.StringOrRawBytes, + Subprocess.StringOrRawBytes + >(minimumCapacity: self.count) + for (key, value) in self { + result[.rawBytes(key.toArray())] = .rawBytes(value.toArray()) + } + return result + } +} + +fileprivate extension Dictionary where Key == Subprocess.StringOrRawBytes, Value == Subprocess.StringOrRawBytes { + var dictionaryDescription: String { + var result = "[\n" + for (key, value) in self { + result += "\t\(key.description) : \(value.description),\n" + } + result += "]" + return result + } +} + +fileprivate extension Data { + func toArray() -> [T] { + return self.withUnsafeBytes { ptr in + return Array(ptr.bindMemory(to: T.self)) + } + } +} + +// MARK: - TerminationStatus +extension Subprocess { + /// An exit status of a subprocess. + @frozen + public enum TerminationStatus: Sendable, Hashable, Codable { + #if canImport(WinSDK) + public typealias Code = DWORD + #else + public typealias Code = CInt + #endif + + /// The subprocess was existed with the given code + case exited(Code) + /// The subprocess was signalled with given exception value + case unhandledException(Code) + /// Whether the current TerminationStatus is successful. + public var isSuccess: Bool { + switch self { + case .exited(let exitCode): + return exitCode == 0 + case .unhandledException(_): + return false + } + } + } +} + +extension Subprocess.TerminationStatus : CustomStringConvertible, CustomDebugStringConvertible { + public var description: String { + switch self { + case .exited(let code): + return "exited(\(code))" + case .unhandledException(let code): + return "unhandledException(\(code))" + } + } + + public var debugDescription: String { + return self.description + } +} + +// MARK: - Internal +extension Subprocess { + internal enum StringOrRawBytes: Sendable, Hashable { + case string(String) + case rawBytes([CChar]) + + // Return value needs to be deallocated manually by callee + func createRawBytes() -> UnsafeMutablePointer { + switch self { + case .string(let string): + return strdup(string) + case .rawBytes(let rawBytes): + return strdup(rawBytes) + } + } + + var stringValue: String? { + switch self { + case .string(let string): + return string + case .rawBytes(let rawBytes): + return String(validatingUTF8: rawBytes) + } + } + + var description: String { + switch self { + case .string(let string): + return string + case .rawBytes(let bytes): + return bytes.description + } + } + + var count: Int { + switch self { + case .string(let string): + return string.count + case .rawBytes(let rawBytes): + return strnlen(rawBytes, Int.max) + } + } + + func hash(into hasher: inout Hasher) { + // If Raw bytes is valid UTF8, hash it as so + switch self { + case .string(let string): + hasher.combine(string) + case .rawBytes(let bytes): + if let stringValue = self.stringValue { + hasher.combine(stringValue) + } else { + hasher.combine(bytes) + } + } + } + } +} + +extension FilePath { + static var currentWorkingDirectory: Self { + let path = getcwd(nil, 0)! + defer { free(path) } + return .init(String(cString: path)) + } +} + +extension Optional where Wrapped : Collection { + func withOptionalUnsafeBufferPointer(_ body: ((UnsafeBufferPointer)?) throws -> R) rethrows -> R { + switch self { + case .some(let wrapped): + guard let array: Array = wrapped as? Array else { + return try body(nil) + } + return try array.withUnsafeBufferPointer { ptr in + return try body(ptr) + } + case .none: + return try body(nil) + } + } +} + +extension Optional where Wrapped == String { + func withOptionalCString(_ body: ((UnsafePointer)?) throws -> R) rethrows -> R { + switch self { + case .none: + return try body(nil) + case .some(let wrapped): + return try wrapped.withCString { + return try body($0) + } + } + } + + var stringValue: String { + return self ?? "nil" + } +} + +// MARK: - Stubs for the one from Foundation +public enum QualityOfService: Int, Sendable { + case userInteractive = 0x21 + case userInitiated = 0x19 + case utility = 0x11 + case background = 0x09 + case `default` = -1 +} + +internal func withAsyncTaskCancellationHandler( + _ body: sending @escaping () async throws -> R, + onCancel handler: sending @escaping () async -> Void +) async rethrows -> R { + return try await withThrowingTaskGroup( + of: R?.self, + returning: R.self + ) { group in + group.addTask { + return try await body() + } + group.addTask { + // wait until cancelled + do { while true { try await Task.sleep(nanoseconds: 1_000_000_000) } } catch {} + // Run task cancel handler + await handler() + return nil + } + + while let result = try await group.next() { + if let result = result { + // As soon as the body finishes, cancel the group + group.cancelAll() + return result + } + } + fatalError("Unreachable") + } +} + diff --git a/Sources/_Subprocess/Subprocess+IO.swift b/Sources/_Subprocess/Subprocess+IO.swift new file mode 100644 index 00000000..1b43e421 --- /dev/null +++ b/Sources/_Subprocess/Subprocess+IO.swift @@ -0,0 +1,437 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if canImport(FoundationEssentials) +import FoundationEssentials +#elseif canImport(Foundation) +import Foundation +#endif + +import Dispatch +import SystemPackage + +// Naive Mutex so we don't have to update the macOS dependency +final class _Mutex: Sendable { + let lock = Lock() + + var value: Value + + init(_ value: Value) { + self.value = value + } + + func withLock(_ body: (inout Value) throws -> R) rethrows -> R { + self.lock.lock() + defer { self.lock.unlock() } + return try body(&self.value) + } +} + +// MARK: - Input +extension Subprocess { + /// `InputMethod` defines how should the standard input + /// of the subprocess receive inputs. + public struct InputMethod: Sendable, Hashable { + internal enum Storage: Sendable, Hashable { + case noInput + case fileDescriptor(FileDescriptor, Bool) + } + + internal let method: Storage + + internal init(method: Storage) { + self.method = method + } + + internal func createExecutionInput() throws -> ExecutionInput { + switch self.method { + case .noInput: + let devnull: FileDescriptor = try .openDevNull(withAcessMode: .readOnly) + return .init(storage: .noInput(devnull)) + case .fileDescriptor(let fileDescriptor, let closeWhenDone): + return .init(storage: .fileDescriptor(fileDescriptor, closeWhenDone)) + } + } + + /// Subprocess should read no input. This option is equivalent + /// to bind the stanard input to `/dev/null`. + public static var noInput: Self { + return .init(method: .noInput) + } + + /// Subprocess should read input from a given file descriptor. + /// - Parameters: + /// - fd: the file descriptor to read from + /// - closeAfterSpawningProcess: whether the file descriptor + /// should be automatically closed after subprocess is spawned. + public static func readFrom( + _ fd: FileDescriptor, + closeAfterSpawningProcess: Bool + ) -> Self { + return .init(method: .fileDescriptor(fd, closeAfterSpawningProcess)) + } + } +} + +extension Subprocess { + /// `CollectedOutputMethod` defines how should Subprocess collect + /// output from child process' standard output and standard error + public struct CollectedOutputMethod: Sendable, Hashable { + internal enum Storage: Sendable, Hashable { + case discarded + case fileDescriptor(FileDescriptor, Bool) + case collected(Int) + } + + internal let method: Storage + + internal init(method: Storage) { + self.method = method + } + + /// Subprocess shold dicard the child process output. + /// This option is equivalent to binding the child process + /// output to `/dev/null`. + public static var discard: Self { + return .init(method: .discarded) + } + /// Subprocess should write the child process output + /// to the file descriptor specified. + /// - Parameters: + /// - fd: the file descriptor to write to + /// - closeAfterSpawningProcess: whether to close the + /// file descriptor once the process is spawned. + public static func writeTo(_ fd: FileDescriptor, closeAfterSpawningProcess: Bool) -> Self { + return .init(method: .fileDescriptor(fd, closeAfterSpawningProcess)) + } + /// Subprocess should collect the child process output + /// as `Data` with the given limit in bytes. The default + /// limit is 128kb. + public static func collect(upTo limit: Int = 128 * 1024) -> Self { + return .init(method: .collected(limit)) + } + + internal func createExecutionOutput() throws -> ExecutionOutput { + switch self.method { + case .discarded: + // Bind to /dev/null + let devnull: FileDescriptor = try .openDevNull(withAcessMode: .writeOnly) + return .init(storage: .discarded(devnull)) + case .fileDescriptor(let fileDescriptor, let closeWhenDone): + return .init(storage: .fileDescriptor(fileDescriptor, closeWhenDone)) + case .collected(let limit): + let (readFd, writeFd) = try FileDescriptor.pipe() + return .init(storage: .collected(limit, readFd, writeFd)) + } + } + } + + /// `CollectedOutputMethod` defines how should Subprocess redirect + /// output from child process' standard output and standard error. + public struct RedirectedOutputMethod: Sendable, Hashable { + typealias Storage = CollectedOutputMethod.Storage + + internal let method: Storage + + internal init(method: Storage) { + self.method = method + } + + /// Subprocess shold dicard the child process output. + /// This option is equivalent to binding the child process + /// output to `/dev/null`. + public static var discard: Self { + return .init(method: .discarded) + } + /// Subprocess should redirect the child process output + /// to `Subprocess.standardOutput` or `Subprocess.standardError` + /// so they can be consumed as an AsyncSequence + public static var redirectToSequence: Self { + return .init(method: .collected(128 * 1024)) + } + /// Subprocess shold write the child process output + /// to the file descriptor specified. + /// - Parameters: + /// - fd: the file descriptor to write to + /// - closeAfterSpawningProcess: whether to close the + /// file descriptor once the process is spawned. + public static func writeTo( + _ fd: FileDescriptor, + closeAfterSpawningProcess: Bool + ) -> Self { + return .init(method: .fileDescriptor(fd, closeAfterSpawningProcess)) + } + + internal func createExecutionOutput() throws -> ExecutionOutput { + switch self.method { + case .discarded: + // Bind to /dev/null + let devnull: FileDescriptor = try .openDevNull(withAcessMode: .writeOnly) + return .init(storage: .discarded(devnull)) + case .fileDescriptor(let fileDescriptor, let closeWhenDone): + return .init(storage: .fileDescriptor(fileDescriptor, closeWhenDone)) + case .collected(let limit): + let (readFd, writeFd) = try FileDescriptor.pipe() + return .init(storage: .collected(limit, readFd, writeFd)) + } + } + } +} + +// MARK: - Execution IO +extension Subprocess { + internal final class ExecutionInput: Sendable, Hashable { + + + internal enum Storage: Sendable, Hashable { + case noInput(FileDescriptor?) + case customWrite(FileDescriptor?, FileDescriptor?) + case fileDescriptor(FileDescriptor?, Bool) + } + + let storage: _Mutex + + internal init(storage: Storage) { + self.storage = .init(storage) + } + + internal func getReadFileDescriptor() -> FileDescriptor? { + return self.storage.withLock { + switch $0 { + case .noInput(let readFd): + return readFd + case .customWrite(let readFd, _): + return readFd + case .fileDescriptor(let readFd, _): + return readFd + } + } + } + + internal func getWriteFileDescriptor() -> FileDescriptor? { + return self.storage.withLock { + switch $0 { + case .noInput(_), .fileDescriptor(_, _): + return nil + case .customWrite(_, let writeFd): + return writeFd + } + } + } + + internal func closeChildSide() throws { + try self.storage.withLock { + switch $0 { + case .noInput(let devnull): + try devnull?.close() + $0 = .noInput(nil) + case .customWrite(let readFd, let writeFd): + try readFd?.close() + $0 = .customWrite(nil, writeFd) + case .fileDescriptor(let fd, let closeWhenDone): + // User passed in fd + if closeWhenDone { + try fd?.close() + $0 = .fileDescriptor(nil, closeWhenDone) + } + } + } + } + + internal func closeParentSide() throws { + try self.storage.withLock { + switch $0 { + case .noInput(_), .fileDescriptor(_, _): + break + case .customWrite(let readFd, let writeFd): + // The parent fd should have been closed + // in the `body` when writer.finish() is called + // But in case it isn't call it agian + try writeFd?.close() + $0 = .customWrite(readFd, nil) + } + } + } + + internal func closeAll() throws { + try self.storage.withLock { + switch $0 { + case .noInput(let readFd): + try readFd?.close() + $0 = .noInput(nil) + case .customWrite(let readFd, let writeFd): + var readFdCloseError: Error? + var writeFdCloseError: Error? + do { + try readFd?.close() + } catch { + readFdCloseError = error + } + do { + try writeFd?.close() + } catch { + writeFdCloseError = error + } + $0 = .customWrite(nil, nil) + if let readFdCloseError { + throw readFdCloseError + } + if let writeFdCloseError { + throw writeFdCloseError + } + case .fileDescriptor(let fd, let closeWhenDone): + if closeWhenDone { + try fd?.close() + $0 = .fileDescriptor(nil, closeWhenDone) + } + } + } + } + + public func hash(into hasher: inout Hasher) { + self.storage.withLock { + hasher.combine($0) + } + } + + public static func == ( + lhs: Subprocess.ExecutionInput, + rhs: Subprocess.ExecutionInput + ) -> Bool { + return lhs.storage.withLock { lhsStorage in + rhs.storage.withLock { rhsStorage in + return lhsStorage == rhsStorage + } + } + } + } + + internal final class ExecutionOutput: Sendable { + internal enum Storage: Sendable { + case discarded(FileDescriptor?) + case fileDescriptor(FileDescriptor?, Bool) + case collected(Int, FileDescriptor?, FileDescriptor?) + } + + private let storage: _Mutex + + internal init(storage: Storage) { + self.storage = .init(storage) + } + + internal func getWriteFileDescriptor() -> FileDescriptor? { + return self.storage.withLock { + switch $0 { + case .discarded(let writeFd): + return writeFd + case .fileDescriptor(let writeFd, _): + return writeFd + case .collected(_, _, let writeFd): + return writeFd + } + } + } + + internal func getReadFileDescriptor() -> FileDescriptor? { + return self.storage.withLock { + switch $0 { + case .discarded(_), .fileDescriptor(_, _): + return nil + case .collected(_, let readFd, _): + return readFd + } + } + } + + internal func consumeCollectedFileDescriptor() -> (limit: Int, fd: FileDescriptor?)? { + return self.storage.withLock { + switch $0 { + case .discarded(_), .fileDescriptor(_, _): + // The output has been written somewhere else + return nil + case .collected(let limit, let readFd, let writeFd): + $0 = .collected(limit, nil, writeFd) + return (limit, readFd) + } + } + } + + internal func closeChildSide() throws { + try self.storage.withLock { + switch $0 { + case .discarded(let writeFd): + try writeFd?.close() + $0 = .discarded(nil) + case .fileDescriptor(let fd, let closeWhenDone): + // User passed fd + if closeWhenDone { + try fd?.close() + $0 = .fileDescriptor(nil, closeWhenDone) + } + case .collected(let limit, let readFd, let writeFd): + try writeFd?.close() + $0 = .collected(limit, readFd, nil) + } + } + } + + internal func closeParentSide() throws { + try self.storage.withLock { + switch $0 { + case .discarded(_), .fileDescriptor(_, _): + break + case .collected(let limit, let readFd, let writeFd): + try readFd?.close() + $0 = .collected(limit, nil, writeFd) + } + } + } + + internal func closeAll() throws { + try self.storage.withLock { + switch $0 { + case .discarded(let writeFd): + try writeFd?.close() + $0 = .discarded(nil) + case .fileDescriptor(let fd, let closeWhenDone): + if closeWhenDone { + try fd?.close() + $0 = .fileDescriptor(nil, closeWhenDone) + } + case .collected(let limit, let readFd, let writeFd): + var readFdCloseError: Error? + var writeFdCloseError: Error? + do { + try readFd?.close() + } catch { + readFdCloseError = error + } + do { + try writeFd?.close() + } catch { + writeFdCloseError = error + } + $0 = .collected(limit, nil, nil) + if let readFdCloseError { + throw readFdCloseError + } + if let writeFdCloseError { + throw writeFdCloseError + } + } + } + } + } +} + diff --git a/Sources/_Subprocess/Subprocess+Teardown.swift b/Sources/_Subprocess/Subprocess+Teardown.swift new file mode 100644 index 00000000..8b9af32d --- /dev/null +++ b/Sources/_Subprocess/Subprocess+Teardown.swift @@ -0,0 +1,125 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if canImport(Darwin) || canImport(Glibc) + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +#if canImport(FoundationEssentials) +import FoundationEssentials +#elseif canImport(Foundation) +import Foundation +#endif + +extension Subprocess { + /// A step in the graceful shutdown teardown sequence. + /// It consists of a signal to send to the child process and the + /// number of nanoseconds allowed for the child process to exit + /// before proceeding to the next step. + public struct TeardownStep: Sendable, Hashable { + internal enum Storage: Sendable, Hashable { + case sendSignal(Signal, allowedNanoseconds: UInt64) + case kill + } + var storage: Storage + + /// Sends `signal` to the process and provides `allowedNanoSecondsToExit` + /// nanoseconds for the process to exit before proceeding to the next step. + /// The final step in the sequence will always send a `.kill` signal. + public static func sendSignal( + _ signal: Signal, + allowedNanoSecondsToExit: UInt64 + ) -> Self { + return Self( + storage: .sendSignal( + signal, + allowedNanoseconds: allowedNanoSecondsToExit + ) + ) + } + } +} + +extension Subprocess { + internal func runTeardownSequence(_ sequence: [TeardownStep]) async { + // First insert the `.kill` step + let finalSequence = sequence + [TeardownStep(storage: .kill)] + for step in finalSequence { + enum TeardownStepCompletion { + case processHasExited + case processStillAlive + case killedTheProcess + } + let stepCompletion: TeardownStepCompletion + + guard self.isAlive() else { + return + } + + switch step.storage { + case .sendSignal(let signal, let allowedNanoseconds): + stepCompletion = await withTaskGroup(of: TeardownStepCompletion.self) { group in + group.addTask { + do { + try await Task.sleep(nanoseconds: allowedNanoseconds) + return .processStillAlive + } catch { + // teardown(using:) cancells this task + // when process has exited + return .processHasExited + } + } + try? self.send(signal, toProcessGroup: false) + return await group.next()! + } + case .kill: + try? self.send(.kill, toProcessGroup: false) + stepCompletion = .killedTheProcess + } + + switch stepCompletion { + case .killedTheProcess, .processHasExited: + return + case .processStillAlive: + // Continue to next step + break + } + } + } +} + +extension Subprocess { + private func isAlive() -> Bool { + return kill(self.processIdentifier.value, 0) == 0 + } +} + +func withUncancelledTask( + returning: R.Type = R.self, + _ body: @Sendable @escaping () async -> R +) async -> R { + // This looks unstructured but it isn't, please note that we `await` `.value` of this task. + // The reason we need this separate `Task` is that in general, we cannot assume that code performs to our + // expectations if the task we run it on is already cancelled. However, in some cases we need the code to + // run regardless -- even if our task is already cancelled. Therefore, we create a new, uncancelled task here. + await Task { + await body() + }.value +} + +#endif // canImport(Darwin) || canImport(Glibc) diff --git a/Sources/_Subprocess/Subprocess.swift b/Sources/_Subprocess/Subprocess.swift new file mode 100644 index 00000000..35f4a2c2 --- /dev/null +++ b/Sources/_Subprocess/Subprocess.swift @@ -0,0 +1,315 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import SystemPackage + +#if canImport(FoundationEssentials) +import FoundationEssentials +#elseif canImport(Foundation) +import Foundation +#endif + +/// An object that represents a subprocess of the current process. +/// +/// Using `Subprocess`, your program can run another program as a subprocess +/// and can monitor that program’s execution. A `Subprocess` object creates a +/// **separate executable** entity; it’s different from `Thread` because it doesn’t +/// share memory space with the process that creates it. +public struct Subprocess: Sendable { + /// The process identifier of the current subprocess + public let processIdentifier: ProcessIdentifier + + internal let executionInput: ExecutionInput + internal let executionOutput: ExecutionOutput + internal let executionError: ExecutionOutput +#if os(Windows) + internal let consoleBehavior: PlatformOptions.ConsoleBehavior +#endif + + /// The standard output of the subprocess. + /// Accessing this property will **fatalError** if + /// - `.output` wasn't set to `.redirectToSequence` when the subprocess was spawned; + /// - This property was accessed multiple times. Subprocess communicates with + /// parent process via pipe under the hood and each pipe can only be consumed ones. + public var standardOutput: some _AsyncSequence { + guard let (_, fd) = self.executionOutput + .consumeCollectedFileDescriptor() else { + fatalError("The standard output was not redirected") + } + guard let fd = fd else { + fatalError("The standard output has already been closed") + } + return AsyncDataSequence(fileDescriptor: fd) + } + + /// The standard error of the subprocess. + /// Accessing this property will **fatalError** if + /// - `.error` wasn't set to `.redirectToSequence` when the subprocess was spawned; + /// - This property was accessed multiple times. Subprocess communicates with + /// parent process via pipe under the hood and each pipe can only be consumed ones. + public var standardError: some _AsyncSequence { + guard let (_, fd) = self.executionError + .consumeCollectedFileDescriptor() else { + fatalError("The standard error was not redirected") + } + guard let fd = fd else { + fatalError("The standard error has already been closed") + } + return AsyncDataSequence(fileDescriptor: fd) + } +} + +// MARK: - Teardown +#if canImport(Darwin) || canImport(Glibc) +extension Subprocess { + /// Performs a sequence of teardown steps on the Subprocess. + /// Teardown sequence always ends with a `.kill` signal + /// - Parameter sequence: The steps to perform. + public func teardown(using sequence: [TeardownStep]) async { + await withUncancelledTask { + await self.runTeardownSequence(sequence) + } + } +} +#endif + +// MARK: - StandardInputWriter +extension Subprocess { + /// A writer that writes to the standard input of the subprocess. + public struct StandardInputWriter { + + private let input: ExecutionInput + + init(input: ExecutionInput) { + self.input = input + } + + /// Write a sequence of UInt8 to the standard input of the subprocess. + /// - Parameter sequence: The sequence of bytes to write. + public func write(_ sequence: S) async throws where S : Sequence, S.Element == UInt8 { + guard let fd: FileDescriptor = self.input.getWriteFileDescriptor() else { + fatalError("Attempting to write to a file descriptor that's already closed") + } + try await fd.write(sequence) + } + + /// Write a sequence of CChar to the standard input of the subprocess. + /// - Parameter sequence: The sequence of bytes to write. + public func write(_ sequence: S) async throws where S : Sequence, S.Element == CChar { + try await self.write(sequence.map { UInt8($0) }) + } + + /// Write a AsyncSequence of CChar to the standard input of the subprocess. + /// - Parameter sequence: The sequence of bytes to write. + public func write(_ asyncSequence: S) async throws where S.Element == CChar { + let sequence = try await Array(asyncSequence).map { UInt8($0) } + try await self.write(sequence) + } + + /// Write a AsyncSequence of UInt8 to the standard input of the subprocess. + /// - Parameter sequence: The sequence of bytes to write. + public func write(_ asyncSequence: S) async throws where S.Element == UInt8 { + let sequence = try await Array(asyncSequence) + try await self.write(sequence) + } + + /// Signal all writes are finished + public func finish() async throws { + try self.input.closeParentSide() + } + } +} + +@available(macOS, unavailable) +@available(iOS, unavailable) +@available(tvOS, unavailable) +@available(watchOS, unavailable) +@available(*, unavailable) +extension Subprocess.StandardInputWriter : Sendable {} + +// MARK: - Result +extension Subprocess { + /// A simple wrapper around the generic result returned by the + /// `run` closures with the corresponding `TerminationStatus` + /// of the child process. + public struct ExecutionResult: Sendable { + /// The termination status of the child process + public let terminationStatus: TerminationStatus + /// The result returned by the closure passed to `.run` methods + public let value: T + + internal init(terminationStatus: TerminationStatus, value: T) { + self.terminationStatus = terminationStatus + self.value = value + } + } + + /// The result of a subprocess execution with its collected + /// standard output and standard error. + public struct CollectedResult: Sendable, Hashable, Codable { + /// The process identifier for the executed subprocess + public let processIdentifier: ProcessIdentifier + /// The termination status of the executed subprocess + public let terminationStatus: TerminationStatus + private let _standardOutput: Data? + private let _standardError: Data? + + /// The collected standard output value for the subprocess. + /// Accessing this property will *fatalError* if the + /// corresponding `CollectedOutputMethod` is not set to + /// `.collect` or `.collect(upTo:)` + public var standardOutput: Data { + guard let output = self._standardOutput else { + fatalError("standardOutput is only available if the Subprocess was ran with .collect as output") + } + return output + } + /// The collected standard error value for the subprocess. + /// Accessing this property will *fatalError* if the + /// corresponding `CollectedOutputMethod` is not set to + /// `.collect` or `.collect(upTo:)` + public var standardError: Data { + guard let output = self._standardError else { + fatalError("standardError is only available if the Subprocess was ran with .collect as error ") + } + return output + } + + internal init( + processIdentifier: ProcessIdentifier, + terminationStatus: TerminationStatus, + standardOutput: Data?, + standardError: Data?) { + self.processIdentifier = processIdentifier + self.terminationStatus = terminationStatus + self._standardOutput = standardOutput + self._standardError = standardError + } + } +} + +extension Subprocess.ExecutionResult: Equatable where T : Equatable {} + +extension Subprocess.ExecutionResult: Hashable where T : Hashable {} + +extension Subprocess.ExecutionResult: Codable where T : Codable {} + +extension Subprocess.ExecutionResult: CustomStringConvertible where T : CustomStringConvertible { + public var description: String { + return """ +Subprocess.ExecutionResult( + terminationStatus: \(self.terminationStatus.description), + value: \(self.value.description) +) +""" + } +} + +extension Subprocess.ExecutionResult: CustomDebugStringConvertible where T : CustomDebugStringConvertible { + public var debugDescription: String { + return """ +Subprocess.ExecutionResult( + terminationStatus: \(self.terminationStatus.debugDescription), + value: \(self.value.debugDescription) +) +""" + } +} + +extension Subprocess.CollectedResult : CustomStringConvertible, CustomDebugStringConvertible { + public var description: String { + return """ +Subprocess.CollectedResult( + processIdentifier: \(self.processIdentifier.description), + terminationStatus: \(self.terminationStatus.description), + standardOutput: \(self._standardOutput?.description ?? "not captured"), + standardError: \(self._standardError?.description ?? "not captured") +) +""" + } + + public var debugDescription: String { + return """ +Subprocess.CollectedResult( + processIdentifier: \(self.processIdentifier.debugDescription), + terminationStatus: \(self.terminationStatus.debugDescription), + standardOutput: \(self._standardOutput?.debugDescription ?? "not captured"), + standardError: \(self._standardError?.debugDescription ?? "not captured") +) +""" + } +} + +// MARK: - Internal +extension Subprocess { + internal enum OutputCapturingState { + case standardOutputCaptured(Data?) + case standardErrorCaptured(Data?) + } + + internal typealias CapturedIOs = (standardOutput: Data?, standardError: Data?) + + private func capture(fileDescriptor: FileDescriptor, maxLength: Int) async throws -> Data { + return try await fileDescriptor.readUntilEOF(upToLength: maxLength) + } + + internal func captureStandardOutput() async throws -> Data? { + guard let (limit, readFd) = self.executionOutput + .consumeCollectedFileDescriptor(), + let readFd = readFd else { + return nil + } + defer { + try? readFd.close() + } + return try await self.capture(fileDescriptor: readFd, maxLength: limit) + } + + internal func captureStandardError() async throws -> Data? { + guard let (limit, readFd) = self.executionError + .consumeCollectedFileDescriptor(), + let readFd = readFd else { + return nil + } + defer { + try? readFd.close() + } + return try await self.capture(fileDescriptor: readFd, maxLength: limit) + } + + internal func captureIOs() async throws -> CapturedIOs { + return try await withThrowingTaskGroup(of: OutputCapturingState.self) { group in + group.addTask { + let stdout = try await self.captureStandardOutput() + return .standardOutputCaptured(stdout) + } + group.addTask { + let stderr = try await self.captureStandardError() + return .standardErrorCaptured(stderr) + } + + var stdout: Data? + var stderror: Data? + while let state = try await group.next() { + switch state { + case .standardOutputCaptured(let output): + stdout = output + case .standardErrorCaptured(let error): + stderror = error + } + } + return (standardOutput: stdout, standardError: stderror) + } + } +} diff --git a/Sources/_Subprocess/_nio_locks.swift b/Sources/_Subprocess/_nio_locks.swift new file mode 100644 index 00000000..49053d04 --- /dev/null +++ b/Sources/_Subprocess/_nio_locks.swift @@ -0,0 +1,526 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if canImport(Darwin) +import Darwin +#elseif os(Windows) +import ucrt +import WinSDK +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#else +#error("The concurrency lock module was unable to identify your C library.") +#endif + +/// A threading lock based on `libpthread` instead of `libdispatch`. +/// +/// This object provides a lock on top of a single `pthread_mutex_t`. This kind +/// of lock is safe to use with `libpthread`-based threading models, such as the +/// one used by NIO. On Windows, the lock is based on the substantially similar +/// `SRWLOCK` type. +@available(*, deprecated, renamed: "NIOLock") +public final class Lock { +#if os(Windows) + fileprivate let mutex: UnsafeMutablePointer = + UnsafeMutablePointer.allocate(capacity: 1) +#else + fileprivate let mutex: UnsafeMutablePointer = + UnsafeMutablePointer.allocate(capacity: 1) +#endif + + /// Create a new lock. + public init() { +#if os(Windows) + InitializeSRWLock(self.mutex) +#else + var attr = pthread_mutexattr_t() + pthread_mutexattr_init(&attr) + debugOnly { + pthread_mutexattr_settype(&attr, .init(PTHREAD_MUTEX_ERRORCHECK)) + } + + let err = pthread_mutex_init(self.mutex, &attr) + precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)") +#endif + } + + deinit { +#if os(Windows) + // SRWLOCK does not need to be free'd +#else + let err = pthread_mutex_destroy(self.mutex) + precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)") +#endif + mutex.deallocate() + } + + /// Acquire the lock. + /// + /// Whenever possible, consider using `withLock` instead of this method and + /// `unlock`, to simplify lock handling. + public func lock() { +#if os(Windows) + AcquireSRWLockExclusive(self.mutex) +#else + let err = pthread_mutex_lock(self.mutex) + precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)") +#endif + } + + /// Release the lock. + /// + /// Whenever possible, consider using `withLock` instead of this method and + /// `lock`, to simplify lock handling. + public func unlock() { +#if os(Windows) + ReleaseSRWLockExclusive(self.mutex) +#else + let err = pthread_mutex_unlock(self.mutex) + precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)") +#endif + } + + /// Acquire the lock for the duration of the given block. + /// + /// This convenience method should be preferred to `lock` and `unlock` in + /// most situations, as it ensures that the lock will be released regardless + /// of how `body` exits. + /// + /// - Parameter body: The block to execute while holding the lock. + /// - Returns: The value returned by the block. + @inlinable + public func withLock(_ body: () throws -> T) rethrows -> T { + self.lock() + defer { + self.unlock() + } + return try body() + } + + // specialise Void return (for performance) + @inlinable + public func withLockVoid(_ body: () throws -> Void) rethrows -> Void { + try self.withLock(body) + } +} + +/// A `Lock` with a built-in state variable. +/// +/// This class provides a convenience addition to `Lock`: it provides the ability to wait +/// until the state variable is set to a specific value to acquire the lock. +public final class ConditionLock { + private var _value: T + private let mutex: NIOLock +#if os(Windows) + private let cond: UnsafeMutablePointer = + UnsafeMutablePointer.allocate(capacity: 1) +#else + private let cond: UnsafeMutablePointer = + UnsafeMutablePointer.allocate(capacity: 1) +#endif + + /// Create the lock, and initialize the state variable to `value`. + /// + /// - Parameter value: The initial value to give the state variable. + public init(value: T) { + self._value = value + self.mutex = NIOLock() +#if os(Windows) + InitializeConditionVariable(self.cond) +#else + let err = pthread_cond_init(self.cond, nil) + precondition(err == 0, "\(#function) failed in pthread_cond with error \(err)") +#endif + } + + deinit { +#if os(Windows) + // condition variables do not need to be explicitly destroyed +#else + let err = pthread_cond_destroy(self.cond) + precondition(err == 0, "\(#function) failed in pthread_cond with error \(err)") +#endif + self.cond.deallocate() + } + + /// Acquire the lock, regardless of the value of the state variable. + public func lock() { + self.mutex.lock() + } + + /// Release the lock, regardless of the value of the state variable. + public func unlock() { + self.mutex.unlock() + } + + /// The value of the state variable. + /// + /// Obtaining the value of the state variable requires acquiring the lock. + /// This means that it is not safe to access this property while holding the + /// lock: it is only safe to use it when not holding it. + public var value: T { + self.lock() + defer { + self.unlock() + } + return self._value + } + + /// Acquire the lock when the state variable is equal to `wantedValue`. + /// + /// - Parameter wantedValue: The value to wait for the state variable + /// to have before acquiring the lock. + public func lock(whenValue wantedValue: T) { + self.lock() + while true { + if self._value == wantedValue { + break + } + self.mutex.withLockPrimitive { mutex in +#if os(Windows) + let result = SleepConditionVariableSRW(self.cond, mutex, INFINITE, 0) + precondition(result, "\(#function) failed in SleepConditionVariableSRW with error \(GetLastError())") +#else + let err = pthread_cond_wait(self.cond, mutex) + precondition(err == 0, "\(#function) failed in pthread_cond with error \(err)") +#endif + } + } + } + + /// Acquire the lock when the state variable is equal to `wantedValue`, + /// waiting no more than `timeoutSeconds` seconds. + /// + /// - Parameter wantedValue: The value to wait for the state variable + /// to have before acquiring the lock. + /// - Parameter timeoutSeconds: The number of seconds to wait to acquire + /// the lock before giving up. + /// - Returns: `true` if the lock was acquired, `false` if the wait timed out. + public func lock(whenValue wantedValue: T, timeoutSeconds: Double) -> Bool { + precondition(timeoutSeconds >= 0) + +#if os(Windows) + var dwMilliseconds: DWORD = DWORD(timeoutSeconds * 1000) + + self.lock() + while true { + if self._value == wantedValue { + return true + } + + let dwWaitStart = timeGetTime() + if !SleepConditionVariableSRW(self.cond, self.mutex._storage.mutex, + dwMilliseconds, 0) { + let dwError = GetLastError() + if (dwError == ERROR_TIMEOUT) { + self.unlock() + return false + } + fatalError("SleepConditionVariableSRW: \(dwError)") + } + + // NOTE: this may be a spurious wakeup, adjust the timeout accordingly + dwMilliseconds = dwMilliseconds - (timeGetTime() - dwWaitStart) + } +#else + let nsecPerSec: Int64 = 1000000000 + self.lock() + /* the timeout as a (seconds, nano seconds) pair */ + let timeoutNS = Int64(timeoutSeconds * Double(nsecPerSec)) + + var curTime = timeval() + gettimeofday(&curTime, nil) + + let allNSecs: Int64 = timeoutNS + Int64(curTime.tv_usec) * 1000 + var timeoutAbs = timespec(tv_sec: curTime.tv_sec + Int((allNSecs / nsecPerSec)), + tv_nsec: Int(allNSecs % nsecPerSec)) + assert(timeoutAbs.tv_nsec >= 0 && timeoutAbs.tv_nsec < Int(nsecPerSec)) + assert(timeoutAbs.tv_sec >= curTime.tv_sec) + return self.mutex.withLockPrimitive { mutex -> Bool in + while true { + if self._value == wantedValue { + return true + } + switch pthread_cond_timedwait(self.cond, mutex, &timeoutAbs) { + case 0: + continue + case ETIMEDOUT: + self.unlock() + return false + case let e: + fatalError("caught error \(e) when calling pthread_cond_timedwait") + } + } + } +#endif + } + + /// Release the lock, setting the state variable to `newValue`. + /// + /// - Parameter newValue: The value to give to the state variable when we + /// release the lock. + public func unlock(withValue newValue: T) { + self._value = newValue + self.unlock() +#if os(Windows) + WakeAllConditionVariable(self.cond) +#else + let err = pthread_cond_broadcast(self.cond) + precondition(err == 0, "\(#function) failed in pthread_cond with error \(err)") +#endif + } +} + +/// A utility function that runs the body code only in debug builds, without +/// emitting compiler warnings. +/// +/// This is currently the only way to do this in Swift: see +/// https://forums.swift.org/t/support-debug-only-code/11037 for a discussion. +@inlinable +internal func debugOnly(_ body: () -> Void) { + assert({ body(); return true }()) +} + +@available(*, deprecated) +extension Lock: @unchecked Sendable {} +extension ConditionLock: @unchecked Sendable {} + +#if os(Windows) +@usableFromInline +typealias LockPrimitive = SRWLOCK +#else +@usableFromInline +typealias LockPrimitive = pthread_mutex_t +#endif + +@usableFromInline +enum LockOperations { } + +extension LockOperations { + @inlinable + static func create(_ mutex: UnsafeMutablePointer) { + mutex.assertValidAlignment() + +#if os(Windows) + InitializeSRWLock(mutex) +#else + var attr = pthread_mutexattr_t() + pthread_mutexattr_init(&attr) + debugOnly { + pthread_mutexattr_settype(&attr, .init(PTHREAD_MUTEX_ERRORCHECK)) + } + + let err = pthread_mutex_init(mutex, &attr) + precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)") +#endif + } + + @inlinable + static func destroy(_ mutex: UnsafeMutablePointer) { + mutex.assertValidAlignment() + +#if os(Windows) + // SRWLOCK does not need to be free'd +#else + let err = pthread_mutex_destroy(mutex) + precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)") +#endif + } + + @inlinable + static func lock(_ mutex: UnsafeMutablePointer) { + mutex.assertValidAlignment() + +#if os(Windows) + AcquireSRWLockExclusive(mutex) +#else + let err = pthread_mutex_lock(mutex) + precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)") +#endif + } + + @inlinable + static func unlock(_ mutex: UnsafeMutablePointer) { + mutex.assertValidAlignment() + +#if os(Windows) + ReleaseSRWLockExclusive(mutex) +#else + let err = pthread_mutex_unlock(mutex) + precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)") +#endif + } +} + +// Tail allocate both the mutex and a generic value using ManagedBuffer. +// Both the header pointer and the elements pointer are stable for +// the class's entire lifetime. +// +// However, for safety reasons, we elect to place the lock in the "elements" +// section of the buffer instead of the head. The reasoning here is subtle, +// so buckle in. +// +// _As a practical matter_, the implementation of ManagedBuffer ensures that +// the pointer to the header is stable across the lifetime of the class, and so +// each time you call `withUnsafeMutablePointers` or `withUnsafeMutablePointerToHeader` +// the value of the header pointer will be the same. This is because ManagedBuffer uses +// `Builtin.addressOf` to load the value of the header, and that does ~magic~ to ensure +// that it does not invoke any weird Swift accessors that might copy the value. +// +// _However_, the header is also available via the `.header` field on the ManagedBuffer. +// This presents a problem! The reason there's an issue is that `Builtin.addressOf` and friends +// do not interact with Swift's exclusivity model. That is, the various `with` functions do not +// conceptually trigger a mutating access to `.header`. For elements this isn't a concern because +// there's literally no other way to perform the access, but for `.header` it's entirely possible +// to accidentally recursively read it. +// +// Our implementation is free from these issues, so we don't _really_ need to worry about it. +// However, out of an abundance of caution, we store the Value in the header, and the LockPrimitive +// in the trailing elements. We still don't use `.header`, but it's better to be safe than sorry, +// and future maintainers will be happier that we were cautious. +// +// See also: https://github.com/apple/swift/pull/40000 +@usableFromInline +final class LockStorage: ManagedBuffer { + + @inlinable + static func create(value: Value) -> Self { + let buffer = Self.create(minimumCapacity: 1) { _ in + return value + } + let storage = unsafeDowncast(buffer, to: Self.self) + + storage.withUnsafeMutablePointers { _, lockPtr in + LockOperations.create(lockPtr) + } + + return storage + } + + @inlinable + func lock() { + self.withUnsafeMutablePointerToElements { lockPtr in + LockOperations.lock(lockPtr) + } + } + + @inlinable + func unlock() { + self.withUnsafeMutablePointerToElements { lockPtr in + LockOperations.unlock(lockPtr) + } + } + + @inlinable + deinit { + self.withUnsafeMutablePointerToElements { lockPtr in + LockOperations.destroy(lockPtr) + } + } + + @inlinable + func withLockPrimitive(_ body: (UnsafeMutablePointer) throws -> T) rethrows -> T { + try self.withUnsafeMutablePointerToElements { lockPtr in + return try body(lockPtr) + } + } + + @inlinable + func withLockedValue(_ mutate: (inout Value) throws -> T) rethrows -> T { + try self.withUnsafeMutablePointers { valuePtr, lockPtr in + LockOperations.lock(lockPtr) + defer { LockOperations.unlock(lockPtr) } + return try mutate(&valuePtr.pointee) + } + } +} + +extension LockStorage: @unchecked Sendable { } + +/// A threading lock based on `libpthread` instead of `libdispatch`. +/// +/// - note: ``NIOLock`` has reference semantics. +/// +/// This object provides a lock on top of a single `pthread_mutex_t`. This kind +/// of lock is safe to use with `libpthread`-based threading models, such as the +/// one used by NIO. On Windows, the lock is based on the substantially similar +/// `SRWLOCK` type. +public struct NIOLock { + @usableFromInline + internal let _storage: LockStorage + + /// Create a new lock. + @inlinable + public init() { + self._storage = .create(value: ()) + } + + /// Acquire the lock. + /// + /// Whenever possible, consider using `withLock` instead of this method and + /// `unlock`, to simplify lock handling. + @inlinable + public func lock() { + self._storage.lock() + } + + /// Release the lock. + /// + /// Whenever possible, consider using `withLock` instead of this method and + /// `lock`, to simplify lock handling. + @inlinable + public func unlock() { + self._storage.unlock() + } + + @inlinable + internal func withLockPrimitive(_ body: (UnsafeMutablePointer) throws -> T) rethrows -> T { + return try self._storage.withLockPrimitive(body) + } +} + +extension NIOLock { + /// Acquire the lock for the duration of the given block. + /// + /// This convenience method should be preferred to `lock` and `unlock` in + /// most situations, as it ensures that the lock will be released regardless + /// of how `body` exits. + /// + /// - Parameter body: The block to execute while holding the lock. + /// - Returns: The value returned by the block. + @inlinable + public func withLock(_ body: () throws -> T) rethrows -> T { + self.lock() + defer { + self.unlock() + } + return try body() + } + + @inlinable + public func withLockVoid(_ body: () throws -> Void) rethrows -> Void { + try self.withLock(body) + } +} + +extension NIOLock: Sendable {} + +extension UnsafeMutablePointer { + @inlinable + func assertValidAlignment() { + assert(UInt(bitPattern: self) % UInt(MemoryLayout.alignment) == 0) + } +}