Fix test output parser to be incremental and show failed tests (#505)

Author: kateinoigakukun

Sources/CartonHelpers/Process+run.swift

@@ -17,21 +17,12 @@ import Dispatch
 import Foundation
 
 struct ProcessError: Error {
-  let stderr: String?
-  let stdout: String?
+  let exitCode: Int32
 }
 
 extension ProcessError: CustomStringConvertible {
   var description: String {
-    var result = "Process failed with non-zero exit status"
-    if let stdout = stdout {
-      result += " and following output:\n\(stdout)"
-    }
-
-    if let stderr = stderr {
-      result += " and following error output:\n\(stderr)"
-    }
-    return result
+    return "Process failed with exit code \(exitCode)"
   }
 }
 
@@ -41,11 +32,10 @@ extension Foundation.Process {
     _ arguments: [String],
     environment: [String: String] = [:],
     loadingMessage: String = "Running...",
-    parser: ProcessOutputParser? = nil,
     _ terminal: InteractiveWriter
   ) async throws {
     terminal.clearLine()
-    terminal.write("\(loadingMessage)\n", inColor: .yellow)
+    terminal.write("Running \(arguments.joined(separator: " "))\n")
 
     if !environment.isEmpty {
       terminal.write(environment.map { "\($0)=\($1)" }.joined(separator: " ") + " ")
@@ -54,91 +44,26 @@ extension Foundation.Process {
     let processName = URL(fileURLWithPath: arguments[0]).lastPathComponent
 
     do {
-      try await withCheckedThrowingContinuation {
-        (continuation: CheckedContinuation<(), Swift.Error>) in
-        DispatchQueue.global().async {
-          var stdoutBuffer = ""
-
-          let stdout: Process.OutputClosure = {
-            guard let string = String(data: Data($0), encoding: .utf8) else { return }
-            if parser != nil {
-              // Aggregate this for formatting later
-              stdoutBuffer += string
-            } else {
-              terminal.write(string)
-            }
-          }
-
-          var stderrBuffer = [UInt8]()
-
-          let stderr: Process.OutputClosure = {
-            stderrBuffer.append(contentsOf: $0)
-          }
-
-          let process = Process(
-            arguments: arguments,
-            environmentBlock: ProcessEnvironmentBlock(
-              ProcessInfo.processInfo.environment.merging(environment) { _, new in new }
-            ),
-            outputRedirection: .stream(stdout: stdout, stderr: stderr),
-            startNewProcessGroup: true,
-            loggingHandler: {
-              terminal.write($0 + "\n")
-            }
-          )
-
-          let result = Result<ProcessResult, Swift.Error> {
-            try process.launch()
-            return try process.waitUntilExit()
-          }
-
-          switch result.map(\.exitStatus) {
-          case .success(.terminated(code: EXIT_SUCCESS)):
-            if let parser = parser {
-              if parser.parsingConditions.contains(.success) {
-                parser.parse(stdoutBuffer, terminal)
-              }
-            } else {
-              terminal.write(stdoutBuffer)
-            }
-            terminal.write(
-              "`\(processName)` process finished successfully\n",
-              inColor: .green,
-              bold: false
-            )
-            continuation.resume()
-
-          case let .failure(error):
-            continuation.resume(throwing: error)
-          default:
-            continuation.resume(
-              throwing: ProcessError(
-                stderr: String(data: Data(stderrBuffer), encoding: .utf8) ?? "",
-                stdout: stdoutBuffer
-              )
-            )
-          }
+      try await Process.checkNonZeroExit(
+        arguments: arguments,
+        environmentBlock: ProcessEnvironmentBlock(
+          ProcessInfo.processInfo.environment.merging(environment) { _, new in new }
+        ),
+        loggingHandler: {
+          terminal.write($0 + "\n")
         }
-      }
+      )
+      terminal.write(
+        "`\(processName)` process finished successfully\n",
+        inColor: .green,
+        bold: false
+      )
     } catch {
-      let errorString = String(describing: error)
-      if errorString.isEmpty {
-        terminal.clearLine()
-        terminal.write(
-          "\(processName) process failed.\n\n",
-          inColor: .red
-        )
-        if let error = error as? ProcessError, let stdout = error.stdout {
-          if let parser = parser {
-            if parser.parsingConditions.contains(.failure) {
-              parser.parse(stdout, terminal)
-            }
-          } else {
-            terminal.write(stdout)
-          }
-        }
-      }
-
+      terminal.clearLine()
+      terminal.write(
+        "\(processName) process failed.\n\n",
+        inColor: .red
+      )
       throw error
     }
   }

Sources/CartonKit/Parsers/DiagnosticsParser.swift

@@ -75,6 +75,9 @@ struct CartonFrontendTestCommand: AsyncParsableCommand {
   @Flag(help: "When running browser tests, run the browser in headless mode")
   var headless: Bool = false
 
+  @Flag(help: "Enable verbose output")
+  var verbose: Bool = false
+
   @Option(help: "Turn on runtime checks for various behavior.")
   private var sanitize: SanitizeVariant?
 
@@ -195,6 +198,8 @@ struct CartonFrontendTestCommand: AsyncParsableCommand {
         env[key] = parentEnv[key]
       }
     }
-    return TestRunnerOptions(env: env, listTestCases: list, testCases: testCases)
+    return TestRunnerOptions(
+      env: env, listTestCases: list, testCases: testCases,
+      testsParser: verbose ? RawTestsParser() : FancyTestsParser())
   }
 }

Sources/carton-frontend-slim/CartonFrontendTestCommand.swift

@@ -49,7 +49,7 @@ struct CommandTestRunner: TestRunner {
     }
 
     arguments += [testFilePath.pathString] + xctestArgs
-    try await Process.run(arguments, parser: TestsParser(), terminal)
+    try await runTestProcess(arguments, parser: options.testsParser, terminal)
   }
 
   func defaultWASIRuntime() throws -> String {

Sources/carton-frontend-slim/TestRunners/CommandTestRunner.swift

@@ -58,6 +58,7 @@ struct JavaScriptTestRunner: TestRunner {
     var arguments =
       ["node"] + nodeArguments + [pluginWorkDirectory.appending(component: testHarness).pathString]
     options.applyXCTestArguments(to: &arguments)
-    try await Process.run(arguments, environment: options.env, parser: TestsParser(), terminal)
+    try await runTestProcess(
+      arguments, environment: options.env, parser: options.testsParser, terminal)
   }
 }

Sources/carton-frontend-slim/TestRunners/JavaScriptTestRunner.swift

@@ -12,13 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import CartonCore
+import CartonHelpers
+import Foundation
+
 struct TestRunnerOptions {
   /// The environment variables to pass to the test process.
   let env: [String: String]
   /// When specified, list all available test cases.
   let listTestCases: Bool
   /// Filter the test cases to run.
   let testCases: [String]
+  /// The parser to use for the test output.
+  let testsParser: any TestsParser
 
   func applyXCTestArguments(to arguments: inout [String]) {
     if listTestCases {
@@ -32,3 +38,58 @@ struct TestRunnerOptions {
 protocol TestRunner {
   func run(options: TestRunnerOptions) async throws
 }
+
+struct LineStream {
+  var buffer: String = ""
+  let onLine: (String) -> Void
+
+  mutating func feed(_ bytes: [UInt8]) {
+    buffer += String(decoding: bytes, as: UTF8.self)
+    while let newlineIndex = buffer.firstIndex(of: "\n") {
+      let line = buffer[..<newlineIndex]
+      buffer.removeSubrange(buffer.startIndex...newlineIndex)
+      onLine(String(line))
+    }
+  }
+}
+
+extension TestRunner {
+  func runTestProcess(
+    _ arguments: [String],
+    environment: [String: String] = [:],
+    parser: any TestsParser,
+    _ terminal: InteractiveWriter
+  ) async throws {
+    do {
+      terminal.clearLine()
+      let commandLine = arguments.map { "\"\($0)\"" }.joined(separator: " ")
+      terminal.write("Running \(commandLine)\n")
+
+      let (lines, continuation) = AsyncStream.makeStream(
+        of: String.self, bufferingPolicy: .unbounded
+      )
+      var lineStream = LineStream { line in
+        continuation.yield(line)
+      }
+      let process = Process(
+        arguments: arguments,
+        environmentBlock: ProcessEnvironmentBlock(
+          ProcessInfo.processInfo.environment.merging(environment) { _, new in new }
+        ),
+        outputRedirection: .stream(
+          stdout: { bytes in
+            lineStream.feed(bytes)
+          }, stderr: { _ in },
+          redirectStderr: true
+        ),
+        startNewProcessGroup: true
+      )
+      async let _ = parser.parse(lines, terminal)
+      try process.launch()
+      let result = try await process.waitUntilExit()
+      guard result.exitStatus == .terminated(code: 0) else {
+        throw ProcessResult.Error.nonZeroExit(result)
+      }
+    }
+  }
+}