From 383becf990b688ff0544e97cac2caf0346dd007d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sakib=20Had=C5=BEiavdi=C4=87?= Date: Fri, 7 Feb 2025 11:18:05 +0100 Subject: [PATCH 1/4] Consolidate jvm subprocess and inprocess functions (#4456) Changes: - added functions based on #3772 description - the process related ones now return os-lib process result or subprocess handle - `callClassLoader` uses a loan pattern function and closes the classloader after usage - added `mill.util.ProcessUtil.toResult(processResult)` to handle process result in same way https://github.com/com-lihaoyi/mill/blob/main/main/util/src/mill/util/Jvm.scala#L352-L357 Some minor questions/suggestions: 1. current `spawnClassloader` is more flexible, you can pass your own classloader rather that use `getClass.getClassLoader`, this is ok I guess? 1. suggestion to name `spawnClassLoader` as `createClassLoader` 1. suggestion to name `callClassloader` as `withClassloader` (common loan pattern name) 1. `getMainMethod` is not public so `runLocal` can't be removed.. so expose `getMainMethod` or leave `runLocal` as is? 1. Some calls are inside build.mill files. I have to wait for new version of Mill with these changes included, and then update them? 1. Not sure if `closeClassLoaderWhenDone` is needed, I don't understand why it is left unclosed in few places. Closes #3772 --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- .../bsp/worker/MillScalaBuildServer.scala | 27 +- .../jmh/src/mill/contrib/jmh/JmhModule.scala | 21 +- .../src/mill/contrib/proguard/Proguard.scala | 8 +- .../src/mill/kotlinlib/KotlinModule.scala | 8 +- .../android/AndroidAppKotlinModule.scala | 10 +- .../mill/kotlinlib/detekt/DetektModule.scala | 9 +- .../mill/kotlinlib/js/KotlinJsModule.scala | 28 +- .../mill/kotlinlib/kover/KoverModule.scala | 8 +- .../mill/kotlinlib/ktfmt/KtfmtModule.scala | 9 +- .../mill/kotlinlib/ktlint/KtlintModule.scala | 9 +- main/api/src/mill/api/ClassLoader.scala | 1 + main/init/src/mill/init/BuildGenModule.scala | 12 +- main/src/mill/main/VisualizeModule.scala | 10 +- main/util/src/mill/util/Jvm.scala | 260 ++++++++++++++++++ .../src/mill/pythonlib/PythonModule.scala | 31 ++- .../javalib/android/AndroidAppModule.scala | 6 +- .../javalib/checkstyle/CheckstyleModule.scala | 9 +- .../palantirformat/PalantirFormatModule.scala | 8 +- .../mill/javalib/revapi/RevapiModule.scala | 9 +- scalalib/src/mill/scalalib/JavaModule.scala | 10 +- scalalib/src/mill/scalalib/RunModule.scala | 118 +++++--- scalalib/src/mill/scalalib/ScalaModule.scala | 18 +- scalalib/src/mill/scalalib/TestModule.scala | 86 +++--- .../src/mill/scalalib/TestModuleUtil.scala | 25 +- .../mill/scalalib/giter8/Giter8Module.scala | 10 +- .../scalalib/publish/SonatypeHelpers.scala | 11 +- .../src/mill/scalalib/AssemblyTestUtils.scala | 21 +- .../scalanativelib/ScalaNativeModule.scala | 12 +- .../mill/testrunner/DiscoverTestsMain.scala | 33 +-- .../mill/testrunner/GetTestTasksMain.scala | 32 +-- .../src/mill/testrunner/TestRunner.scala | 14 +- 31 files changed, 620 insertions(+), 253 deletions(-) diff --git a/bsp/worker/src/mill/bsp/worker/MillScalaBuildServer.scala b/bsp/worker/src/mill/bsp/worker/MillScalaBuildServer.scala index 08dded47156..8ab739b7ce8 100644 --- a/bsp/worker/src/mill/bsp/worker/MillScalaBuildServer.scala +++ b/bsp/worker/src/mill/bsp/worker/MillScalaBuildServer.scala @@ -119,21 +119,18 @@ private trait MillScalaBuildServer extends ScalaBuildServer { this: MillBuildSer ) { case (ev, state, id, m: TestModule, Some((classpath, testFramework, testClasspath))) => val (frameworkName, classFingerprint): (String, Agg[(Class[_], Fingerprint)]) = - Jvm.inprocess( - classpath.map(_.path), - classLoaderOverrideSbtTesting = true, - isolated = true, - closeContextClassLoaderWhenDone = false, - cl => { - val framework = Framework.framework(testFramework)(cl) - val discoveredTests = TestRunnerUtils.discoverTests( - cl, - framework, - Agg.from(testClasspath.map(_.path)) - ) - (framework.name(), discoveredTests) - } - )(new mill.api.Ctx.Home { def home = os.home }) + Jvm.withClassLoader( + classPath = classpath.map(_.path).toVector, + sharedPrefixes = Seq("sbt.testing.") + ) { classLoader => + val framework = Framework.framework(testFramework)(classLoader) + val discoveredTests = TestRunnerUtils.discoverTests( + classLoader, + framework, + Agg.from(testClasspath.map(_.path)) + ) + (framework.name(), discoveredTests) + } val classes = Seq.from(classFingerprint.map(classF => classF._1.getName.stripSuffix("$"))) new ScalaTestClassesItem(id, classes.asJava).tap { it => it.setFramework(frameworkName) diff --git a/contrib/jmh/src/mill/contrib/jmh/JmhModule.scala b/contrib/jmh/src/mill/contrib/jmh/JmhModule.scala index 1ac56b6769b..ef8f6e28d26 100644 --- a/contrib/jmh/src/mill/contrib/jmh/JmhModule.scala +++ b/contrib/jmh/src/mill/contrib/jmh/JmhModule.scala @@ -38,14 +38,17 @@ trait JmhModule extends JavaModule { def runJmh(args: String*) = Task.Command { val (_, resources) = generateBenchmarkSources() - Jvm.runSubprocess( - "org.openjdk.jmh.Main", + Jvm.callProcess( + mainClass = "org.openjdk.jmh.Main", classPath = (runClasspath() ++ generatorDeps()).map(_.path) ++ Seq(compileGeneratedSources().path, resources), mainArgs = args, - workingDir = Task.ctx().dest, - javaHome = zincWorker().javaHome().map(_.path) + cwd = Task.ctx().dest, + javaHome = zincWorker().javaHome().map(_.path), + stdin = os.Inherit, + stdout = os.Inherit ) + () } def listJmhBenchmarks(args: String*) = runJmh(("-l" +: args): _*) @@ -82,9 +85,9 @@ trait JmhModule extends JavaModule { os.remove.all(resourcesDir) os.makeDir.all(resourcesDir) - Jvm.runSubprocess( - "org.openjdk.jmh.generators.bytecode.JmhBytecodeGenerator", - (runClasspath() ++ generatorDeps()).map(_.path), + Jvm.callProcess( + mainClass = "org.openjdk.jmh.generators.bytecode.JmhBytecodeGenerator", + classPath = (runClasspath() ++ generatorDeps()).map(_.path), mainArgs = Seq( compile().classes.path.toString, sourcesDir.toString, @@ -92,7 +95,9 @@ trait JmhModule extends JavaModule { "default" ), javaHome = zincWorker().javaHome().map(_.path), - jvmArgs = forkedArgs + jvmArgs = forkedArgs, + stdin = os.Inherit, + stdout = os.Inherit ) (sourcesDir, resourcesDir) diff --git a/contrib/proguard/src/mill/contrib/proguard/Proguard.scala b/contrib/proguard/src/mill/contrib/proguard/Proguard.scala index 744faa3e758..8d7c3db52b9 100644 --- a/contrib/proguard/src/mill/contrib/proguard/Proguard.scala +++ b/contrib/proguard/src/mill/contrib/proguard/Proguard.scala @@ -111,11 +111,13 @@ trait Proguard extends ScalaModule { // val result = os.proc(cmd).call(stdout = Task.dest / "stdout.txt", stderr = Task.dest / "stderr.txt") // Task.log.debug(s"result: ${result}") - Jvm.runSubprocess( + Jvm.callProcess( mainClass = "proguard.ProGuard", - classPath = proguardClasspath().map(_.path), + classPath = proguardClasspath().map(_.path).toVector, mainArgs = args, - workingDir = Task.dest + cwd = Task.dest, + stdin = os.Inherit, + stdout = os.Inherit ) // the call above already throws an exception on a non-zero exit code, diff --git a/kotlinlib/src/mill/kotlinlib/KotlinModule.scala b/kotlinlib/src/mill/kotlinlib/KotlinModule.scala index c54e38b8a91..58cee482078 100644 --- a/kotlinlib/src/mill/kotlinlib/KotlinModule.scala +++ b/kotlinlib/src/mill/kotlinlib/KotlinModule.scala @@ -179,11 +179,13 @@ trait KotlinModule extends JavaModule { outer => Task.log.info("dokka options: " + options) - Jvm.runSubprocess( + Jvm.callProcess( mainClass = "", - classPath = Agg.empty, + classPath = Seq.empty, jvmArgs = Seq("-jar", dokkaCliClasspath().head.path.toString()), - mainArgs = options + mainArgs = options, + stdin = os.Inherit, + stdout = os.Inherit ) } diff --git a/kotlinlib/src/mill/kotlinlib/android/AndroidAppKotlinModule.scala b/kotlinlib/src/mill/kotlinlib/android/AndroidAppKotlinModule.scala index f6d11880830..74423e91ce0 100644 --- a/kotlinlib/src/mill/kotlinlib/android/AndroidAppKotlinModule.scala +++ b/kotlinlib/src/mill/kotlinlib/android/AndroidAppKotlinModule.scala @@ -243,14 +243,18 @@ trait AndroidAppKotlinModule extends AndroidAppModule with KotlinModule { outer * @return */ def generatePreviews: T[Agg[PathRef]] = Task { - val previewGenOut = mill.util.Jvm.callSubprocess( + val previewGenOut = mill.util.Jvm.callProcess( mainClass = "com.android.tools.render.compose.MainKt", - classPath = composePreviewRenderer().map(_.path) ++ layoutLibRenderer().map(_.path), + classPath = + composePreviewRenderer().map(_.path).toVector ++ layoutLibRenderer().map(_.path).toVector, jvmArgs = Seq( "-Dlayoutlib.thread.profile.timeoutms=10000", "-Djava.security.manager=allow" ), - mainArgs = Seq(composePreviewArgs().path.toString()) + mainArgs = Seq(composePreviewArgs().path.toString()), + cwd = Task.dest, + stdin = os.Inherit, + stdout = os.Inherit ).out.lines() Task.log.info(previewGenOut.mkString("\n")) diff --git a/kotlinlib/src/mill/kotlinlib/detekt/DetektModule.scala b/kotlinlib/src/mill/kotlinlib/detekt/DetektModule.scala index 92118e3bee0..3229dcf8d2b 100644 --- a/kotlinlib/src/mill/kotlinlib/detekt/DetektModule.scala +++ b/kotlinlib/src/mill/kotlinlib/detekt/DetektModule.scala @@ -27,12 +27,13 @@ trait DetektModule extends KotlinModule { Task.log.info("running detekt ...") Task.log.debug(s"with $args") - Jvm.callSubprocess( + Jvm.callProcess( mainClass = "io.gitlab.arturbosch.detekt.cli.Main", - classPath = detektClasspath().map(_.path), + classPath = detektClasspath().map(_.path).toVector, mainArgs = args, - workingDir = millSourcePath, // allow passing relative paths for sources like src/a/b - streamOut = true, + cwd = millSourcePath, // allow passing relative paths for sources like src/a/b + stdin = os.Inherit, + stdout = os.Inherit, check = false ).exitCode } diff --git a/kotlinlib/src/mill/kotlinlib/js/KotlinJsModule.scala b/kotlinlib/src/mill/kotlinlib/js/KotlinJsModule.scala index 3e1c5473e7d..e0bf373aec9 100644 --- a/kotlinlib/src/mill/kotlinlib/js/KotlinJsModule.scala +++ b/kotlinlib/src/mill/kotlinlib/js/KotlinJsModule.scala @@ -161,12 +161,18 @@ trait KotlinJsModule extends KotlinModule { outer => case Some(RunTarget.Node) => val binaryPath = (binaryDir / s"$artifactId.${moduleKind.extension}") .toIO.getAbsolutePath - Jvm.runSubprocessWithResult( - commandArgs = Seq( - "node" - ) ++ args.value ++ Seq(binaryPath), - envArgs = envArgs, - workingDir = workingDir + val processResult = os.call( + cmd = Seq("node") ++ args.value ++ Seq(binaryPath), + env = envArgs, + cwd = workingDir, + stdin = os.Inherit, + stdout = os.Inherit, + check = false + ) + if (processResult.exitCode == 0) Result.Success(processResult.exitCode) + else Result.Failure( + "Interactive Subprocess Failed (exit code " + processResult.exitCode + ")", + Some(processResult.exitCode) ) case Some(x) => Result.Failure(s"Run target $x is not supported") @@ -474,10 +480,12 @@ trait KotlinJsModule extends KotlinModule { outer => // TODO may be optimized if there is a single folder for all modules // but may be problematic if modules use different NPM packages versions private def nodeModulesDir = Task(persistent = true) { - Jvm.runSubprocess( - commandArgs = Seq("npm", "install", "mocha@10.2.0", "source-map-support@0.5.21"), - envArgs = Task.env, - workingDir = Task.dest + os.call( + cmd = Seq("npm", "install", "mocha@10.2.0", "source-map-support@0.5.21"), + env = Task.env, + cwd = Task.dest, + stdin = os.Inherit, + stdout = os.Inherit ) PathRef(Task.dest) } diff --git a/kotlinlib/src/mill/kotlinlib/kover/KoverModule.scala b/kotlinlib/src/mill/kotlinlib/kover/KoverModule.scala index 0da60977c96..5382aead5ca 100644 --- a/kotlinlib/src/mill/kotlinlib/kover/KoverModule.scala +++ b/kotlinlib/src/mill/kotlinlib/kover/KoverModule.scala @@ -212,12 +212,14 @@ object Kover extends ExternalModule with KoverReportBaseModule { s"${reportPath.toString()}.xml" } else reportPath.toString() args ++= Seq(s"--${reportType.toString.toLowerCase(Locale.US)}", output) - Jvm.runSubprocess( + Jvm.callProcess( mainClass = "kotlinx.kover.cli.MainKt", - classPath = classpath, + classPath = classpath.toVector, jvmArgs = Seq.empty[String], mainArgs = args.result(), - workingDir = workingDir + cwd = workingDir, + stdin = os.Inherit, + stdout = os.Inherit ) PathRef(os.Path(output)) } diff --git a/kotlinlib/src/mill/kotlinlib/ktfmt/KtfmtModule.scala b/kotlinlib/src/mill/kotlinlib/ktfmt/KtfmtModule.scala index 505b2bad9ae..adffa1e3886 100644 --- a/kotlinlib/src/mill/kotlinlib/ktfmt/KtfmtModule.scala +++ b/kotlinlib/src/mill/kotlinlib/ktfmt/KtfmtModule.scala @@ -120,12 +120,13 @@ object KtfmtModule extends ExternalModule with KtfmtBaseModule with TaskModule { if (!format) args += "--set-exit-if-changed" args ++= sources.iterator.map(_.path.toString()) - val exitCode = Jvm.callSubprocess( + val exitCode = Jvm.callProcess( mainClass = "com.facebook.ktfmt.cli.Main", - classPath = classPath.map(_.path), + classPath = classPath.map(_.path).toVector, mainArgs = args.result(), - workingDir = millSourcePath, // allow passing relative paths for sources like src/a/b - streamOut = true, + cwd = millSourcePath, // allow passing relative paths for sources like src/a/b + stdin = os.Inherit, + stdout = os.Inherit, check = false ).exitCode diff --git a/kotlinlib/src/mill/kotlinlib/ktlint/KtlintModule.scala b/kotlinlib/src/mill/kotlinlib/ktlint/KtlintModule.scala index cfdfd8c3dd5..2fb7c628030 100644 --- a/kotlinlib/src/mill/kotlinlib/ktlint/KtlintModule.scala +++ b/kotlinlib/src/mill/kotlinlib/ktlint/KtlintModule.scala @@ -122,12 +122,13 @@ object KtlintModule extends ExternalModule with KtlintModule with TaskModule { .filter(f => os.exists(f) && (f.ext == "kt" || f.ext == "kts")) .map(_.toString()) - val exitCode = Jvm.callSubprocess( + val exitCode = Jvm.callProcess( mainClass = "com.pinterest.ktlint.Main", - classPath = classPath.map(_.path), + classPath = classPath.map(_.path).toVector, mainArgs = args.result(), - workingDir = millSourcePath, - streamOut = true, + cwd = millSourcePath, + stdin = os.Inherit, + stdout = os.Inherit, check = false ).exitCode diff --git a/main/api/src/mill/api/ClassLoader.scala b/main/api/src/mill/api/ClassLoader.scala index f5ce4dc35c2..8a56c95d8c7 100644 --- a/main/api/src/mill/api/ClassLoader.scala +++ b/main/api/src/mill/api/ClassLoader.scala @@ -18,6 +18,7 @@ object ClassLoader { } def java9OrAbove: Boolean = !System.getProperty("java.specification.version").startsWith("1.") + @deprecated("Use callClassLoader", "Mill 0.12.7") def create( urls: Seq[URL], parent: java.lang.ClassLoader, diff --git a/main/init/src/mill/init/BuildGenModule.scala b/main/init/src/mill/init/BuildGenModule.scala index d2b07362bf7..353cf003790 100644 --- a/main/init/src/mill/init/BuildGenModule.scala +++ b/main/init/src/mill/init/BuildGenModule.scala @@ -27,14 +27,16 @@ trait BuildGenModule extends TaskModule { val mainClass = buildGenMainClass() val classPath = buildGenClasspath().map(_.path) - val exit = Jvm.callSubprocess( + val exitCode = Jvm.callProcess( mainClass = mainClass, - classPath = classPath, + classPath = classPath.toVector, mainArgs = args, - workingDir = root + cwd = root, + stdin = os.Inherit, + stdout = os.Inherit ).exitCode - if (exit == 0) { + if (exitCode == 0) { val files = BuildGenUtil.buildFiles(root).map(PathRef(_)).toSeq val config = buildGenScalafmtConfig() Task.log.info("formatting Mill build files") @@ -42,7 +44,7 @@ trait BuildGenModule extends TaskModule { Task.log.info("init completed, run \"mill resolve _\" to list available tasks") } else { - throw BuildGenException(s"$mainClass exit($exit)") + throw BuildGenException(s"$mainClass exit($exitCode)") } } } diff --git a/main/src/mill/main/VisualizeModule.scala b/main/src/mill/main/VisualizeModule.scala index f73e0a84e5e..d3495d9ccd1 100644 --- a/main/src/mill/main/VisualizeModule.scala +++ b/main/src/mill/main/VisualizeModule.scala @@ -102,10 +102,12 @@ trait VisualizeModule extends mill.define.TaskModule { g = g.graphAttr().`with`(Rank.dir(RankDir.LEFT_TO_RIGHT)) - mill.util.Jvm.runSubprocess( - "mill.main.graphviz.GraphvizTools", - classpath().map(_.path), - mainArgs = Seq(s"${os.temp(g.toString)};$dest;txt,dot,json,png,svg") + mill.util.Jvm.callProcess( + mainClass = "mill.main.graphviz.GraphvizTools", + classPath = classpath().map(_.path).toVector, + mainArgs = Seq(s"${os.temp(g.toString)};$dest;txt,dot,json,png,svg"), + stdin = os.Inherit, + stdout = os.Inherit ) os.list(dest).sorted.map(PathRef(_)) diff --git a/main/util/src/mill/util/Jvm.scala b/main/util/src/mill/util/Jvm.scala index f786d1ab03b..2fa9181c985 100644 --- a/main/util/src/mill/util/Jvm.scala +++ b/main/util/src/mill/util/Jvm.scala @@ -7,6 +7,7 @@ import os.{ProcessOutput, SubProcess} import java.io._ import java.lang.reflect.Modifier +import java.net.URLClassLoader import java.nio.file.attribute.PosixFilePermission import java.nio.file.Files import scala.util.Properties.isWin @@ -14,10 +15,169 @@ import os.CommandResult object Jvm extends CoursierSupport { + /** + * Runs a JVM subprocess with the given configuration and returns a + * [[os.CommandResult]] with it's aggregated output and error streams. + * + * @param mainClass The main class to run + * @param mainArgs Args passed to the `mainClass` main method + * @param javaHome Optional Java Home override + * @param jvmArgs Arguments given to the forked JVM + * @param classPath The classpath + * @param cpPassingJarPath When `None`, the `-cp` parameter is used to pass the classpath + * to the forked JVM. + * When `Some`, a temporary empty JAR is created + * which contains a `Class-Path` manifest entry containing the actual classpath. + * This might help with long classpaths on OS'es (like Windows) + * which only supports limited command-line length + * @param env Environment variables used when starting the forked JVM + * @param propagateEnv If `true` then the current process' environment variables are propagated to subprocess + * @param cwd The working directory to be used by the forked JVM + * @param stdin Standard input + * @param stdout Standard output + * @param stderr Standard error + * @param mergeErrIntoOut If `true` then the error output is merged into standard output + * @param timeout how long to wait in milliseconds for the subprocess to complete (-1 for no timeout) + * @param shutdownGracePeriod if the timeout is enabled, how long in milliseconds for the subprocess + * to gracefully terminate before attempting to forcibly kill it + * (-1 for no kill, 0 for always kill immediately) + * @param destroyOnExit Destroy on JVM exit + * @param check if `true`, an exception will be thrown if process exits with a non-zero exit code + */ + def callProcess( + mainClass: String, + mainArgs: Iterable[String] = Seq.empty, + javaHome: Option[os.Path] = None, + jvmArgs: Iterable[String] = Seq.empty, + classPath: Iterable[os.Path], + cpPassingJarPath: Option[os.Path] = None, + env: Map[String, String] = Map.empty, + propagateEnv: Boolean = true, + cwd: os.Path = null, + stdin: os.ProcessInput = os.Pipe, + stdout: ProcessOutput = os.Pipe, + stderr: ProcessOutput = os.Inherit, + mergeErrIntoOut: Boolean = false, + timeout: Long = -1, + shutdownGracePeriod: Long = 100, + destroyOnExit: Boolean = true, + check: Boolean = true + ): CommandResult = { + val cp = cpPassingJarPath match { + case Some(passingJarPath) if classPath.nonEmpty => + createClasspathPassingJar(passingJarPath, classPath) + Seq(passingJarPath) + case _ => classPath + } + + val commandArgs = (Vector(javaExe(javaHome)) ++ + jvmArgs ++ + Option.when(cp.nonEmpty)(Vector( + "-cp", + cp.mkString(java.io.File.pathSeparator) + )).getOrElse(Vector.empty) ++ + Vector(mainClass) ++ + mainArgs).filterNot(_.isBlank) + + if (cwd != null) os.makeDir.all(cwd) + + val processResult = os.proc(commandArgs) + .call( + cwd = cwd, + env = env, + propagateEnv = propagateEnv, + stdin = stdin, + stdout = stdout, + stderr = stderr, + mergeErrIntoOut = mergeErrIntoOut, + timeout = timeout, + shutdownGracePeriod = shutdownGracePeriod, + destroyOnExit = destroyOnExit, + check = check + ) + processResult + } + + /** + * Runs a JVM subprocess with the given configuration and streams + * it's stdout and stderr to the console. + * + * @param mainClass The main class to run + * @param mainArgs Args passed to the `mainClass` main method + * @param javaHome Optional Java Home override + * @param jvmArgs Arguments given to the forked JVM + * @param classPath The classpath + * @param cpPassingJarPath When `None`, the `-cp` parameter is used to pass the classpath + * to the forked JVM. + * When `Some`, a temporary empty JAR is created + * which contains a `Class-Path` manifest entry containing the actual classpath. + * This might help with long classpaths on OS'es (like Windows) + * which only supports limited command-line length + * @param env Environment variables used when starting the forked JVM + * @param propagateEnv If `true` then the current process' environment variables are propagated to subprocess + * @param cwd The working directory to be used by the forked JVM + * @param stdin Standard input override + * @param stdout Standard output override + * @param stderr Standard error override + * @param mergeErrIntoOut If `true` then the error output is merged into standard output + * @param shutdownGracePeriod if the timeout is enabled, how long in milliseconds for the subprocess + * to gracefully terminate before attempting to forcibly kill it + * (-1 for no kill, 0 for always kill immediately) + * @param destroyOnExit Destroy on JVM exit + */ + def spawnProcess( + mainClass: String, + mainArgs: Iterable[String] = Seq.empty, + javaHome: Option[os.Path] = None, + jvmArgs: Iterable[String] = Seq.empty, + classPath: Iterable[os.Path], + cpPassingJarPath: Option[os.Path] = None, + env: Map[String, String] = Map.empty, + propagateEnv: Boolean = true, + cwd: os.Path = null, + stdin: os.ProcessInput = os.Pipe, + stdout: ProcessOutput = os.Pipe, + stderr: ProcessOutput = os.Inherit, + mergeErrIntoOut: Boolean = false, + shutdownGracePeriod: Long = 100, + destroyOnExit: Boolean = true + ): os.SubProcess = { + val cp = cpPassingJarPath match { + case Some(passingJarPath) if classPath.nonEmpty => + createClasspathPassingJar(passingJarPath, classPath) + Seq(passingJarPath) + case _ => classPath + } + + val commandArgs = (Vector(javaExe(javaHome)) ++ + jvmArgs ++ + Option.when(cp.nonEmpty)( + Vector("-cp", cp.mkString(java.io.File.pathSeparator)) + ).getOrElse(Vector.empty) ++ + Vector(mainClass) ++ + mainArgs).filterNot(_.isBlank) + + if (cwd != null) os.makeDir.all(cwd) + + val process = os.proc(commandArgs).spawn( + cwd = cwd, + env = env, + stdin = stdin, + stdout = stdout, + stderr = stderr, + mergeErrIntoOut = mergeErrIntoOut, + propagateEnv = propagateEnv, + shutdownGracePeriod = shutdownGracePeriod, + destroyOnExit = destroyOnExit + ) + process + } + /** * Runs a JVM subprocess with the given configuration and returns a * [[os.CommandResult]] with it's aggregated output and error streams */ + @deprecated("Use callProcess", "Mill 0.12.8") def callSubprocess( mainClass: String, classPath: Agg[os.Path], @@ -52,6 +212,7 @@ object Jvm extends CoursierSupport { * Runs a JVM subprocess with the given configuration and returns a * [[os.CommandResult]] with it's aggregated output and error streams */ + @deprecated("Use callProcess", "Mill 0.12.8") def callSubprocess( mainClass: String, classPath: Agg[os.Path], @@ -79,6 +240,7 @@ object Jvm extends CoursierSupport { * Runs a JVM subprocess with the given configuration and returns a * [[os.CommandResult]] with it's aggregated output and error streams */ + @deprecated("Use callProcess", "Mill 0.12.8") def callSubprocess( mainClass: String, classPath: Agg[os.Path], @@ -132,6 +294,7 @@ object Jvm extends CoursierSupport { * This might help with long classpaths on OS'es (like Windows) * which only supports limited command-line length */ + @deprecated("Use spawnProcess or callProcess", "Mill 0.12.8") def runSubprocess( mainClass: String, classPath: Agg[os.Path], @@ -232,6 +395,7 @@ object Jvm extends CoursierSupport { * This might help with long classpaths on OS'es (like Windows) * which only supports limited command-line length */ + @deprecated("Use spawnProcess or callProcess", "Mill 0.12.8") def runSubprocessWithBackgroundOutputs( mainClass: String, classPath: Agg[os.Path], @@ -306,6 +470,7 @@ object Jvm extends CoursierSupport { * Runs a generic subprocess and waits for it to terminate. If process exited with non-zero code, exception * will be thrown. If you want to manually handle exit code, check [[runSubprocessWithResult]] */ + @deprecated("Use os.call", "Mill 0.12.8") def runSubprocess( commandArgs: Seq[String], envArgs: Map[String, String], @@ -320,6 +485,7 @@ object Jvm extends CoursierSupport { * * @return Result with exit code. */ + @deprecated("Use os.call", "Mill 0.12.8") def runSubprocessWithResult( commandArgs: Seq[String], envArgs: Map[String, String], @@ -362,6 +528,7 @@ object Jvm extends CoursierSupport { * that the subprocess's stdout and stderr streams go to the substituted * streams. */ + @deprecated("Use os.spawn", "Mill 0.12.8") def spawnSubprocess( commandArgs: Seq[String], envArgs: Map[String, String], @@ -384,6 +551,7 @@ object Jvm extends CoursierSupport { * respectively must be defined in the backgroundOutputs tuple. Non-background process should set * backgroundOutputs to [[None]]. */ + @deprecated("Use os.spawn", "Mill 0.12.8") def spawnSubprocessWithBackgroundOutputs( commandArgs: Seq[String], envArgs: Map[String, String], @@ -400,6 +568,7 @@ object Jvm extends CoursierSupport { ) } + @deprecated("Use withClassLoader", "Mill 0.12.8") def runLocal( mainClass: String, classPath: Agg[os.Path], @@ -430,6 +599,7 @@ object Jvm extends CoursierSupport { method } + @deprecated("Use withClassLoader", "Mill 0.12.8") def runClassloader[T](classPath: Agg[os.Path])(body: ClassLoader => T)(implicit ctx: mill.api.Ctx.Home ): T = { @@ -442,6 +612,7 @@ object Jvm extends CoursierSupport { ) } + @deprecated("Use createClassLoader", "Mill 0.12.8") def spawnClassloader( classPath: Iterable[os.Path], sharedPrefixes: Seq[String] = Nil, @@ -454,6 +625,55 @@ object Jvm extends CoursierSupport { )(new Ctx.Home { override def home = os.home }) } + /** + * Creates a `java.net.URLClassLoader` with specified parameters + * @param classPath URLs from which to load classes and resources + * @param parent parent class loader for delegation + * @param sharedLoader loader used for shared classes + * @param sharedPrefixes package prefix for classes that will be loaded by the `sharedLoader` + * @return new classloader + */ + def createClassLoader( + classPath: Iterable[os.Path], + parent: ClassLoader = null, + sharedLoader: ClassLoader = getClass.getClassLoader, + sharedPrefixes: Iterable[String] = Seq() + ): URLClassLoader = + new URLClassLoader( + classPath.iterator.map(_.toNIO.toUri.toURL).toArray, + refinePlatformParent(parent) + ) { + override def findClass(name: String): Class[?] = + if (sharedPrefixes.exists(name.startsWith)) sharedLoader.loadClass(name) + else super.findClass(name) + } + + /** + * @param classPath URLs from which to load classes and resources + * @param parent parent class loader for delegation + * @param sharedPrefixes package prefix for classes that will be loaded by the shared loader + * @param f function that will be called with newly created classloader + * @tparam T the return type of this function + * @return return value of the function `f` + */ + def withClassLoader[T]( + classPath: Iterable[os.Path], + parent: ClassLoader = null, + sharedPrefixes: Seq[String] = Seq.empty + )(f: ClassLoader => T): T = { + val oldClassloader = Thread.currentThread().getContextClassLoader + val newClassloader = + createClassLoader(classPath = classPath, parent = parent, sharedPrefixes = sharedPrefixes) + Thread.currentThread().setContextClassLoader(newClassloader) + try { + f(newClassloader) + } finally { + Thread.currentThread().setContextClassLoader(oldClassloader) + newClassloader.close() + } + } + + @deprecated("Use withClassLoader", "Mill 0.12.8") def inprocess[T]( classPath: Agg[os.Path], classLoaderOverrideSbtTesting: Boolean, @@ -642,4 +862,44 @@ object Jvm extends CoursierSupport { @deprecated("Use mill.api.JarManifest instead", "Mill after 0.11.0-M4") val JarManifest = mill.api.JarManifest + /** + * Return `ClassLoader.getPlatformClassLoader` for java 9 and above, if parent class loader is null, + * otherwise return same parent class loader. + * More details: https://docs.oracle.com/javase/9/migrate/toc.htm#JSMIG-GUID-A868D0B9-026F-4D46-B979-901834343F9E + * + * `ClassLoader.getPlatformClassLoader` call is implemented via runtime reflection, cause otherwise + * mill could be compiled only with jdk 9 or above. We don't want to introduce this restriction now. + */ + private def refinePlatformParent(parent: java.lang.ClassLoader): ClassLoader = { + if (parent != null) parent + else if (java9OrAbove) { + // Make sure when `parent == null`, we only delegate java.* classes + // to the parent getPlatformClassLoader. This is necessary because + // in Java 9+, somehow the getPlatformClassLoader ends up with all + // sorts of other non-java stuff on it's classpath, which is not what + // we want for an "isolated" classloader! + classOf[ClassLoader] + .getMethod("getPlatformClassLoader") + .invoke(null) + .asInstanceOf[ClassLoader] + } else { + // With Java 8 we want a clean classloader that still contains classes + // coming from com.sun.* etc. + // We get the application classloader parent which happens to be of + // type sun.misc.Launcher$ExtClassLoader + // We can't call the method directly since it would not compile on Java 9+ + // So we load it via reflection to allow compilation in Java 9+ but only + // on Java 8 + val launcherClass = getClass.getClassLoader().loadClass("sun.misc.Launcher") + val getLauncherMethod = launcherClass.getMethod("getLauncher") + val launcher = getLauncherMethod.invoke(null) + val getClassLoaderMethod = launcher.getClass().getMethod("getClassLoader") + val appClassLoader = getClassLoaderMethod.invoke(launcher).asInstanceOf[ClassLoader] + appClassLoader.getParent() + } + } + + private val java9OrAbove: Boolean = + !System.getProperty("java.specification.version").startsWith("1.") + } diff --git a/pythonlib/src/mill/pythonlib/PythonModule.scala b/pythonlib/src/mill/pythonlib/PythonModule.scala index b7a3dfe414c..d87d7419a29 100644 --- a/pythonlib/src/mill/pythonlib/PythonModule.scala +++ b/pythonlib/src/mill/pythonlib/PythonModule.scala @@ -5,6 +5,7 @@ import mill.api.Result import mill.util.Util import mill.util.Jvm import mill.api.Ctx +import mill.main.client.ServerFiles trait PythonModule extends PipModule with TaskModule { outer => @@ -174,12 +175,13 @@ trait PythonModule extends PipModule with TaskModule { outer => */ def runBackground(args: mill.define.Args) = Task.Command { val (procUuidPath, procLockfile, procUuid) = mill.scalalib.RunModule.backgroundSetup(Task.dest) + val pwd0 = os.Path(java.nio.file.Paths.get(".").toAbsolutePath) - Jvm.runSubprocess( + Jvm.spawnProcess( mainClass = "mill.scalalib.backgroundwrapper.MillBackgroundWrapper", classPath = mill.scalalib.ZincWorkerModule.backgroundWrapperClasspath().map(_.path).toSeq, jvmArgs = Nil, - envArgs = runnerEnvTask(), + env = runnerEnvTask(), mainArgs = Seq( procUuidPath.toString, procLockfile.toString, @@ -189,10 +191,13 @@ trait PythonModule extends PipModule with TaskModule { outer => pythonExe().path.toString, mainScript().path.toString ) ++ args.value, - workingDir = Task.workspace, - background = true, - useCpPassingJar = false, - runBackgroundLogToConsole = true, + cwd = Task.workspace, + stdin = "", + // Hack to forward the background subprocess output to the Mill server process + // stdout/stderr files, so the output will get properly slurped up by the Mill server + // and shown to any connected Mill client even if the current command has completed + stdout = os.PathAppendRedirect(pwd0 / ".." / ServerFiles.stdout), + stderr = os.PathAppendRedirect(pwd0 / ".." / ServerFiles.stderr), javaHome = mill.scalalib.ZincWorkerModule.javaHome().map(_.path) ) () @@ -261,11 +266,15 @@ object PythonModule { command: String = null, env: Map[String, String] = null, workingDir: os.Path = null - )(implicit ctx: Ctx): Unit = - Jvm.runSubprocess( - commandArgs = Seq(Option(command).getOrElse(command0)) ++ options ++ args.value, - envArgs = Option(env).getOrElse(env0), - workingDir = Option(workingDir).getOrElse(workingDir0) + )(implicit ctx: Ctx): Unit = { + os.call( + cmd = Seq(Option(command).getOrElse(command0)) ++ options ++ args.value, + env = Option(env).getOrElse(env0), + cwd = Option(workingDir).getOrElse(workingDir0), + stdin = os.Inherit, + stdout = os.Inherit, + check = true ) + } } } diff --git a/scalalib/src/mill/javalib/android/AndroidAppModule.scala b/scalalib/src/mill/javalib/android/AndroidAppModule.scala index c77f08699ca..af4b3c90156 100644 --- a/scalalib/src/mill/javalib/android/AndroidAppModule.scala +++ b/scalalib/src/mill/javalib/android/AndroidAppModule.scala @@ -545,7 +545,7 @@ trait AndroidAppModule extends JavaModule { val libManifests = androidUnpackArchives().flatMap(_.manifest) val mergedManifestPath = Task.dest / "AndroidManifest.xml" // TODO put it to the dedicated worker if cost of classloading is too high - Jvm.runSubprocess( + Jvm.callProcess( mainClass = "com.android.manifmerger.Merger", mainArgs = Seq( "--main", @@ -562,7 +562,9 @@ trait AndroidAppModule extends JavaModule { "--out", mergedManifestPath.toString() ) ++ libManifests.flatMap(m => Seq("--libs", m.path.toString())), - classPath = manifestMergerClasspath().map(_.path) + classPath = manifestMergerClasspath().map(_.path).toVector, + stdin = os.Inherit, + stdout = os.Inherit ) PathRef(mergedManifestPath) } diff --git a/scalalib/src/mill/javalib/checkstyle/CheckstyleModule.scala b/scalalib/src/mill/javalib/checkstyle/CheckstyleModule.scala index 493af8bf669..ce392ca680c 100644 --- a/scalalib/src/mill/javalib/checkstyle/CheckstyleModule.scala +++ b/scalalib/src/mill/javalib/checkstyle/CheckstyleModule.scala @@ -35,12 +35,13 @@ trait CheckstyleModule extends JavaModule { Task.log.info("running checkstyle ...") Task.log.debug(s"with $args") - val exitCode = Jvm.callSubprocess( + val exitCode = Jvm.callProcess( mainClass = "com.puppycrawl.tools.checkstyle.Main", - classPath = checkstyleClasspath().map(_.path), + classPath = checkstyleClasspath().map(_.path).toVector, mainArgs = args, - workingDir = millSourcePath, // allow passing relative paths for sources like src/a/b - streamOut = true, + cwd = millSourcePath, // allow passing relative paths for sources like src/a/b + stdin = os.Inherit, + stdout = os.Inherit, check = false ).exitCode diff --git a/scalalib/src/mill/javalib/palantirformat/PalantirFormatModule.scala b/scalalib/src/mill/javalib/palantirformat/PalantirFormatModule.scala index 06c606bb395..aa4597f08cd 100644 --- a/scalalib/src/mill/javalib/palantirformat/PalantirFormatModule.scala +++ b/scalalib/src/mill/javalib/palantirformat/PalantirFormatModule.scala @@ -136,12 +136,14 @@ object PalantirFormatModule extends ExternalModule with PalantirFormatBaseModule ctx.log.debug(s"running palantirformat with $mainArgs") - val exitCode = Jvm.callSubprocess( + val exitCode = Jvm.callProcess( mainClass = "com.palantir.javaformat.java.Main", - classPath = classPath.map(_.path), + classPath = classPath.map(_.path).toVector, jvmArgs = jvmArgs, mainArgs = mainArgs, - workingDir = ctx.dest, + cwd = ctx.dest, + stdin = os.Inherit, + stdout = os.Inherit, check = false ).exitCode diff --git a/scalalib/src/mill/javalib/revapi/RevapiModule.scala b/scalalib/src/mill/javalib/revapi/RevapiModule.scala index c74a7ac00b8..6f75deb8160 100644 --- a/scalalib/src/mill/javalib/revapi/RevapiModule.scala +++ b/scalalib/src/mill/javalib/revapi/RevapiModule.scala @@ -47,14 +47,15 @@ trait RevapiModule extends PublishModule { .result() Task.log.info("running revapi cli") - Jvm.runSubprocess( + Jvm.callProcess( mainClass = mainClass, - classPath = revapiClasspath().map(_.path), + classPath = revapiClasspath().map(_.path).toVector, jvmArgs = revapiJvmArgs(), mainArgs = mainArgs, - workingDir = workingDir + cwd = workingDir, + stdin = os.Inherit, + stdout = os.Inherit ) - PathRef(workingDir) } diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 39f70348241..aaff671bb89 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -1163,10 +1163,12 @@ trait JavaModule Task.log.info("options: " + cmdArgs) - Jvm.runSubprocess( - commandArgs = Seq(Jvm.jdkTool("javadoc")) ++ cmdArgs, - envArgs = Map(), - workingDir = Task.dest + os.call( + cmd = Seq(Jvm.jdkTool("javadoc")) ++ cmdArgs, + env = Map(), + cwd = Task.dest, + stdin = os.Inherit, + stdout = os.Inherit ) } diff --git a/scalalib/src/mill/scalalib/RunModule.scala b/scalalib/src/mill/scalalib/RunModule.scala index 5e5c62e9202..365175e5ca3 100644 --- a/scalalib/src/mill/scalalib/RunModule.scala +++ b/scalalib/src/mill/scalalib/RunModule.scala @@ -1,11 +1,13 @@ package mill.scalalib +import java.lang.reflect.Modifier import mainargs.arg import mill.api.JsonFormatters.pathReadWrite import mill.api.{Ctx, PathRef, Result} import mill.define.{Command, Task} import mill.util.Jvm import mill.{Agg, Args, T} +import mill.main.client.ServerFiles import os.{Path, ProcessOutput} import scala.util.control.NonFatal @@ -46,17 +48,15 @@ trait RunModule extends WithZincWorker { def allLocalMainClasses: T[Seq[String]] = Task { val classpath = localRunClasspath().map(_.path) if (zincWorker().javaHome().isDefined) { - Jvm - .callSubprocess( - mainClass = "mill.scalalib.worker.DiscoverMainClassesMain", - classPath = zincWorker().classpath().map(_.path), - mainArgs = Seq(classpath.mkString(",")), - javaHome = zincWorker().javaHome().map(_.path), - streamOut = false - ) - .out - .lines() - + Jvm.callProcess( + mainClass = "mill.scalalib.worker.DiscoverMainClassesMain", + classPath = zincWorker().classpath().map(_.path).toVector, + mainArgs = Seq(classpath.mkString(",")), + javaHome = zincWorker().javaHome().map(_.path), + stdin = os.Inherit, + stdout = os.Pipe, + cwd = Task.dest + ).out.lines() } else { zincWorker().worker().discoverMainClasses(classpath) } @@ -157,11 +157,11 @@ trait RunModule extends WithZincWorker { def runLocalTask(mainClass: Task[String], args: Task[Args] = Task.Anon(Args())): Task[Unit] = Task.Anon { - Jvm.runLocal( - mainClass(), - runClasspath().map(_.path), - args().value - ) + Jvm.withClassLoader( + classPath = runClasspath().map(_.path).toVector + ) { classloader => + RunModule.getMainMethod(mainClass(), classloader).invoke(null, args().value.toArray) + } } def runBackgroundTask(mainClass: Task[String], args: Task[Args] = Task.Anon(Args())): Task[Unit] = @@ -249,7 +249,8 @@ trait RunModule extends WithZincWorker { * code, without the Mill process. Useful for deployment & other places where * you do not want a build tool running */ - def launcher = Task { launcher0() } + def launcher: T[PathRef] = Task { launcher0() } + } object RunModule { @@ -260,6 +261,21 @@ object RunModule { val procLockfile = dest / ".mill-background-process-lock" (procUuidPath, procLockfile, procUuid) } + + private[mill] def getMainMethod(mainClassName: String, cl: ClassLoader) = { + val mainClass = cl.loadClass(mainClassName) + val method = mainClass.getMethod("main", classOf[Array[String]]) + // jvm allows the actual main class to be non-public and to run a method in the non-public class, + // we need to make it accessible + method.setAccessible(true) + val modifiers = method.getModifiers + if (!Modifier.isPublic(modifiers)) + throw new NoSuchMethodException(mainClassName + ".main is not public") + if (!Modifier.isStatic(modifiers)) + throw new NoSuchMethodException(mainClassName + ".main is not static") + method + } + trait Runner { def run( args: os.Shellable, @@ -293,21 +309,59 @@ object RunModule { background: Boolean = false, runBackgroundLogToConsole: Boolean = false )(implicit ctx: Ctx): Unit = { - Jvm.runSubprocess( - Option(mainClass).getOrElse(mainClass0.fold(sys.error, identity)), - runClasspath ++ extraRunClasspath, - Option(forkArgs).getOrElse(forkArgs0), - Option(forkEnv).getOrElse(forkEnv0), - args.value, - Option(workingDir).getOrElse(ctx.dest), - background = background, - Option(useCpPassingJar) match { - case Some(b) => b - case None => useCpPassingJar0 - }, - runBackgroundLogToConsole = runBackgroundLogToConsole, - javaHome = javaHome - ) + val dest = ctx.dest + val cwd = Option(workingDir).getOrElse(dest) + val mainClass1 = Option(mainClass).getOrElse(mainClass0.fold(sys.error, identity)) + val mainArgs = args.value + val classPath = runClasspath ++ extraRunClasspath + val jvmArgs = Option(forkArgs).getOrElse(forkArgs0) + Option(useCpPassingJar) match { + case Some(b) => b: Boolean + case None => useCpPassingJar0 + } + val env = Option(forkEnv).getOrElse(forkEnv0) + if (background) { + val (stdout, stderr) = if (runBackgroundLogToConsole) { + // Hack to forward the background subprocess output to the Mill server process + // stdout/stderr files, so the output will get properly slurped up by the Mill server + // and shown to any connected Mill client even if the current command has completed + val pwd0 = os.Path(java.nio.file.Paths.get(".").toAbsolutePath) + ( + os.PathAppendRedirect(pwd0 / ".." / ServerFiles.stdout), + os.PathAppendRedirect(pwd0 / ".." / ServerFiles.stderr) + ) + } else { + (dest / "stdout.log": os.ProcessOutput, dest / "stderr.log": os.ProcessOutput) + } + Jvm.spawnProcess( + mainClass = mainClass1, + classPath = classPath, + jvmArgs = jvmArgs, + env = env, + mainArgs = mainArgs, + cwd = cwd, + stdin = "", + stdout = stdout, + stderr = stderr, + cpPassingJarPath = Some(os.temp(prefix = "run-", suffix = ".jar", deleteOnExit = false)), + javaHome = javaHome, + destroyOnExit = false + ) + } else { + Jvm.callProcess( + mainClass = mainClass1, + classPath = classPath, + jvmArgs = jvmArgs, + env = env, + mainArgs = mainArgs, + cwd = cwd, + stdin = os.Inherit, + stdout = os.Inherit, + stderr = os.Inherit, + cpPassingJarPath = Some(os.temp(prefix = "run-", suffix = ".jar", deleteOnExit = false)), + javaHome = javaHome + ) + } } } } diff --git a/scalalib/src/mill/scalalib/ScalaModule.scala b/scalalib/src/mill/scalalib/ScalaModule.scala index 739cc19bb85..e0afd126954 100644 --- a/scalalib/src/mill/scalalib/ScalaModule.scala +++ b/scalalib/src/mill/scalalib/ScalaModule.scala @@ -434,7 +434,7 @@ trait ScalaModule extends JavaModule with TestModule.ScalaModuleBase { outer => } else { val useJavaCp = "-usejavacp" - Jvm.runSubprocess( + Jvm.callProcess( mainClass = if (ZincWorkerUtil.isDottyOrScala3(scalaVersion())) "dotty.tools.repl.Main" @@ -444,9 +444,11 @@ trait ScalaModule extends JavaModule with TestModule.ScalaModuleBase { outer => _.path ), jvmArgs = forkArgs(), - envArgs = forkEnv(), + env = forkEnv(), mainArgs = Seq(useJavaCp) ++ consoleScalacOptions().filterNot(Set(useJavaCp)), - workingDir = forkWorkingDir() + cwd = forkWorkingDir(), + stdin = os.Inherit, + stdout = os.Inherit ) Result.Success(()) } @@ -509,13 +511,15 @@ trait ScalaModule extends JavaModule with TestModule.ScalaModuleBase { outer => } else { val mainClass = ammoniteMainClass() Task.log.debug(s"Using ammonite main class: ${mainClass}") - Jvm.runSubprocess( + Jvm.callProcess( mainClass = mainClass, - classPath = ammoniteReplClasspath().map(_.path), + classPath = ammoniteReplClasspath().map(_.path).toVector, jvmArgs = forkArgs(), - envArgs = forkEnv(), + env = forkEnv(), mainArgs = replOptions, - workingDir = forkWorkingDir() + cwd = forkWorkingDir(), + stdin = os.Inherit, + stdout = os.Inherit ) Result.Success(()) } diff --git a/scalalib/src/mill/scalalib/TestModule.scala b/scalalib/src/mill/scalalib/TestModule.scala index ebdf94a5454..476cbe165bd 100644 --- a/scalalib/src/mill/scalalib/TestModule.scala +++ b/scalalib/src/mill/scalalib/TestModule.scala @@ -42,15 +42,17 @@ trait TestModule def discoveredTestClasses: T[Seq[String]] = Task { val classes = if (zincWorker().javaHome().isDefined) { - Jvm.callSubprocess( + Jvm.callProcess( mainClass = "mill.testrunner.DiscoverTestsMain", - classPath = zincWorker().scalalibClasspath().map(_.path), + classPath = zincWorker().scalalibClasspath().map(_.path).toVector, mainArgs = runClasspath().flatMap(p => Seq("--runCp", p.path.toString())) ++ testClasspath().flatMap(p => Seq("--testCp", p.path.toString())) ++ Seq("--framework", testFramework()), javaHome = zincWorker().javaHome().map(_.path), - streamOut = false + stdin = os.Inherit, + stdout = os.Pipe, + cwd = Task.dest ).out.lines() } else { mill.testrunner.DiscoverTestsMain.main0( @@ -148,7 +150,7 @@ trait TestModule val testArgs = TestArgs( framework = testFramework(), - classpath = runClasspath().map(_.path), + classpath = runClasspath().map(_.path).toVector, arguments = args(), sysProps = Map.empty, outputPath = outputPath, @@ -282,45 +284,43 @@ object TestModule { * override this method. */ override def discoveredTestClasses: T[Seq[String]] = Task { - Jvm.inprocess( - runClasspath().map(_.path), - classLoaderOverrideSbtTesting = true, - isolated = true, - closeContextClassLoaderWhenDone = true, - cl => { - val builderClass: Class[_] = - cl.loadClass("com.github.sbt.junit.jupiter.api.JupiterTestCollector$Builder") - val builder = builderClass.getConstructor().newInstance() - - builderClass.getMethod("withClassDirectory", classOf[java.io.File]).invoke( - builder, - compile().classes.path.wrapped.toFile - ) - builderClass.getMethod("withRuntimeClassPath", classOf[Array[java.net.URL]]).invoke( - builder, - testClasspath().map(_.path.wrapped.toUri().toURL()).toArray - ) - builderClass.getMethod("withClassLoader", classOf[ClassLoader]).invoke(builder, cl) - - val testCollector = builderClass.getMethod("build").invoke(builder) - val testCollectorClass = - cl.loadClass("com.github.sbt.junit.jupiter.api.JupiterTestCollector") - - val result = testCollectorClass.getMethod("collectTests").invoke(testCollector) - val resultClass = - cl.loadClass("com.github.sbt.junit.jupiter.api.JupiterTestCollector$Result") - - val items = resultClass.getMethod( - "getDiscoveredTests" - ).invoke(result).asInstanceOf[java.util.List[_]] - val itemClass = cl.loadClass("com.github.sbt.junit.jupiter.api.JupiterTestCollector$Item") - - import scala.jdk.CollectionConverters._ - items.asScala.map { item => - itemClass.getMethod("getFullyQualifiedClassName").invoke(item).asInstanceOf[String] - }.toSeq - } - ) + Jvm.withClassLoader( + classPath = runClasspath().map(_.path).toVector, + sharedPrefixes = Seq("sbt.testing.") + ) { classLoader => + val builderClass: Class[_] = + classLoader.loadClass("com.github.sbt.junit.jupiter.api.JupiterTestCollector$Builder") + val builder = builderClass.getConstructor().newInstance() + + builderClass.getMethod("withClassDirectory", classOf[java.io.File]).invoke( + builder, + compile().classes.path.wrapped.toFile + ) + builderClass.getMethod("withRuntimeClassPath", classOf[Array[java.net.URL]]).invoke( + builder, + testClasspath().map(_.path.wrapped.toUri().toURL()).toArray + ) + builderClass.getMethod("withClassLoader", classOf[ClassLoader]).invoke(builder, classLoader) + + val testCollector = builderClass.getMethod("build").invoke(builder) + val testCollectorClass = + classLoader.loadClass("com.github.sbt.junit.jupiter.api.JupiterTestCollector") + + val result = testCollectorClass.getMethod("collectTests").invoke(testCollector) + val resultClass = + classLoader.loadClass("com.github.sbt.junit.jupiter.api.JupiterTestCollector$Result") + + val items = resultClass.getMethod( + "getDiscoveredTests" + ).invoke(result).asInstanceOf[java.util.List[_]] + val itemClass = + classLoader.loadClass("com.github.sbt.junit.jupiter.api.JupiterTestCollector$Item") + + import scala.jdk.CollectionConverters._ + items.asScala.map { item => + itemClass.getMethod("getFullyQualifiedClassName").invoke(item).asInstanceOf[String] + }.toSeq + } } } diff --git a/scalalib/src/mill/scalalib/TestModuleUtil.scala b/scalalib/src/mill/scalalib/TestModuleUtil.scala index 574e0036269..6af0afbdecc 100644 --- a/scalalib/src/mill/scalalib/TestModuleUtil.scala +++ b/scalalib/src/mill/scalalib/TestModuleUtil.scala @@ -63,18 +63,17 @@ private[scalalib] object TestModuleUtil { os.makeDir.all(sandbox) - Jvm.runSubprocess( + Jvm.callProcess( mainClass = "mill.testrunner.entrypoint.TestRunnerMain", - classPath = - (runClasspath ++ testrunnerEntrypointClasspath).map( - _.path - ), + classPath = (runClasspath ++ testrunnerEntrypointClasspath).map(_.path), jvmArgs = jvmArgs, - envArgs = forkEnv ++ resourceEnv, + env = forkEnv ++ resourceEnv, mainArgs = Seq(testRunnerClasspathArg, argsFile.toString), - workingDir = if (testSandboxWorkingDir) sandbox else forkWorkingDir, - useCpPassingJar = useArgsFile, - javaHome = javaHome + cwd = if (testSandboxWorkingDir) sandbox else forkWorkingDir, + cpPassingJarPath = Some(os.temp(prefix = "run-", suffix = ".jar", deleteOnExit = false)), + javaHome = javaHome, + stdin = os.Inherit, + stdout = os.Inherit ) if (!os.exists(outputPath)) Left(s"Test reporting Failed: ${outputPath} does not exist") @@ -102,9 +101,9 @@ private[scalalib] object TestModuleUtil { // tests to run and shut down val discoveredTests = if (javaHome.isDefined) { - Jvm.callSubprocess( + Jvm.callProcess( mainClass = "mill.testrunner.GetTestTasksMain", - classPath = scalalibClasspath.map(_.path), + classPath = scalalibClasspath.map(_.path).toVector, mainArgs = (runClasspath ++ testrunnerEntrypointClasspath).flatMap(p => Seq("--runCp", p.path.toString) @@ -114,7 +113,9 @@ private[scalalib] object TestModuleUtil { selectors.flatMap(s => Seq("--selectors", s)) ++ args.flatMap(s => Seq("--args", s)), javaHome = javaHome, - streamOut = false + stdin = os.Inherit, + stdout = os.Pipe, + cwd = Task.dest ).out.lines().toSet } else { mill.testrunner.GetTestTasksMain.main0( diff --git a/scalalib/src/mill/scalalib/giter8/Giter8Module.scala b/scalalib/src/mill/scalalib/giter8/Giter8Module.scala index e3b9e679044..a479dd611a0 100644 --- a/scalalib/src/mill/scalalib/giter8/Giter8Module.scala +++ b/scalalib/src/mill/scalalib/giter8/Giter8Module.scala @@ -33,11 +33,13 @@ trait Giter8Module extends CoursierModule { throw e } - Jvm.runSubprocess( - "giter8.Giter8", - giter8Dependencies.map(_.path), + Jvm.callProcess( + mainClass = "giter8.Giter8", + classPath = giter8Dependencies.map(_.path).toVector, mainArgs = args, - workingDir = Task.workspace + cwd = Task.workspace, + stdin = os.Inherit, + stdout = os.Inherit ) } } diff --git a/scalalib/src/mill/scalalib/publish/SonatypeHelpers.scala b/scalalib/src/mill/scalalib/publish/SonatypeHelpers.scala index a5534b4340d..c5b292220d0 100644 --- a/scalalib/src/mill/scalalib/publish/SonatypeHelpers.scala +++ b/scalalib/src/mill/scalalib/publish/SonatypeHelpers.scala @@ -1,7 +1,5 @@ package mill.scalalib.publish -import mill.util.Jvm - import java.math.BigInteger import java.security.MessageDigest @@ -54,7 +52,14 @@ object SonatypeHelpers { val fileName = file.toString val command = "gpg" +: args :+ fileName - Jvm.runSubprocess(command, env, workspace) + os.call( + command, + env, + workspace, + stdin = os.Inherit, + stdout = os.Inherit, + stderr = os.Inherit + ) os.Path(fileName + ".asc") } diff --git a/scalalib/test/src/mill/scalalib/AssemblyTestUtils.scala b/scalalib/test/src/mill/scalalib/AssemblyTestUtils.scala index c4e635a3037..d840e0ee79e 100644 --- a/scalalib/test/src/mill/scalalib/AssemblyTestUtils.scala +++ b/scalalib/test/src/mill/scalalib/AssemblyTestUtils.scala @@ -50,16 +50,21 @@ trait AssemblyTestUtils { val sources = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "assembly" def runAssembly(file: os.Path, wd: os.Path, checkExe: Boolean = false): Unit = { println(s"File size: ${os.stat(file).size}") - Jvm.runSubprocess( - commandArgs = Seq(Jvm.javaExe, "-jar", file.toString(), "--text", "tutu"), - envArgs = Map.empty[String, String], - workingDir = wd + os.call( + cmd = Seq(Jvm.javaExe, "-jar", file.toString(), "--text", "tutu"), + env = Map.empty[String, String], + cwd = wd, + stdin = os.Inherit, + stdout = os.Inherit ) + if (checkExe) { - Jvm.runSubprocess( - commandArgs = Seq(file.toString(), "--text", "tutu"), - envArgs = Map.empty[String, String], - workingDir = wd + os.call( + cmd = Seq(file.toString(), "--text", "tutu"), + env = Map.empty[String, String], + cwd = wd, + stdin = os.Inherit, + stdout = os.Inherit ) } } diff --git a/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala b/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala index 1d3b5538e4c..ed4ff85a2ab 100644 --- a/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala +++ b/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala @@ -5,7 +5,6 @@ import mainargs.Flag import mill.api.Loose.Agg import mill.api.{Result, internal} import mill.define.{Command, Task} -import mill.util.Jvm import mill.util.Util.millProjectModule import mill.scalalib.api.ZincWorkerUtil import mill.scalalib.bsp.{ScalaBuildTarget, ScalaPlatform} @@ -282,10 +281,13 @@ trait ScalaNativeModule extends ScalaModule { outer => // Runs the native binary override def run(args: Task[Args] = Task.Anon(Args())) = Task.Command { - Jvm.runSubprocess( - commandArgs = Vector(nativeLink().toString) ++ args().value, - envArgs = forkEnv(), - workingDir = forkWorkingDir() + os.call( + cmd = Vector(nativeLink().toString) ++ args().value, + env = forkEnv(), + cwd = forkWorkingDir(), + stdin = os.Inherit, + stdout = os.Inherit, + stderr = os.Inherit ) } diff --git a/testrunner/src/mill/testrunner/DiscoverTestsMain.scala b/testrunner/src/mill/testrunner/DiscoverTestsMain.scala index f9bdb77c794..bf0287a8ffd 100644 --- a/testrunner/src/mill/testrunner/DiscoverTestsMain.scala +++ b/testrunner/src/mill/testrunner/DiscoverTestsMain.scala @@ -1,6 +1,6 @@ package mill.testrunner -import mill.api.{Ctx, internal} +import mill.api.internal import os.Path @internal object DiscoverTestsMain { @@ -12,24 +12,19 @@ import os.Path main0(runCp, testCp, framework).foreach(println) } def main0(runCp: Seq[os.Path], testCp: Seq[os.Path], framework: String): Seq[String] = { - mill.util.Jvm.inprocess( - runCp, - classLoaderOverrideSbtTesting = true, - isolated = true, - closeContextClassLoaderWhenDone = false, - body = classLoader => { - TestRunnerUtils - .discoverTests(classLoader, Framework.framework(framework)(classLoader), testCp) - .toSeq - .map(_._1.getName()) - .map { - case s if s.endsWith("$") => s.dropRight(1) - case s => s - } - } - )(new Ctx.Home { - def home: Path = os.home - }) + mill.util.Jvm.withClassLoader( + classPath = runCp, + sharedPrefixes = Seq("sbt.testing.") + ) { classLoader => + TestRunnerUtils + .discoverTests(classLoader, Framework.framework(framework)(classLoader), testCp) + .toSeq + .map(_._1.getName()) + .map { + case s if s.endsWith("$") => s.dropRight(1) + case s => s + } + } } def main(args: Array[String]): Unit = mainargs.ParserForMethods(this).runOrExit(args) diff --git a/testrunner/src/mill/testrunner/GetTestTasksMain.scala b/testrunner/src/mill/testrunner/GetTestTasksMain.scala index a3f63575d77..a3f6e5a7ead 100644 --- a/testrunner/src/mill/testrunner/GetTestTasksMain.scala +++ b/testrunner/src/mill/testrunner/GetTestTasksMain.scala @@ -1,7 +1,7 @@ package mill.testrunner import mill.api.Loose.Agg -import mill.api.{Ctx, internal} +import mill.api.internal import os.Path @internal object GetTestTasksMain { @@ -26,23 +26,19 @@ import os.Path args: Seq[String] ): Seq[String] = { val globFilter = TestRunnerUtils.globFilter(selectors) - mill.util.Jvm.inprocess( - runCp, - classLoaderOverrideSbtTesting = true, - isolated = true, - closeContextClassLoaderWhenDone = false, - classLoader => - TestRunnerUtils - .getTestTasks0( - Framework.framework(framework), - Agg.from(testCp), - args, - cls => globFilter(cls.getName), - classLoader - ) - )(new Ctx.Home { - def home: Path = os.home - }) + mill.util.Jvm.withClassLoader( + classPath = runCp, + sharedPrefixes = Seq("sbt.testing.") + ) { classLoader => + TestRunnerUtils + .getTestTasks0( + Framework.framework(framework), + Agg.from(testCp), + args, + cls => globFilter(cls.getName), + classLoader + ) + } } def main(args: Array[String]): Unit = mainargs.ParserForMethods(this).runOrExit(args) diff --git a/testrunner/src/mill/testrunner/TestRunner.scala b/testrunner/src/mill/testrunner/TestRunner.scala index 8ce8efc7430..ff2991926b3 100644 --- a/testrunner/src/mill/testrunner/TestRunner.scala +++ b/testrunner/src/mill/testrunner/TestRunner.scala @@ -14,20 +14,18 @@ import mill.util.Jvm testReporter: TestReporter, classFilter: Class[_] => Boolean = _ => true )(implicit ctx: Ctx.Log with Ctx.Home): (String, Seq[mill.testrunner.TestResult]) = { - // Leave the context class loader set and open so that shutdown hooks can access it - Jvm.inprocess( - entireClasspath, - classLoaderOverrideSbtTesting = true, - isolated = true, - closeContextClassLoaderWhenDone = false, + Jvm.withClassLoader( + classPath = entireClasspath.toVector, + sharedPrefixes = Seq("sbt.testing.") + ) { classLoader => TestRunnerUtils.runTestFramework0( frameworkInstances, testClassfilePath, args, classFilter, - _, + classLoader, testReporter ) - ) + } } } From 2c96aa8d967475992a4598e214e4240f56263dbd Mon Sep 17 00:00:00 2001 From: Sakib Hadziavdic Date: Fri, 7 Feb 2025 14:31:32 +0100 Subject: [PATCH 2/4] Try fix compile --- scalalib/src/mill/scalalib/RunModule.scala | 1 + scalalib/src/mill/scalalib/giter8/Giter8Module.scala | 1 + 2 files changed, 2 insertions(+) diff --git a/scalalib/src/mill/scalalib/RunModule.scala b/scalalib/src/mill/scalalib/RunModule.scala index 365175e5ca3..981637d1d36 100644 --- a/scalalib/src/mill/scalalib/RunModule.scala +++ b/scalalib/src/mill/scalalib/RunModule.scala @@ -162,6 +162,7 @@ trait RunModule extends WithZincWorker { ) { classloader => RunModule.getMainMethod(mainClass(), classloader).invoke(null, args().value.toArray) } + () } def runBackgroundTask(mainClass: Task[String], args: Task[Args] = Task.Anon(Args())): Task[Unit] = diff --git a/scalalib/src/mill/scalalib/giter8/Giter8Module.scala b/scalalib/src/mill/scalalib/giter8/Giter8Module.scala index a479dd611a0..91c57682da5 100644 --- a/scalalib/src/mill/scalalib/giter8/Giter8Module.scala +++ b/scalalib/src/mill/scalalib/giter8/Giter8Module.scala @@ -41,5 +41,6 @@ trait Giter8Module extends CoursierModule { stdin = os.Inherit, stdout = os.Inherit ) + () } } From 4338aae067bcf7f5409f5249eeb0a95a6816b8ad Mon Sep 17 00:00:00 2001 From: Sakib Hadziavdic Date: Fri, 7 Feb 2025 14:41:00 +0100 Subject: [PATCH 3/4] Try fix compile --- scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala b/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala index ed4ff85a2ab..5d9e29e2b19 100644 --- a/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala +++ b/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala @@ -289,6 +289,7 @@ trait ScalaNativeModule extends ScalaModule { outer => stdout = os.Inherit, stderr = os.Inherit ) + () } @internal From f2f019e113d5fc22ae080a425e61be101db60de2 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Fri, 7 Feb 2025 13:49:07 +0000 Subject: [PATCH 4/4] [autofix.ci] apply automated fixes --- testrunner/src/mill/testrunner/DiscoverTestsMain.scala | 1 - testrunner/src/mill/testrunner/GetTestTasksMain.scala | 1 - 2 files changed, 2 deletions(-) diff --git a/testrunner/src/mill/testrunner/DiscoverTestsMain.scala b/testrunner/src/mill/testrunner/DiscoverTestsMain.scala index bf0287a8ffd..db6a026031a 100644 --- a/testrunner/src/mill/testrunner/DiscoverTestsMain.scala +++ b/testrunner/src/mill/testrunner/DiscoverTestsMain.scala @@ -1,7 +1,6 @@ package mill.testrunner import mill.api.internal -import os.Path @internal object DiscoverTestsMain { private implicit def PathTokensReader2: mainargs.TokensReader.Simple[os.Path] = diff --git a/testrunner/src/mill/testrunner/GetTestTasksMain.scala b/testrunner/src/mill/testrunner/GetTestTasksMain.scala index a3f6e5a7ead..2d8f4692171 100644 --- a/testrunner/src/mill/testrunner/GetTestTasksMain.scala +++ b/testrunner/src/mill/testrunner/GetTestTasksMain.scala @@ -2,7 +2,6 @@ package mill.testrunner import mill.api.Loose.Agg import mill.api.internal -import os.Path @internal object GetTestTasksMain { private implicit def PathTokensReader2: mainargs.TokensReader.Simple[os.Path] =