Handle --jvm flag in process with JNI instantiated JVM (#12843)

Fixes #12842 by designing a _JNI OS environment_, providing its implementations for supported platforms and using it to handle `--jvm` flag. All the _Standard Library Tests (jvm)_ use this new launch-in-the-same-process technique.

Author: JaroslavTulach

build.sbt

@@ -363,6 +363,7 @@ lazy val enso = (project in file("."))
     `persistance`,
     `persistance-dsl`,
     pkg,
+    `poi-wrapper`,
     `polyglot-api`,
     `polyglot-api-macros`,
     `process-utils`,
@@ -618,7 +619,7 @@ val bouncyCastle = Seq(
 val jlineVersion = "3.26.3"
 val jline = Seq(
   "org.jline" % "jline-terminal"     % jlineVersion,
-  "org.jline" % "jline-terminal-jna" % jlineVersion,
+  "org.jline" % "jline-terminal-jni" % jlineVersion, // The terminal provider jna has been deprecated, check your configuration.
   "org.jline" % "jline-reader"       % jlineVersion,
   "org.jline" % "jline-native"       % jlineVersion
 )
@@ -1384,6 +1385,20 @@ lazy val `jna-wrapper` = project
     }
   )
 
+lazy val `poi-wrapper` = project
+  .in(file("lib/java/poi-wrapper"))
+  .settings(
+    frgaalJavaCompilerSetting,
+    version := "0.1",
+    autoScalaLibrary := false,
+    libraryDependencies ++= Seq(
+      "org.apache.poi" % "poi-ooxml" % poiOoxmlVersion
+    ),
+    assemblyMergeStrategy := { case _ =>
+      MergeStrategy.preferProject
+    }
+  )
+
 lazy val `runtime-utils` = project
   .in(file("lib/java/runtime-utils"))
   .enablePlugins(JPMSPlugin)
@@ -1435,6 +1450,7 @@ lazy val `directory-watcher-wrapper` = project
       )
     }
   )
+  .dependsOn(`jna-wrapper` % "provided")
 
 lazy val `fansi-wrapper` = project
   .in(file("lib/java/fansi-wrapper"))
@@ -1495,14 +1511,16 @@ lazy val `akka-wrapper` = project
       "com.google.protobuf"       % "protobuf-java"            % googleProtobufVersion,
       "io.github.java-diff-utils" % "java-diff-utils"          % javaDiffVersion,
       "org.reactivestreams"       % "reactive-streams"         % reactiveStreamsVersion,
-      "net.java.dev.jna"          % "jna"                      % jnaVersion,
       "io.spray"                 %% "spray-json"               % sprayJsonVersion
     ),
     javaModuleName := "org.enso.akka.wrapper",
     Compile / moduleDependencies ++= slf4jApi ++ Seq(
       "com.google.protobuf" % "protobuf-java"    % googleProtobufVersion,
       "org.reactivestreams" % "reactive-streams" % reactiveStreamsVersion
     ),
+    Compile / internalModuleDependencies := Seq(
+      (`jna-wrapper` / Compile / exportedModule).value
+    ),
     assembly / assemblyExcludedJars := {
       val excludedJars = JPMSUtils.filterModulesFromUpdate(
         update.value,
@@ -1511,8 +1529,7 @@ lazy val `akka-wrapper` = project
           "com.typesafe"              % "config"             % typesafeConfigVersion,
           "io.github.java-diff-utils" % "java-diff-utils"    % javaDiffVersion,
           "com.google.protobuf"       % "protobuf-java"      % googleProtobufVersion,
-          "org.reactivestreams"       % "reactive-streams"   % reactiveStreamsVersion,
-          "net.java.dev.jna"          % "jna"                % jnaVersion
+          "org.reactivestreams"       % "reactive-streams"   % reactiveStreamsVersion
         ),
         streams.value.log,
         moduleName.value,
@@ -1549,6 +1566,7 @@ lazy val `akka-wrapper` = project
       )
     }
   )
+  .dependsOn(`jna-wrapper` % "provided")
 
 lazy val `zio-wrapper` = project
   .in(file("lib/java/zio-wrapper"))
@@ -3902,7 +3920,7 @@ lazy val `engine-runner` = project
       val NI_MODULES =
         "org.graalvm.nativeimage,org.graalvm.nativeimage.builder,org.graalvm.nativeimage.base,org.graalvm.nativeimage.driver,org.graalvm.nativeimage.librarysupport,org.graalvm.nativeimage.objectfile,org.graalvm.nativeimage.pointsto,com.oracle.graal.graal_enterprise,com.oracle.svm.svm_enterprise"
       val JDK_MODULES =
-        "java.desktop,java.naming,java.net.http,jdk.charsets,jdk.crypto.ec,jdk.localedata,jdk.httpserver,java.rmi"
+        "java.naming,java.net.http,jdk.charsets,jdk.crypto.ec,jdk.localedata,jdk.httpserver,java.rmi"
       val DEBUG_MODULES  = "jdk.jdwp.agent"
       val PYTHON_MODULES = "jdk.security.auth,java.naming"
 
@@ -3982,6 +4000,7 @@ lazy val `engine-runner` = project
               "-H:+AddAllCharsets",
               "-H:+IncludeAllLocales",
               "-H:+RunReachabilityHandlersConcurrently",
+              "-R:-InstallSegfaultHandler",
               // Workaround a problem with build-/runtime-initialization conflict
               // by disabling this service provider
               "-H:ServiceLoaderFeatureExcludeServiceProviders=net.snowflake.client.core.FileTypeDetector",
@@ -4285,12 +4304,13 @@ lazy val `os-environment` =
         val targetDir = (Test / target).value
         NativeImage.buildNativeImage(
           "test-os-env",
-          staticOnLinux = true,
+          staticOnLinux = false,
           targetDir     = targetDir,
           mainClass     = Some("org.enso.os.environment.TestRunner"),
           additionalOptions = Seq(
             "-ea",
-            "--features=org.enso.os.environment.TestCollectorFeature"
+            "--features=org.enso.os.environment.TestCollectorFeature",
+            "-R:-InstallSegfaultHandler"
           )
         )
       }.value,
@@ -4301,7 +4321,7 @@ lazy val `os-environment` =
           val exeFile =
             (Test / target).value / ("test-os-env" + exeSuffix)
           val binPath = exeFile.getAbsolutePath
-          val res     = binPath ! logger
+          val res     = Process(Seq(binPath), None, "JAVA_OPTS" -> "") ! logger
           if (res != 0) {
             logger.error("Some test in os-environment failed")
             throw new TestsFailedException()
@@ -5027,6 +5047,11 @@ lazy val `std-table` = project
       "org.mockito"              % "mockito-core"            % mockitoJavaVersion        % Test,
       "org.mockito"              % "mockito-junit-jupiter"   % mockitoJavaVersion        % Test
     ),
+    Compile / unmanagedJars := {
+      Seq(
+        Attributed.blank((`poi-wrapper` / assembly).value)
+      )
+    },
     Compile / packageBin := {
       val result            = (Compile / packageBin).value
       val cacheStoreFactory = streams.value.cacheStoreFactory
@@ -5037,13 +5062,21 @@ lazy val `std-table` = project
           ignoreScalaLibrary = true,
           libraryUpdates     = (Compile / update).value,
           unmanagedClasspath = (Compile / unmanagedJars).value,
-          logger             = streams.value.log,
-          cacheStoreFactory  = cacheStoreFactory,
-          previousRun        = None
+          ignoreDependencies = Some(
+            Seq(
+              "org.apache.poi" % "poi"            % poiOoxmlVersion,
+              "org.apache.poi" % "poi-ooxml"      % poiOoxmlVersion,
+              "org.apache.poi" % "poi-ooxml-lite" % poiOoxmlVersion
+            )
+          ),
+          logger            = streams.value.log,
+          cacheStoreFactory = cacheStoreFactory,
+          previousRun       = None
         )
       result
     }
   )
+  .dependsOn(`poi-wrapper`)
   .dependsOn(`std-base` % "provided")
 
 lazy val extractNativeLibs = taskKey[AnalysisOfExtractedNativeLibs](
@@ -5082,7 +5115,8 @@ lazy val `std-image` = project
           `image-polyglot-root`,
           Seq("std-image.jar", "opencv.jar"),
           ignoreScalaLibrary = true,
-          ignoreDependency   = Some("org.openpnp" % "opencv" % opencvVersion),
+          ignoreDependencies =
+            Some(Seq("org.openpnp" % "opencv" % opencvVersion)),
           libraryUpdates     = (Compile / update).value,
           logger             = logger,
           cacheStoreFactory  = cacheStoreFactory,
@@ -5515,7 +5549,6 @@ lazy val `std-tableau` = project
           unmanagedClasspath = unmanagedClasspath,
           previousRun        = prev
         )
-
       StdBits
         .extractNativeLibsFromTableau(
           `std-tableau-polyglot-root`,

distribution/engine/THIRD-PARTY/NOTICE

@@ -361,11 +361,6 @@ The license file can be found at `licenses/APACHE2.0`.
 Copyright notices related to this dependency can be found in the directory `jakarta.inject.jakarta.inject-api-2.0.1`.
 
 
-'jna', licensed under the Apache-2.0, is distributed with the engine.
-The license file can be found at `licenses/APACHE2.0`.
-Copyright notices related to this dependency can be found in the directory `net.java.dev.jna.jna-5.14.0`.
-
-
 'commons-compress', licensed under the Apache-2.0, is distributed with the engine.
 The license file can be found at `licenses/APACHE2.0`.
 Copyright notices related to this dependency can be found in the directory `org.apache.commons.commons-compress-1.23.0`.
@@ -531,9 +526,9 @@ The license file can be found at `licenses/BSD-3-Clause`.
 Copyright notices related to this dependency can be found in the directory `org.jline.jline-terminal-3.26.3`.
 
 
-'jline-terminal-jna', licensed under the The BSD License, is distributed with the engine.
+'jline-terminal-jni', licensed under the The BSD License, is distributed with the engine.
 The license file can be found at `licenses/BSD-3-Clause`.
-Copyright notices related to this dependency can be found in the directory `org.jline.jline-terminal-jna-3.26.3`.
+Copyright notices related to this dependency can be found in the directory `org.jline.jline-terminal-jni-3.26.3`.
 
 
 'org-netbeans-modules-sampler', licensed under the The Apache Software License, Version 2.0, is distributed with the engine.

distribution/engine/THIRD-PARTY/net.java.dev.jna.jna-5.14.0/NOTICES

@@ -0,0 +1,9 @@
+Copyright (C) 2022 the original author(s).
+
+Copyright (c) 2002-2017, the original author(s).
+
+Copyright (c) 2002-2020, the original author or authors.
+
+Copyright (c) 2002-2020, the original author(s).
+
+Copyright (c) 2009-2018, the original author(s).

distribution/engine/THIRD-PARTY/org.jline.jline-terminal-jna-3.26.3/NOTICES

@@ -1,5 +1,6 @@
 package org.enso.runner;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Comparator;
@@ -27,10 +28,10 @@ private JavaFinder() {}
    * @return null if cannot be found. Otherwise, returns the absolute path to the executable, or
    *     simply {@code java} if it is on the {@code PATH}.
    */
-  static String findJavaExecutable() {
+  static File findJavaExecutable() {
     var javaInRuntime = findJavaExecutableInDistributionRuntimes();
     if (javaInRuntime != null) {
-      return javaInRuntime.toAbsolutePath().toString();
+      return javaInRuntime.toAbsolutePath().toFile();
     }
     logger.warn("No appropriate JDK found in the distribution runtimes. Trying system-wide JDK.");
     var javaHome = System.getenv("JAVA_HOME");
@@ -44,7 +45,7 @@ static String findJavaExecutable() {
       }
       if (javaExe.toFile().exists()) {
         logger.info("Found JDK in JAVA_HOME: {}", javaHome);
-        return javaExe.toAbsolutePath().toString();
+        return javaExe.toAbsolutePath().toFile();
       } else {
         logger.warn(
             "No JDK found in JAVA_HOME (missing Java executable at {}). Trying java on PATH.",
@@ -54,10 +55,9 @@ static String findJavaExecutable() {
       logger.warn("JAVA_HOME is not set. Trying java on PATH.");
     }
 
-    if (isJavaOnPath()) {
-      var javaExe = isOnWindows() ? "java.exe" : "java";
-      logger.warn("Falling back to java on PATH: {}", javaExe);
-      return javaExe;
+    if (findJavaOnPath() instanceof File javaExecutable) {
+      logger.warn("Falling back to java on PATH: {}", javaExecutable);
+      return javaExecutable;
     }
     logger.warn("No JDK found on PATH. Cannot start the runtime.");
     return null;
@@ -107,7 +107,7 @@ private static Path findJavaExecutableInDistributionRuntimes() {
     return null;
   }
 
-  private static boolean isJavaOnPath() {
+  private static File findJavaOnPath() {
     try {
       ProcessBuilder processBuilder;
       if (isOnWindows()) {
@@ -116,10 +116,11 @@ private static boolean isJavaOnPath() {
         processBuilder = new ProcessBuilder("java", "-h");
       }
       Process process = processBuilder.start();
+      var pathOpt = process.info().command();
       boolean exitSucc = process.waitFor(5L, TimeUnit.SECONDS);
-      return exitSucc;
+      return exitSucc && pathOpt.isPresent() ? new File(pathOpt.get()) : null;
     } catch (IOException | InterruptedException e) {
-      return false;
+      return null;
     }
   }
 }