[GR-47781] [GR-45161] Split unit tests into embedding and internal tests and run embedding tests on module path

PullRequest: truffleruby/3954

Author: eregon

.gitattributes

@@ -28,7 +28,8 @@
 /src/main/c/truffleposix/Makefile linguist-vendored
 /lib/gems/gems/*/bin/setup linguist-vendored
 /lib/truffle/post_install_hook.sh linguist-vendored
-/src/test/* linguist-vendored
+/src/test-embedding/* linguist-vendored
+/src/test-internal/* linguist-vendored
 /src/processor/* linguist-vendored
 
 # Generated code

.rubocop.yml

@@ -17,7 +17,7 @@ AllCops:
     - 'lib/truffle/socket/mri.rb' # taken from MRI
     - 'lib/truffle/ffi/**/*.rb' # taken from FFI gem
     - 'src/main/ruby/truffleruby/core/truffle/ffi/pointer_extra.rb' # taken from FFI gem
-    - 'src/test/ruby/types.rb' # deliberately strange code for debugging purposes
+    - 'src/test-internal/ruby/types.rb' # deliberately strange code for debugging purposes
     - 'src/tck/ruby/lexical-context.rb' # deliberately strange code for debugging purposes
 
 # Type 'Layout' (166):

CHANGELOG.md

@@ -49,6 +49,7 @@ Performance:
 
 Changes:
 
+* The TruffleRuby `ScriptEngine` implementation is removed in favor of the generic [ScriptEngine](https://github.com/oracle/graal/blob/master/docs/reference-manual/embedding/embed-languages.md#compatibility-with-jsr-223-scriptengine) in GraalVM docs.
 
 Memory Footprint:
 

common.json

@@ -4,7 +4,7 @@
     "Jsonnet files should not include this file directly but use ci/common.jsonnet instead."
   ],
 
-  "mx_version": "6.41.0",
+  "mx_version": "6.42.0",
 
   "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet",
   "jdks": {
@@ -42,7 +42,7 @@
     "labsjdk-ee-21Debug": {"name": "labsjdk",   "version": "ee-21+35-jvmci-23.1-b13-debug", "platformspecific": true },
     "labsjdk-ee-21-llvm": {"name": "labsjdk",   "version": "ee-21+35-jvmci-23.1-b13-sulong", "platformspecific": true },
 
-    "oraclejdk22":        {"name": "jpg-jdk",   "version": "22",      "build_id": "2", "release": true, "platformspecific": true, "extrabundles": ["static-libs"]}
+    "oraclejdk22":        {"name": "jpg-jdk",   "version": "22",      "build_id": "8", "release": true, "platformspecific": true, "extrabundles": ["static-libs"]}
   },
 
   "eclipse": {

doc/contributor/using-eclipse.md

@@ -37,11 +37,7 @@ We can now import the projects:
 * Select this repository as root directory
 * Click `Finish`
 
-There should be now 4 projects in your workspace:
-* `truffleruby`
-* `truffleruby-test`
-* `TRUFFLERUBY-TEST.dist`
-* `TRUFFLERUBY.dist`
+There should be now some projects in your workspace, like `truffleruby` and `TRUFFLERUBY.dist`.
 
 ## Advanced Setup
 

doc/user/jruby-migration.md

@@ -21,10 +21,11 @@ The standalone distribution does not allow for Java interoperability.
 
 JRuby supports many different ways to embed Ruby in Java, including JSR 223 (also know as `javax.script`), the Bean Scripting Framework (BSF), JRuby Embed (also known as Red Bridge), and the JRuby direct embedding API.
 
-Thes best way to embed TruffleRuby is to use the Polyglot API, which is part of GraalVM.
+The best way to embed TruffleRuby is to use the Polyglot API, which is part of GraalVM.
 The API is different because it is designed to support many languages, not just Ruby.
 
 TruffleRuby also supports JSR 223, compatible with JRuby, to make it easier to run legacy JRuby code.
+See [this documentation](https://github.com/oracle/graal/blob/master/docs/reference-manual/embedding/embed-languages.md#compatibility-with-jsr-223-scriptengine) for how to use it.
 
 You will need to use GraalVM to use both of these APIs.
 

mx.truffleruby/mx_truffleruby.py

@@ -119,7 +119,7 @@ def contents(self, result):
             '--disable-gems',
             '--disable-rubyopt',
         ]
-        command = [jdk.java] + jvm_args + [main_class] + ruby_options + ['"$@"']
+        command = [jdk.java] + jvm_args + ['-m', 'org.graalvm.ruby.launcher/' + main_class] + ruby_options + ['"$@"']
         return "#!/usr/bin/env bash\n" + "exec " + " ".join(command) + "\n"
 
 
@@ -143,14 +143,14 @@ def build_truffleruby(args):
 
 def ruby_check_heap_dump(input_args, out=None):
     print("mx ruby_check_heap_dump " + " ".join(input_args))
-    dists = ['TRUFFLERUBY', 'TRUFFLE_NFI', 'SULONG_NATIVE', 'TRUFFLERUBY-TEST']
+    dists = ['TRUFFLERUBY', 'TRUFFLE_NFI', 'SULONG_NATIVE', 'TRUFFLERUBY-TEST-INTERNAL']
     mx.command_function('build')(['--dependencies', ','.join(dists)])
     args = input_args
     args.insert(0, "--experimental-options")
     vm_args, truffleruby_args = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=False)
     vm_args += mx.get_runtime_jvm_args(dists)
     # vm_args.append("-agentlib:jdwp=transport=dt_socket,server=y,address=8000,suspend=y")
-    vm_args.append("org.truffleruby.LeakTest")
+    vm_args.append("org.truffleruby.test.internal.LeakTest")
     out = mx.OutputCapture() if out is None else out
     retval = mx.run_java(vm_args + truffleruby_args, jdk=jdk, nonZeroIsFatal=False, out=out)
     if retval == 0:
@@ -273,9 +273,6 @@ def verify_ci(args):
         'truffleruby:JCODINGS',
         'truffleruby:JONI',
     ],
-    boot_jars=[
-        'truffleruby:TRUFFLERUBY-SERVICES'
-    ],
     support_distributions=[
         'truffleruby:TRUFFLERUBY_GRAALVM_SUPPORT',
         'truffleruby:TRUFFLERUBY_GRAALVM_SUPPORT_NO_NI_RESOURCES',

mx.truffleruby/suite.py

@@ -1,5 +1,5 @@
 suite = {
-    "mxversion": "6.41.0",
+    "mxversion": "6.42.0",
     "name": "truffleruby",
     "groupId": "org.graalvm.ruby",
     "url": "https://www.graalvm.org/ruby/",
@@ -20,7 +20,7 @@
             {
                 "name": "regex",
                 "subdir": True,
-                "version": "97a02e74911e7446dbf53098f885bc2fceba6770",
+                "version": "e7b40b1010e34bde2e64993ba8cece3833ec60e8",
                 "urls": [
                     {"url": "https://github.com/oracle/graal.git", "kind": "git"},
                     {"url": "https://curio.ssw.jku.at/nexus/content/repositories/snapshots", "kind": "binary"},
@@ -29,7 +29,7 @@
             {
                 "name": "sulong",
                 "subdir": True,
-                "version": "97a02e74911e7446dbf53098f885bc2fceba6770",
+                "version": "e7b40b1010e34bde2e64993ba8cece3833ec60e8",
                 "urls": [
                     {"url": "https://github.com/oracle/graal.git", "kind": "git"},
                     {"url": "https://curio.ssw.jku.at/nexus/content/repositories/snapshots", "kind": "binary"},
@@ -128,7 +128,6 @@
                 "test/truffle/ecosystem/blog6",
                 "test/truffle/ecosystem/hello-world",
                 "test/truffle/ecosystem/rails-app",
-                "test/truffle/offline",
                 "tool/docker",
                 "rubyspec_temp",
             ]
@@ -320,15 +319,31 @@
             "license": ["EPL-2.0"],
         },
 
-        "org.truffleruby.test": {
-            "dir": "src/test",
+        "org.truffleruby.test.embedding": {
+            "testProject": True,
+            "dir": "src/test-embedding",
+            "sourceDirs": ["java"],
+            "requires": ["java.scripting"],
+            "dependencies": [
+                # Distributions
+                "sdk:POLYGLOT",
+                # Libraries
+                "mx:JUNIT",
+            ],
+            "javaCompliance": "17+",
+            "checkstyle": "org.truffleruby",
+            "license": ["EPL-2.0"],
+        },
+
+        "org.truffleruby.test.internal": {
+            "testProject": True,
+            "dir": "src/test-internal",
             "sourceDirs": ["java", "ruby"],
-            "requires": ["java.scripting", "java.management", "jdk.management"],
+            "requires": ["java.management", "jdk.management"],
             "dependencies": [
                 # Distributions
                 "sdk:LAUNCHER_COMMON",
                 "TRUFFLERUBY",
-                "TRUFFLERUBY-SERVICES",
                 # Libraries
                 "mx:JUNIT",
                 "truffleruby:NETBEANS-LIB-PROFILER",
@@ -415,6 +430,7 @@
                 "tag": ["default", "public"],
             },
             "noMavenJavadoc": True,
+            "useModulePath": True,
         },
 
         # Required to share code between the launcher and the rest,
@@ -442,6 +458,7 @@
                 "tag": ["default", "public"],
             },
             "noMavenJavadoc": True,
+            "useModulePath": True,
         },
 
         "TRUFFLERUBY-PROCESSOR": {
@@ -457,24 +474,6 @@
             "maven": False,
         },
 
-        "TRUFFLERUBY-SERVICES": {
-            "moduleInfo": {
-                "name": "org.graalvm.ruby.services",
-                "exports": [
-                    "org.truffleruby.services.scriptengine",
-                ],
-            },
-            "dependencies": [
-                "org.truffleruby.services"
-            ],
-            "distDependencies": [
-                "sdk:GRAAL_SDK",
-            ],
-            "description": "TruffleRuby services",
-            "license": ["EPL-2.0"],
-            "maven": False,
-        },
-
         "TRUFFLERUBY": {
             "moduleInfo": {
                 "name": "org.graalvm.ruby",
@@ -492,6 +491,9 @@
                 "sulong:SULONG_API",
                 "sulong:SULONG_NFI",
                 "sdk:JLINE3",
+                # runtime-only dependencies
+                "truffle:TRUFFLE_NFI_LIBFFI",
+                "sulong:SULONG_NATIVE",
             ],
             "exclude": [ # Keep in sync with org.truffleruby dependencies and truffle_jars in mx_truffleruby.py
                 "truffleruby:JCODINGS",
@@ -509,6 +511,7 @@
                 "tag": ["default", "public"],
             },
             "noMavenJavadoc": True,
+            "useModulePath": True,
         },
 
         "RUBY_COMMUNITY": {
@@ -560,6 +563,7 @@
             "description": "TruffleRuby Launcher",
             "license": ["EPL-2.0"],
             "maven": False,
+            "useModulePath": True,
         },
 
         "TRUFFLERUBY_GRAALVM_SUPPORT": {
@@ -675,15 +679,35 @@
             },
         },
 
-        "TRUFFLERUBY-TEST": {
+        "TRUFFLERUBY-TEST-EMBEDDING": {
+            "testDistribution": True,
+            "dependencies": [
+                "org.truffleruby.test.embedding",
+            ],
+            "distDependencies": [
+                "sdk:POLYGLOT",
+                # runtime-only dependencies
+                "TRUFFLERUBY",
+            ],
+            "exclude": [
+                "mx:HAMCREST",
+                "mx:JUNIT",
+            ],
+            "javaProperties": {
+                "org.graalvm.language.ruby.home": "<path:TRUFFLERUBY_GRAALVM_SUPPORT>"
+            },
+            "license": ["EPL-2.0"],
+            "maven": False,
+        },
+
+        "TRUFFLERUBY-TEST-INTERNAL": {
             "testDistribution": True,
             "dependencies": [
-                "org.truffleruby.test",
+                "org.truffleruby.test.internal",
             ],
             "distDependencies": [
                 "sdk:LAUNCHER_COMMON",
                 "TRUFFLERUBY",
-                "TRUFFLERUBY-SERVICES",
             ],
             "exclude": [
                 "mx:HAMCREST",

src/launcher/java/org/truffleruby/launcher/RubyLauncher.java

@@ -371,7 +371,7 @@ private void printPreRunInformation(CommandLineOptions config) {
 
     private String getImplementationNameFromEngine() {
         if (implementationName == null) {
-            try (Engine engine = Engine.create()) {
+            try (Engine engine = Engine.newBuilder().option("engine.WarnInterpreterOnly", "false").build()) {
                 implementationName = engine.getImplementationName();
             }
         }

src/main/java/org/truffleruby/language/loader/EmbeddedScript.java

@@ -103,7 +103,7 @@ private boolean isShebangLine(byte[] bytes, int lineStart) {
         return bytes.length - lineStart >= 2 && bytes[lineStart] == '#' && bytes[lineStart + 1] == '!';
     }
 
-    static boolean lineContainsRuby(byte[] bytes, int lineStart, int lineLength) {
+    public static boolean lineContainsRuby(byte[] bytes, int lineStart, int lineLength) {
         if (lineLength < 4) {
             return false;
         }