[GR-50666] Use exclusively standalones and no longer legacy GraalVM layout

eregon · eregon · commit e97707f54128 · 2024-02-29T10:43:05.000Z
PullRequest: truffleruby/4173
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -71,8 +71,8 @@ jobs:
 
     - name: Create archive
       run: |
-        cd $(dirname $(jt graalvm-home))
-        mv $(basename $(jt graalvm-home)) truffleruby-jvm
+        cd $(dirname $(jt ruby-home))
+        mv $(basename $(jt ruby-home)) truffleruby-jvm
         tar cf ${{ github.workspace }}/truffleruby-jvm.tar truffleruby-jvm
     - uses: actions/upload-artifact@v2
       with:
@@ -114,7 +114,7 @@ jobs:
       - name: Create archive
         run: |
           df -h
-          mv "$(jt -u native graalvm-home)" "${{ github.workspace }}/truffleruby-native"
+          mv "$(jt -u native ruby-home)" "${{ github.workspace }}/truffleruby-native"
           rm -r mxbuild ../graal/sdk/mxbuild
           df -h
           cd ${{ github.workspace }}
diff --git a/ci.jsonnet b/ci.jsonnet
@@ -368,14 +368,6 @@ local part_definitions = {
             jt(["test", "bundle"]),
     },
 
-    test_cexts_sulong: {
-      mx_env:: "toolchain",
-      environment+: {
-        TRUFFLERUBYOPT: "--experimental-options --cexts-sulong",
-      },
-      run+: jt(["test", "specs", ":cext"]),
-    },
-
     testdownstream_aot: { run+: [["mx", "ruby_testdownstream_aot", "$RUBY_BIN"]] },
 
     test_make_standalone_distribution: {
@@ -536,12 +528,10 @@ local composition_environment = utils.add_inclusion_tracking(part_definitions, "
       "ruby-test-cexts-linux-aarch64":  $.platform.linux_aarch64 + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.use.sqlite331 + $.run.test_cexts,
       "ruby-test-cexts-darwin-amd64":   $.platform.darwin_amd64 + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.run.test_cexts + { timelimit: "01:30:00" },
       "ruby-test-cexts-darwin-aarch64": $.platform.darwin_aarch64 + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.run.test_cexts + { timelimit: "00:40:00" },
-      "ruby-test-cexts-sulong":         $.platform.linux  + $.jdk.stable + $.env.jvm + gate + $.run.test_cexts_sulong + { timelimit: "20:00" },
       "ruby-test-gems-linux-amd64":     $.platform.linux  + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.run.test_gems,
       "ruby-test-gems-darwin-amd64":    $.platform.darwin_amd64 + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.run.test_gems,
       "ruby-test-gems-darwin-aarch64":  $.platform.darwin_aarch64 + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.run.test_gems,
       "ruby-test-ecosystem-linux-amd64":  $.platform.linux  + $.jdk.stable + $.env.jvm + gate + $.use.node + $.use.sqlite331 + $.use.gem_test_pack + $.run.test_ecosystem,
-      "ruby-test-standalone-linux-amd64": $.platform.linux  + $.jdk.stable+ gate_no_build + $.run.test_make_standalone_distribution,
 
       "ruby-test-compiler-ce-stable": $.platform.linux + $.jdk.stable + $.env.jvm_ce + gate + $.use.truffleruby + $.run.test_compiler,
       "ruby-test-compiler-ce-latest": $.platform.linux + $.jdk.latest + $.env.jvm_ce + gate + $.use.truffleruby + $.run.test_compiler,
diff --git a/doc/contributor/launchers.md b/doc/contributor/launchers.md
@@ -10,8 +10,8 @@ This file explains the requirements and architecture of the second category of t
 
 ## Requirements
 
-1.  When TruffleRuby tar (GraalVM) is downloaded and unpacked, all files in `bin` and 
-    `languages/ruby/bin` has to always run the TruffleRuby provided by the tar.
+1.  When TruffleRuby tar (GraalVM) is downloaded and unpacked, all files in `bin`
+    have to always run the TruffleRuby provided by the tar.
     -   Therefore `PATH` is not involved (`/usr/bin/env` cannot be used in shebang).
     -   Therefore there are no absolute paths in `bin` executables.
     -   Therefore all scripts in `bin` are always resolved to `truffleruby` in the same dir.
diff --git a/mx.truffleruby/jvm b/mx.truffleruby/jvm
@@ -3,4 +3,4 @@ DYNAMIC_IMPORTS=/tools
 COMPONENTS=TruffleRuby,suite:tools
 # To also create the standalone
 INSTALLABLES=TruffleRuby
-BUILD_TARGETS=GRAALVM,GRAALVM_STANDALONES
+BUILD_TARGETS=GRAALVM_STANDALONES
diff --git a/mx.truffleruby/jvm-ce b/mx.truffleruby/jvm-ce
@@ -3,4 +3,4 @@ DYNAMIC_IMPORTS=/tools,/compiler
 COMPONENTS=TruffleRuby,suite:tools,GraalVM compiler
 # To also create the standalone
 INSTALLABLES=TruffleRuby
-BUILD_TARGETS=GRAALVM,GRAALVM_STANDALONES
+BUILD_TARGETS=GRAALVM_STANDALONES
diff --git a/mx.truffleruby/jvm-ce-libgraal b/mx.truffleruby/jvm-ce-libgraal
@@ -4,4 +4,4 @@ COMPONENTS=TruffleRuby,suite:tools,GraalVM compiler,SubstrateVM,LibGraal
 NATIVE_IMAGES=lib:jvmcicompiler
 # To also create the standalone
 INSTALLABLES=TruffleRuby
-BUILD_TARGETS=GRAALVM,GRAALVM_STANDALONES
+BUILD_TARGETS=GRAALVM_STANDALONES
diff --git a/mx.truffleruby/jvm-ee b/mx.truffleruby/jvm-ee
@@ -4,5 +4,5 @@ COMPONENTS=TruffleRuby,suite:tools,GraalVM enterprise compiler,Truffle enterpris
 NATIVE_IMAGES=false
 # To also create the standalone
 INSTALLABLES=TruffleRuby
-BUILD_TARGETS=GRAALVM,GRAALVM_STANDALONES
+BUILD_TARGETS=GRAALVM_STANDALONES
 # NOTE: SVM EE is added as a workaround to tell the JVM standalone to be EE
diff --git a/mx.truffleruby/jvm-ee-libgraal b/mx.truffleruby/jvm-ee-libgraal
@@ -4,4 +4,4 @@ COMPONENTS=TruffleRuby,suite:tools,GraalVM enterprise compiler,Truffle enterpris
 NATIVE_IMAGES=lib:jvmcicompiler
 # To also create the standalone
 INSTALLABLES=TruffleRuby
-BUILD_TARGETS=GRAALVM,GRAALVM_STANDALONES
+BUILD_TARGETS=GRAALVM_STANDALONES
diff --git a/mx.truffleruby/mx_truffleruby.py b/mx.truffleruby/mx_truffleruby.py
@@ -18,6 +18,7 @@
 import mx_gate
 import mx_sdk
 import mx_sdk_vm
+import mx_sdk_vm_impl
 import mx_subst
 import mx_spotbugs
 
@@ -190,9 +191,9 @@ def ruby_check_heap_dump(input_args, out=None):
         raise Exception("heap dump check failed")
 
 def ruby_run_ruby(args):
-    """run TruffleRuby in $(mx graalvm-home), use bin/jt for more control and shortcuts"""
-    graalvm_home = mx_sdk_vm.graalvm_home(fatalIfMissing=True)
-    ruby = join(graalvm_home, 'languages/ruby/bin/ruby')
+    """run TruffleRuby in $(mx standalone-home --type=jvm ruby), needs an env including a ruby standalone. Use bin/jt for more control and shortcuts"""
+    standalone_home = mx_sdk_vm_impl.standalone_home('ruby', is_jvm=True)
+    ruby = join(standalone_home, 'bin/ruby')
     os.execlp(ruby, ruby, *args)
 
 def ruby_run_specs(ruby, args):
@@ -223,7 +224,9 @@ def ruby_testdownstream_aot(args):
 
 def ruby_spotbugs(args):
     """Run SpotBugs with custom options to detect more issues"""
-    mx.command_function('build')(['--no-native']) # SpotBugs needs all Java projects to be built
+    # SpotBugs needs all Java projects to be built
+    # GR-52408: mx.command_function('build')(['--no-native']) should be enough but it fails
+    mx.command_function('build')([])
 
     filters = join(root, 'mx.truffleruby', 'spotbugs-filters.xml')
     spotbugsArgs = ['-textui', '-low', '-longBugCodes', '-include', filters]
diff --git a/mx.truffleruby/native b/mx.truffleruby/native
@@ -6,4 +6,4 @@ EXTRA_IMAGE_BUILDER_ARGUMENTS=rubyvm:-H:+UnlockExperimentalVMOptions rubyvm:-H:B
 GENERATE_DEBUGINFO=false
 # To also create the standalone
 INSTALLABLES=TruffleRuby
-BUILD_TARGETS=GRAALVM,GRAALVM_STANDALONES
+BUILD_TARGETS=GRAALVM_STANDALONES
diff --git a/mx.truffleruby/native-ee b/mx.truffleruby/native-ee
@@ -6,4 +6,4 @@ EXTRA_IMAGE_BUILDER_ARGUMENTS=rubyvm:-H:+UnlockExperimentalVMOptions rubyvm:-H:B
 GENERATE_DEBUGINFO=false
 # To also create the standalone
 INSTALLABLES=TruffleRuby
-BUILD_TARGETS=GRAALVM,GRAALVM_STANDALONES
+BUILD_TARGETS=GRAALVM_STANDALONES
diff --git a/mx.truffleruby/native-ee-aux b/mx.truffleruby/native-ee-aux
@@ -7,4 +7,4 @@ GENERATE_DEBUGINFO=false
 NATIVE_IMAGE_AUXILIARY_ENGINE_CACHE=true
 # To also create the standalone
 INSTALLABLES=TruffleRuby
-BUILD_TARGETS=GRAALVM,GRAALVM_STANDALONES
+BUILD_TARGETS=GRAALVM_STANDALONES
diff --git a/mx.truffleruby/native-ee-host-inlining b/mx.truffleruby/native-ee-host-inlining
@@ -6,4 +6,4 @@ EXTRA_IMAGE_BUILDER_ARGUMENTS=rubyvm:-H:+UnlockExperimentalVMOptions rubyvm:-H:B
 GENERATE_DEBUGINFO=false
 # To also create the standalone
 INSTALLABLES=TruffleRuby
-BUILD_TARGETS=GRAALVM,GRAALVM_STANDALONES
+BUILD_TARGETS=GRAALVM_STANDALONES
diff --git a/mx.truffleruby/native-ee-polybench b/mx.truffleruby/native-ee-polybench
@@ -6,4 +6,4 @@ EXTRA_IMAGE_BUILDER_ARGUMENTS=rubyvm:-H:+UnlockExperimentalVMOptions rubyvm:-H:B
 GENERATE_DEBUGINFO=false
 # To also create the standalone
 INSTALLABLES=TruffleRuby
-BUILD_TARGETS=GRAALVM,GRAALVM_STANDALONES
+BUILD_TARGETS=GRAALVM_STANDALONES
diff --git a/mx.truffleruby/native-host-inlining b/mx.truffleruby/native-host-inlining
@@ -6,4 +6,4 @@ EXTRA_IMAGE_BUILDER_ARGUMENTS=rubyvm:-H:+UnlockExperimentalVMOptions rubyvm:-H:B
 GENERATE_DEBUGINFO=false
 # To also create the standalone
 INSTALLABLES=TruffleRuby
-BUILD_TARGETS=GRAALVM,GRAALVM_STANDALONES
+BUILD_TARGETS=GRAALVM_STANDALONES
diff --git a/mx.truffleruby/native-profiling b/mx.truffleruby/native-profiling
@@ -6,4 +6,4 @@ EXTRA_IMAGE_BUILDER_ARGUMENTS=rubyvm:-H:+UnlockExperimentalVMOptions rubyvm:-H:B
 GENERATE_DEBUGINFO=false
 # To also create the standalone
 INSTALLABLES=TruffleRuby
-BUILD_TARGETS=GRAALVM,GRAALVM_STANDALONES
+BUILD_TARGETS=GRAALVM_STANDALONES
diff --git a/mx.truffleruby/suite.py b/mx.truffleruby/suite.py
@@ -20,7 +20,7 @@
             {
                 "name": "regex",
                 "subdir": True,
-                "version": "3f39def1e63211bae1359722528902f979a8afcf",
+                "version": "8f380d326998db82b7c6c7a354eac800ac1ca8a2",
                 "urls": [
                     {"url": "https://github.com/oracle/graal.git", "kind": "git"},
                     {"url": "https://curio.ssw.jku.at/nexus/content/repositories/snapshots", "kind": "binary"},
@@ -29,7 +29,7 @@
             {
                 "name": "sulong",
                 "subdir": True,
-                "version": "3f39def1e63211bae1359722528902f979a8afcf",
+                "version": "8f380d326998db82b7c6c7a354eac800ac1ca8a2",
                 "urls": [
                     {"url": "https://github.com/oracle/graal.git", "kind": "git"},
                     {"url": "https://curio.ssw.jku.at/nexus/content/repositories/snapshots", "kind": "binary"},
diff --git a/mx.truffleruby/toolchain b/mx.truffleruby/toolchain
diff --git a/src/test-internal/java/org/truffleruby/test/internal/RubyDebugTest.java b/src/test-internal/java/org/truffleruby/test/internal/RubyDebugTest.java
@@ -286,7 +286,7 @@ public void testInlineModifiesFrame() throws Throwable {
     }
 
     // To debug how the breakpoint is set in this test, it is useful to run:
-    // $ mx -d unittest -Dorg.graalvm.language.ruby.home=$PWD/mxbuild/truffleruby-jvm/languages/ruby -Dtruffle.instrumentation.trace=true RubyDebugTest#testProperties
+    // $ mx -d unittest -Dorg.graalvm.language.ruby.home=$PWD/mxbuild/truffleruby-jvm -Dtruffle.instrumentation.trace=true RubyDebugTest#testProperties
     // and put a breakpoint on com.oracle.truffle.api.debug.SuspendableLocationFinder.findNearestBound.
     @Test
     public void testProperties() throws Throwable {
diff --git a/tool/jt.rb b/tool/jt.rb
@@ -256,7 +256,7 @@ def ruby_launcher
                       File.directory?(@ruby_name) ? "#{@ruby_name}/bin/ruby" : @ruby_name
                     else
                       graalvm = "#{TRUFFLERUBY_DIR}/mxbuild/truffleruby-#{@ruby_name}"
-                      "#{graalvm}/languages/ruby/bin/ruby"
+                      "#{graalvm}/bin/ruby"
                     end
 
     raise "The Ruby executable #{ruby_launcher} does not exist" unless File.exist?(ruby_launcher)
@@ -268,7 +268,6 @@ def ruby_launcher
     unless @silent
       shortened_path = @ruby_launcher.sub(%r[^#{Regexp.escape TRUFFLERUBY_DIR}/], '')
       shortened_path = shortened_path.sub(%r[/bin/(ruby|truffleruby)$], '')
-      shortened_path = shortened_path.sub(%r[/languages/ruby$], '') if graalvm_home
       tags = [*('Interpreted' if truffleruby? && !truffleruby_compiler?),
               truffleruby? ? 'TruffleRuby' : 'a Ruby',
               *('with Native' if truffleruby_native_built?),
@@ -287,18 +286,7 @@ def ruby_home
     require_ruby_launcher!
     File.expand_path('../..', @ruby_launcher_realpath)
   end
-
-  def graalvm_home
-    up = if ruby_home.end_with?('jre/languages/ruby')
-           3
-         elsif ruby_home.end_with?('languages/ruby')
-           2
-         else
-           nil # standalone
-         end
-    return nil unless up
-    File.expand_path((['..'] * up).join('/'), ruby_home)
-  end
+  alias_method :'ruby-home', :ruby_home
 
   def truffleruby!
     raise 'This command requires TruffleRuby.' unless truffleruby?
@@ -350,8 +338,8 @@ def truffleruby_compiler?
     return @truffleruby_compiler = false unless truffleruby?
     return @truffleruby_compiler = true if truffleruby_native_built?
 
-    # Detect if the compiler is present by reading the $graalvm_home/release file
-    @truffleruby_compiler = File.readlines("#{graalvm_home || ruby_home}/release").grep(/^COMMIT_INFO=/).any? do |line|
+    # Detect if the compiler is present by reading the $ruby_home/release file
+    @truffleruby_compiler = File.readlines("#{ruby_home}/release").grep(/^COMMIT_INFO=/).any? do |line|
       line.include?('"compiler":') || line.include?("'compiler':")
     end
   end
@@ -898,7 +886,7 @@ def help
       jt sync                                        continuously synchronize changes from the Ruby source files to the GraalVM build
       jt idea                                        generates IntelliJ projects
       jt format                                      run eclipse code formatter
-      jt graalvm-home                                prints the GraalVM home of the RUBY_SELECTOR
+      jt ruby-home                                   prints the Ruby home of the RUBY_SELECTOR
 
       you can also put --build or --rebuild in front of any command to build or rebuild first
 
@@ -929,10 +917,6 @@ def launcher
     puts ruby_launcher
   end
 
-  define_method(:'graalvm-home') do
-    puts graalvm_home
-  end
-
   def build(*options)
     in_truffleruby_repo_root!
     project = options.shift
@@ -2540,11 +2524,11 @@ def sforceimports
     mx_options, mx_build_options = args_split(options)
     mx_args = mx_base_args + mx_options
 
+    standalone_type = env.include?('native') ? 'native' : 'jvm'
     mx(*mx_args, 'build', *mx_build_options, primary_suite: TRUFFLERUBY_DIR)
-    build_dir = mx(*mx_args, 'graalvm-home', primary_suite: TRUFFLERUBY_DIR, capture: :out).lines.last.chomp
+    build_dir = mx(*mx_args, 'standalone-home', "--type=#{standalone_type}", 'ruby', primary_suite: TRUFFLERUBY_DIR, capture: :out).lines.last.chomp
 
     dest = "#{TRUFFLERUBY_DIR}/mxbuild/#{name}"
-    dest_ruby = "#{dest}/languages/ruby"
     FileUtils.rm_rf dest
     if @ruby_name != @mx_env
       # if `--name NAME` is passed, we want to copy so we don't end up with two symlinks
@@ -2573,7 +2557,7 @@ def sforceimports
 
       link_path = "#{rubies_dir}/#{name}"
       File.delete link_path if File.symlink? link_path or File.exist? link_path
-      File.symlink dest_ruby, link_path
+      File.symlink dest, link_path
     end
   end
 
@@ -3171,10 +3155,6 @@ def docker(*args)
     require_relative 'docker'
     JT::Docker.new.docker(*args)
   end
-
-  def visualvm
-    sh "#{graalvm_home}/bin/jvisualvm"
-  end
 end
 
 class JT
diff --git a/tool/local-backtrace.rb b/tool/local-backtrace.rb
@@ -3,6 +3,5 @@
 contents = STDIN.read
 contents = contents.gsub("/b/b/e/main", ".")
 contents = contents.gsub(%r{/Users/graal\d+/slave/e/main/}, "./")
-contents = contents.gsub("/mxbuild/truffleruby-jvm/jre/languages/ruby/", "/")
-contents = contents.gsub(%r{\S+/languages/ruby/}, "./")
+contents = contents.gsub("/mxbuild/truffleruby-jvm/", "/")
 STDOUT.puts contents
diff --git a/tool/make-standalone-distribution.sh b/tool/make-standalone-distribution.sh
