[GR-26970] Backports for 20.3

PullRequest: truffleruby/2096

Author: gilles-duboscq

.gitattributes

@@ -1,7 +1,3 @@
-# Merge options
-
-/CHANGELOG.md merge=union
-
 # Rules for GitHub's Linguist language-classification system. We're abusing the
 # 'vendored' attribute to exclude files as a lot of this isn't really vendored,
 # and a whole lot of actually vendored code isn't listed! What we want to do is

3rd_party_licenses.txt

@@ -41,7 +41,7 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SU
 ================================================================================
 
-JONI 2.1.30
+JONI 2.1.40
 
 /*
 * Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -365,7 +365,7 @@ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 SUCH DAMAGE.
 ================================================================================
 
-JCodings 1.0.45
+JCodings 1.0.55
 
 /*
 * Permission is hereby granted, free of charge, to any person obtaining a copy

CHANGELOG.md

@@ -6,7 +6,7 @@ New features:
 Bug fixes:
 
 * Handle foreign null object as falsy value (#1902, @ssnickolay)
-* Fixed return value of `Enumerable#first` (#2056, @LillianZ).
+* Fixed return value of `Enumerable#first` with multiple yielded arguments (#2056, @LillianZ).
 * Improve reliability of the post install hook by disabling RubyGems (#2075).
 * Fixed top level exception handler to print exception cause (#2013).
 * Fixed issue when extending FFI from File (#2094).
@@ -15,8 +15,10 @@ Bug fixes:
 * Fixed issue with `spawn` when `:close` redirect is used (#2097).
 * Fixed `coverage` issue when `*eval` is used (#2078).
 * Use expanded load paths for feature matching (#1501).
+* Fixed handling of post arguments for `super()` (#2111).
 * Fixed `SystemStackError` sometimes replaced by an internal Java `NoClassDefFoundError` on JVM (#1743).
 * Fixed constant/identifier detection in lexer for non-ASCII encodings (#2079, #2102, @ivoanjo).
+* Fixed parsing of `--jvm` as an application argument (#2108).
 
 Compatibility:
 
@@ -49,6 +51,8 @@ Compatibility:
 * Convert objects with `#to_path` in `$LOAD_PATH` (#2119).
 * Handle the functions being native for `rb_thread_call_without_gvl()` (#2090).
 * Support refinements for Kernel#respond_to? (#2120, @ssnickolay)
+* JCodings has been updated from 1.0.45 to 1.0.55.
+* Joni has been updated from 2.1.30 to 2.1.40.
 
 Performance:
 

mx.truffleruby/suite.py

@@ -51,21 +51,21 @@
             "maven": {
                 "groupId": "org.jruby.joni",
                 "artifactId": "joni",
-                "version": "2.1.30"
+                "version": "2.1.40"
             },
-            "sha1": "a23a567521996c2a412688763892cddbca7c3bd6",
-            "sourceSha1": "a1444342fc0c275613d43ca3d0a71ce919d85b18",
+            "sha1": "18766fa1b624d615d3a1cac513bf729adfd6b38f",
+            "sourceSha1": "85bb9546e4c63f00f02531eeab795685566baf7b",
             "license": ["MIT"],
         },
 
         "JCODINGS": {
             "maven": {
                 "groupId": "org.jruby.jcodings",
                 "artifactId": "jcodings",
-                "version": "1.0.45"
+                "version": "1.0.55"
             },
-            "sha1": "029404c013b3d51a8c60fac80409bb3d64dfb816",
-            "sourceSha1": "702693ea01e006385f2834ad56dad54b7a4ca248",
+            "sha1": "57169ef6964f44aef67b247cb87f053d93182488",
+            "sourceSha1": "071891163f601416a5426d8f05b7d46139217ff1",
             "license": ["MIT"],
         },
     },

spec/tags/truffle/launcher_tags.txt

@@ -56,6 +56,11 @@ slow:The launcher supports running rdoc symlinked
 slow:The launcher supports running ri symlinked
 slow:The launcher supports running ruby symlinked
 slow:The launcher supports running truffleruby symlinked
+arm64:The launcher for gem can install and uninstall the hello-world gem
 slow:The launcher should recognize ruby --vm options in RUBYOPT
 slow:The launcher should recognize ruby --vm options in TRUFFLERUBYOPT
-arm64:The launcher for gem can install and uninstall the hello-world gem
+slow:The launcher runtime configuration flags should recognize ruby --vm options in RUBYOPT when switching to JVM
+slow:The launcher runtime configuration flags should recognize ruby --vm options in TRUFFLERUBYOPT when switching to JVM
+slow:The launcher runtime configuration flags uses --native by default
+slow:The launcher runtime configuration flags switches to JVM with --jvm as a Ruby argument
+slow:The launcher runtime configuration flags keeps --jvm as an application argument if given as an application argument

spec/truffle/fixtures/argv.rb

@@ -0,0 +1 @@
+p ARGV

spec/truffle/launcher_spec.rb

@@ -145,14 +145,6 @@ def check_status_or_print(stdout_and_stderr)
     end
   end
 
-  ['RUBYOPT', 'TRUFFLERUBYOPT'].each do |var|
-    it "should recognize ruby --vm options in #{var}" do
-      out = ruby_exe('print Truffle::System.get_java_property("foo")', env: { var => "--vm.Dfoo=bar" }, args: @redirect)
-      check_status_and_empty_stderr
-      out.should == 'bar'
-    end
-  end
-
   def should_print_full_java_command(options, env: {})
     out = ruby_exe(nil, options: options, env: env, args: @redirect)
     check_status_and_empty_stderr
@@ -410,4 +402,60 @@ def should_print_full_java_command(options, env: {})
     end
   end
 
+  ['RUBYOPT', 'TRUFFLERUBYOPT'].each do |var|
+    it "should recognize ruby --vm options in #{var}" do
+      out = ruby_exe('print Truffle::System.get_java_property("foo")', env: { var => "--vm.Dfoo=bar" }, args: @redirect)
+      check_status_and_empty_stderr
+      out.should == 'bar'
+    end
+  end
+
+  guard -> {
+    # GraalVM with both --jvm and --native
+    TruffleRuby.graalvm_home and TruffleRuby.native?
+  } do
+    describe "runtime configuration flags" do
+      before :each do
+        @trufflerubyopt = ENV['TRUFFLERUBYOPT']
+        # remove --native/--jvm from $TRUFFLERUBYOPT as they can conflict with command line arguments for these specs
+        ENV['TRUFFLERUBYOPT'] = @trufflerubyopt.to_s.gsub(/--(native|jvm)\b/, '')
+      end
+
+      after :each do
+        ENV['TRUFFLERUBYOPT'] = @trufflerubyopt
+      end
+
+      ['RUBYOPT', 'TRUFFLERUBYOPT'].each do |var|
+        it "should recognize ruby --vm options in #{var} when switching to JVM" do
+          env = { var => "--jvm --vm.Dfoo=bar" }
+          out = ruby_exe('puts RUBY_DESCRIPTION; puts Truffle::System.get_java_property("foo")', env: env, args: @redirect)
+          check_status_and_empty_stderr
+          out = out.lines.map(&:chomp)
+          out[0].should =~ /GraalVM (CE|EE) JVM/
+          out[1].should == 'bar'
+        end
+      end
+
+      it "uses --native by default" do
+        out = ruby_exe(nil, options: "--version", args: @redirect)
+        check_status_and_empty_stderr
+        out.should =~ /GraalVM (CE|EE) Native/
+      end
+
+      it "switches to JVM with --jvm as a Ruby argument" do
+        out = ruby_exe(nil, options: "--jvm --version", args: @redirect)
+        check_status_and_empty_stderr
+        out.should =~ /GraalVM (CE|EE) JVM/
+      end
+
+      it "keeps --jvm as an application argument if given as an application argument" do
+        script = fixture(__FILE__, "argv.rb")
+        out = ruby_exe(nil, options: "-v", args: "#{script} --jvm 1 2 #{@redirect}")
+        check_status_and_empty_stderr
+        out = out.lines.map(&:chomp)
+        out[0].should =~ /GraalVM (CE|EE) Native/
+        out.should.include?('["--jvm", "1", "2"]')
+      end
+    end
+  end
 end

src/launcher/java/org/truffleruby/launcher/CommandLineOptions.java

@@ -31,7 +31,6 @@
 
 import java.io.File;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -58,26 +57,17 @@ public class CommandLineOptions {
     /** A thing to be executed: a file, inline script, etc. Used by executionAction when applicable. */
     String toExecute = "";
 
-    /** This should not be modified, as otherwise when exec()-ing to JVM from a native launcher, these options would be
-     * passed on the command line, which fails if they are experimental. This would also cause parsing the options twice
-     * with the current Launcher design. */
-    private final Map<String, String> polyglotOptions;
     private final Map<String, String> options;
     private String[] arguments;
     private final List<String> unknownArguments;
     private Boolean gemOrBundle = null;
 
-    public CommandLineOptions(Map<String, String> polyglotOptions) {
-        this.polyglotOptions = Collections.unmodifiableMap(polyglotOptions);
+    public CommandLineOptions() {
         this.options = new HashMap<>();
         this.arguments = EMPTY_STRING_ARRAY;
         this.unknownArguments = new ArrayList<>();
     }
 
-    boolean isSetInPolyglotOptions(String optionName) {
-        return polyglotOptions.containsKey(optionName);
-    }
-
     public Map<String, String> getOptions() {
         return options;
     }

src/launcher/java/org/truffleruby/launcher/CommandLineParser.java

@@ -40,7 +40,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.function.BiConsumer;
 import java.util.logging.Handler;
 import java.util.logging.LogRecord;
@@ -56,25 +55,20 @@ public class CommandLineParser {
 
     private final List<String> arguments;
     private int argumentIndex;
-    private boolean processArgv;
+    private final boolean processArgv;
     private final boolean rubyOpts;
     final CommandLineOptions config;
     private int lastInterpreterArgumentIndex;
     private int characterIndex;
 
-    public CommandLineParser(
-            List<String> arguments,
-            CommandLineOptions config,
-            boolean processArgv,
-            boolean rubyOpts) {
-
+    public CommandLineParser(List<String> arguments, CommandLineOptions config, boolean processArgv, boolean rubyOpts) {
         this.argumentIndex = 0;
         this.characterIndex = 0;
         this.lastInterpreterArgumentIndex = -1;
         this.config = config;
         this.processArgv = processArgv;
         this.rubyOpts = rubyOpts;
-        this.arguments = Objects.requireNonNull(arguments);
+        this.arguments = Collections.unmodifiableList(arguments);
     }
 
     public void processArguments() throws CommandLineException {
@@ -102,19 +96,15 @@ public void processArguments() throws CommandLineException {
         }
     }
 
-    public int getLastInterpreterArgumentIndex() {
-        return lastInterpreterArgumentIndex;
-    }
-
     private boolean endOfInterpreterArguments() {
         return lastInterpreterArgumentIndex != -1;
     }
 
     private void processArgv() {
+        boolean argvGlobalsOn = config.getOption(OptionsCatalog.ARGV_GLOBALS);
         ArrayList<String> arglist = new ArrayList<>();
         for (; argumentIndex < arguments.size(); argumentIndex++) {
             String arg = getCurrentArgument();
-            boolean argvGlobalsOn = config.getOption(OptionsCatalog.ARGV_GLOBALS);
             if (argvGlobalsOn && arg.startsWith("-")) {
                 arg = arg.substring(1);
                 int split = arg.indexOf('=');

src/launcher/java/org/truffleruby/launcher/RubyLauncher.java

@@ -61,7 +61,15 @@ protected void printVersion() {
 
     @Override
     protected List<String> preprocessArguments(List<String> args, Map<String, String> polyglotOptions) {
-        config = new CommandLineOptions(polyglotOptions);
+        // Set default options for the launcher which don't match the OptionKey's default.
+        // These options can still be overridden if set explicitly.
+        polyglotOptions.put(OptionsCatalog.EMBEDDED.getName(), "false");
+        if (isAOT()) {
+            final String launcher = ProcessProperties.getExecutableName();
+            polyglotOptions.put(OptionsCatalog.LAUNCHER.getName(), launcher);
+        }
+
+        config = new CommandLineOptions();
 
         try {
             config.executionAction = ExecutionAction.UNSET;
@@ -70,26 +78,18 @@ protected List<String> preprocessArguments(List<String> args, Map<String, String
             argumentCommandLineParser.processArguments();
 
             if (config.readRubyOptEnv) {
+                /* Calling processArguments() here will also add any unrecognized arguments such as
+                 * --jvm/--native/--vm.* arguments and polyglot options to `config.getUnknownArguments()`, which will
+                 * then be processed by AbstractLanguageLauncher and Launcher. If we are going to run Native, Launcher
+                 * will apply VM options to the current process. If we are going to run on JVM, Launcher will collect
+                 * them and pass them when execve()'ing to bin/java. Polyglot options are parsed by
+                 * AbstractLanguageLauncher in the final process. */
                 // Process RUBYOPT
                 final List<String> rubyoptArgs = getArgsFromEnvVariable("RUBYOPT");
                 new CommandLineParser(rubyoptArgs, config, false, true).processArguments();
                 // Process TRUFFLERUBYOPT
                 final List<String> trufflerubyoptArgs = getArgsFromEnvVariable("TRUFFLERUBYOPT");
                 new CommandLineParser(trufflerubyoptArgs, config, false, false).processArguments();
-
-                if (isAOT()) {
-                    /* Append options from ENV variables to args after the last interpreter option, which makes sure
-                     * that maybeExec() processes the --vm.* options. These options are removed and are not passed to
-                     * the new process if exec() is being called as these options need to be passed when starting the
-                     * new VM process. The new process gets all arguments and options including those from ENV
-                     * variables. To avoid processing options from ENV variables twice, --disable-rubyopt is passed.
-                     * Only the native launcher can apply native and jvm options (it is too late for the running JVM to
-                     * apply --vm options), therefore this is not done on JVM. */
-                    final int index = argumentCommandLineParser.getLastInterpreterArgumentIndex();
-                    args.add(index, "--disable-rubyopt");
-                    args.addAll(index + 1, rubyoptArgs);
-                    args.addAll(index + 1 + rubyoptArgs.size(), trufflerubyoptArgs);
-                }
             }
 
             // Process RUBYLIB, must be after arguments and RUBYOPT
@@ -236,15 +236,6 @@ private int runRubyMain(Context.Builder contextBuilder, CommandLineOptions confi
     }
 
     private Context createContext(Context.Builder builder, CommandLineOptions config) {
-        if (isAOT() && !config.isSetInPolyglotOptions(OptionsCatalog.LAUNCHER.getName())) {
-            final String launcher = ProcessProperties.getExecutableName();
-            builder.option(OptionsCatalog.LAUNCHER.getName(), launcher);
-        }
-
-        if (!config.isSetInPolyglotOptions(OptionsCatalog.EMBEDDED.getName())) {
-            builder.option(OptionsCatalog.EMBEDDED.getName(), "false");
-        }
-
         if (config.isGemOrBundle() && getImplementationNameFromEngine().contains("Graal")) {
             // Apply options to run gem/bundle more efficiently
             builder.option("engine.Mode", "latency");

src/main/java/org/truffleruby/core/TruffleSystemNodes.java

@@ -146,6 +146,17 @@ protected Object setTruffleWorkingDir(RubyString dir) {
         }
     }
 
+    @Primitive(name = "working_directory")
+    public abstract static class GetTruffleWorkingDirNode extends PrimitiveArrayArgumentsNode {
+        @Specialization
+        protected RubyString getTruffleWorkingDir(
+                @Cached StringNodes.MakeStringNode makeStringNode) {
+            final String cwd = getContext().getFeatureLoader().getWorkingDirectory();
+            final Encoding externalEncoding = getContext().getEncodingManager().getDefaultExternalEncoding();
+            return makeStringNode.executeMake(cwd, externalEncoding, CodeRange.CR_UNKNOWN);
+        }
+    }
+
     @CoreMethod(names = "get_java_property", onSingleton = true, required = 1)
     public abstract static class GetJavaPropertyNode extends CoreMethodArrayArgumentsNode {
 

src/main/java/org/truffleruby/language/RubyContextNode.java

@@ -9,6 +9,7 @@
  */
 package org.truffleruby.language;
 
+import com.oracle.truffle.api.TruffleLanguage.ContextReference;
 import org.truffleruby.RubyContext;
 import org.truffleruby.RubyLanguage;
 
@@ -18,18 +19,15 @@
 /** Has context but nothing else. */
 public abstract class RubyContextNode extends RubyBaseNode implements RubyNode.WithContext {
 
-    @CompilationFinal private RubyContext context;
-
-    // Accessors
+    @CompilationFinal private ContextReference<RubyContext> contextReference;
 
     @Override
     public RubyContext getContext() {
-        if (context == null) {
+        if (contextReference == null) {
             CompilerDirectives.transferToInterpreterAndInvalidate();
-            context = RubyLanguage.getCurrentContext();
+            contextReference = lookupContextReference(RubyLanguage.class);
         }
 
-        return context;
+        return contextReference.get();
     }
-
 }

src/main/java/org/truffleruby/language/loader/FeatureLoader.java

@@ -170,6 +170,7 @@ public void setWorkingDirectory(String cwd) {
         this.cwd = cwd;
     }
 
+    @TruffleBoundary
     public String getWorkingDirectory() {
         if (cwd != null) {
             return cwd;