Fix an issue where Regexp.union was improperly negotiating the result encoding.

As part of this fix we were able to remove the non-standard `Regexp.convert`.

Co-authored-by: Simon LeVasseur <simon.levasseur@shopify.com>

Author: nirvdrum

CHANGELOG.md

@@ -7,6 +7,7 @@ Bug fixes:
 
 * Fix `rb_enc_left_char_head()` so it is not always `ArgumentError` (#3267, @eregon).
 * Fix `IO.copy_stream` with a `Tempfile` destination (#3280, @eregon).
+* Fix `Regexp.union` negotiating the wrong result encoding (#3287, @nirvdrum, @simonlevasseur).
 
 Compatibility:
 

spec/ruby/core/regexp/union_spec.rb

@@ -43,6 +43,24 @@
     Regexp.union("\u00A9".encode("ISO-8859-1"), "a".encode("UTF-8")).encoding.should == Encoding::ISO_8859_1
   end
 
+  it "returns ASCII-8BIT if the regexp encodings are ASCII-8BIT and at least one has non-ASCII characters" do
+    us_ascii_implicit, us_ascii_explicit, binary = /abc/, /[\x00-\x7f]/n, /[\x80-\xBF]/n
+
+    Regexp.union(us_ascii_implicit, us_ascii_explicit, binary).encoding.should == Encoding::BINARY
+    Regexp.union(us_ascii_implicit, binary, us_ascii_explicit).encoding.should == Encoding::BINARY
+    Regexp.union(us_ascii_explicit, us_ascii_implicit, binary).encoding.should == Encoding::BINARY
+    Regexp.union(us_ascii_explicit, binary, us_ascii_implicit).encoding.should == Encoding::BINARY
+    Regexp.union(binary, us_ascii_implicit, us_ascii_explicit).encoding.should == Encoding::BINARY
+    Regexp.union(binary, us_ascii_explicit, us_ascii_implicit).encoding.should == Encoding::BINARY
+  end
+
+  it "return US-ASCII if all patterns are ASCII-only" do
+    Regexp.union(/abc/e, /def/e).encoding.should == Encoding::US_ASCII
+    Regexp.union(/abc/n, /def/n).encoding.should == Encoding::US_ASCII
+    Regexp.union(/abc/s, /def/s).encoding.should == Encoding::US_ASCII
+    Regexp.union(/abc/u, /def/u).encoding.should == Encoding::US_ASCII
+  end
+
   it "returns a Regexp with UTF-8 if one part is UTF-8" do
     Regexp.union(/probl[éeè]me/i, /help/i).encoding.should == Encoding::UTF_8
   end

src/main/ruby/truffleruby/core/regexp.rb

@@ -55,22 +55,23 @@ def self.try_convert(obj)
     Truffle::Type.try_convert obj, Regexp, :to_regexp
   end
 
-  def self.convert(pattern)
-    return pattern if Primitive.is_a?(pattern, Regexp)
-    if Primitive.is_a?(pattern, Array)
-      union(*pattern)
-    else
-      Regexp.quote(pattern.to_s)
-    end
-  end
+  def self.negotiate_union_encoding(*patterns)
+    res = nil
+
+    patterns.each do |pattern|
+      converted = Primitive.is_a?(pattern, Regexp) ? pattern : Regexp.quote(pattern)
 
-  def self.compatible?(*patterns)
-    encodings = patterns.map { |r| convert(r).encoding }
-    last_enc = encodings.pop
-    encodings.each do |encoding|
-      raise ArgumentError, "incompatible encodings: #{encoding} and #{last_enc}" unless Primitive.encoding_compatible?(last_enc, encoding)
-      last_enc = encoding
+      enc = converted.encoding
+
+      if Primitive.nil?(res)
+        res = enc
+      else
+        res = Primitive.encoding_compatible?(enc, res)
+        raise ArgumentError, "incompatible encodings: #{enc} and #{res}" unless res
+      end
     end
+
+    res
   end
 
   def self.last_match(index = nil)
@@ -96,37 +97,35 @@ def self.last_match(index = nil)
   def self.union(*patterns)
     case patterns.size
     when 0
-      return %r/(?!)/
+      %r/(?!)/
     when 1
       pattern = patterns.first
       case pattern
       when Array
-        return union(*pattern)
+        union(*pattern)
       else
         converted = Truffle::Type.rb_check_convert_type(pattern, Regexp, :to_regexp)
         if Primitive.nil? converted
-          return Regexp.new(Regexp.quote(pattern))
+          Regexp.new(Regexp.quote(pattern))
         else
-          return converted
+          converted
         end
       end
     else
-      compatible?(*patterns)
-      enc = convert(patterns.first).encoding
-    end
+      patterns = patterns.map do |pat|
+        if Primitive.is_a?(pat, Regexp)
+          pat
+        else
+          StringValue(pat)
+        end
+      end
 
-    sep = '|'.encode(enc)
-    str = ''.encode(enc)
+      enc = negotiate_union_encoding(*patterns)
+      sep = '|'.encode(enc)
+      str = ''.encode(enc)
 
-    patterns = patterns.map do |pat|
-      if Primitive.is_a?(pat, Regexp)
-        pat
-      else
-        StringValue(pat)
-      end
+      Truffle::RegexpOperations.union(str, sep, *patterns)
     end
-
-    Truffle::RegexpOperations.union(str, sep, *patterns)
   end
   Truffle::Graal.always_split(method(:union))