Fix String#lines for multibyte characters

PullRequest: truffleruby/543

Author: chrisseaton

CHANGELOG.md

@@ -1,3 +1,10 @@
+# 1.0 RC 12
+
+Bug fixes:
+
+* Fixed a bug with `String#lines` and similar methods with multibyte
+  characters (#1543).
+
 # 1.0 RC 11
 
 New features:

spec/ruby/core/string/shared/each_line.rb

@@ -27,6 +27,19 @@
     c.should == ["hello\n", "\n", "\n", "world"]
   end
 
+  it "splits strings containing multibyte characters" do
+    s = <<~EOS
+      foo
+      🤡🤡🤡🤡🤡🤡🤡
+      bar
+      baz
+    EOS
+
+    b = []
+    s.send(@method) { |s| b << s }
+    b.should == ["foo\n", "🤡🤡🤡🤡🤡🤡🤡\n", "bar\n", "baz\n"]
+  end
+
   it "taints substrings that are passed to the block if self is tainted" do
     "one\ntwo\r\nthree".taint.send(@method) { |s| s.tainted?.should == true }
 

src/main/ruby/core/string.rb

@@ -934,7 +934,7 @@ def each_line(sep=$/, chomp: false)
       sep = "\n\n"
       data = bytes
 
-      while pos < size
+      while pos < bytesize
         nxt = Truffle.invoke_primitive(:find_string, self, sep, pos)
         break unless nxt
 
@@ -964,10 +964,10 @@ def each_line(sep=$/, chomp: false)
     else
 
       # This is the normal case.
-      pat_size = sep.size
+      pat_size = sep.bytesize
       unmodified_self = clone
 
-      while pos < size
+      while pos < bytesize
         nxt = Truffle.invoke_primitive(:find_string, unmodified_self, sep, pos)
         break unless nxt