[GR-14880] Fix and specs for return value of rb_protect when an error occurs.

PullRequest: truffleruby/763

Author: aardvark179

CHANGELOG.md

@@ -6,6 +6,7 @@ Bug fixes:
 * Fixed `String#inspect` when the string uses a non-UTF-8 ASCII-compatible
   encoding and has non-ASCII characters.
 * Fixed `puts` for strings with non-ASCII-compatible encodings.
+* `rb_protect` now returns `Qnil` when an error occurs.
 
 New features:
 

lib/truffle/truffle/cext.rb

@@ -1084,7 +1084,11 @@ def rb_proc_call(prc, args)
   # throw it and allow normal error handling to continue.
 
   def rb_protect_with_block(function, arg)
-    res = nil
+    # We wrap nil here to avoid wrapping any result returned, as the
+    # function called will do that. In general we try not to touch the
+    # values passed in or out of protected functions as C extensions
+    # may accept or return arbitrary pointers rather than ruby VALUEs.
+    res = rb_tr_wrap(nil)
     pos = 0
     e = capture_exception do
       res = Truffle::Interop.execute_without_conversion(function, arg)

spec/ruby/optional/capi/ext/kernel_spec.c

@@ -168,6 +168,7 @@ static VALUE kernel_spec_rb_protect_yield(VALUE self, VALUE obj, VALUE ary) {
   int status = 0;
   VALUE res = rb_protect(rb_yield, obj, &status);
   rb_ary_store(ary, 0, INT2NUM(23));
+  rb_ary_store(ary, 1, res);
   if (status) {
     rb_jump_tag(status);
   }

spec/ruby/optional/capi/kernel_spec.rb

@@ -281,6 +281,15 @@
       end.should raise_error(NameError)
       proof[0].should == 23
     end
+
+    it "will return nil if an error was raised" do
+      proof = [] # Hold proof of work performed after the yield.
+      lambda do
+        @s.rb_protect_yield(7, proof) { |x| raise NameError}
+      end.should raise_error(NameError)
+      proof[0].should == 23
+      proof[1].should == nil
+    end
   end
 
   describe "rb_rescue" do