Fix Float#<=> when comparing Infinity to other infinite values

Author: bjfish

CHANGELOG.md

@@ -35,6 +35,7 @@ Compatibility:
 * Implement `IO#set_encoding_by_bom` (#2372, pawandubey).
 * Implemented `Enumerator::Lazy#with_index` (#2356).
 * Implement `rb_backref_set`.
+* Fix `Float#<=>` when comparing `Infinity` to other `#infinite?` values.
 
 Performance:
 

spec/ruby/core/float/comparison_spec.rb

@@ -63,4 +63,28 @@ def coerce(other)
   it "returns 1 when self is negative and other is -Infinity" do
     (-Float::MAX.to_i*2 <=> -infinity_value).should == 1
   end
+
+  it "returns 0 when self is Infinity and other other is infinite?=1" do
+    obj = Object.new
+    def obj.infinite?
+      1
+    end
+    (infinity_value <=> obj).should == 0
+  end
+
+  it "returns 1 when self is Infinity and other is infinite?=-1" do
+    obj = Object.new
+    def obj.infinite?
+      -1
+    end
+    (infinity_value <=> obj).should == 1
+  end
+
+  it "returns 1 when self is Infinity and other is infinite?=nil (which means finite)" do
+    obj = Object.new
+    def obj.infinite?
+        nil
+    end
+    (infinity_value <=> obj).should == 1
+  end
 end

src/main/ruby/truffleruby/core/numeric.rb

@@ -236,7 +236,17 @@ def redo_compare(meth, right)
     a <=> b
   end
 
+  # only used by Float#<=>
   private def redo_compare_bad_coerce_return_error(right)
+    if self.infinite? and !Primitive.undefined?(val = Truffle::Type.check_funcall(right, :infinite?))
+      if val
+        cmp = val <=> 0
+        return self > 0.0 ? (cmp > 0 ? 0 : 1) : (cmp < 0 ? 0 : -1)
+      else
+        # right.infinite? returned false or nil, which means right is finite
+        return self > 0.0 ? 1 : -1
+      end
+    end
     b, a = math_coerce(right, :bad_coerce_return_error)
     return nil unless b
     a <=> b