Merge pull request #23557 from JuliaLang/rf/IntSet-symdiff-overflow

fix StackOverflowError in symdiff(::IntSet, ::NotInteger)

Author: JeffBezanson

base/intset.jl

@@ -155,13 +155,15 @@ symdiff(s::IntSet, ns) = symdiff!(copy(s), ns)
 
 For each element in `itr`, destructively toggle its inclusion in set `s`.
 """
-symdiff!(s::IntSet, ns) = (for n in ns; symdiff!(s, n); end; s)
+symdiff!(s::IntSet, ns) = (for n in ns; int_symdiff!(s, n); end; s)
 """
     symdiff!(s, n)
 
 The set `s` is destructively modified to toggle the inclusion of integer `n`.
 """
-function symdiff!(s::IntSet, n::Integer)
+symdiff!(s::IntSet, n::Integer) = int_symdiff!(s, n)
+
+function int_symdiff!(s::IntSet, n::Integer)
     0 < n < typemax(Int) || _throw_intset_bounds_err()
     val = !(n in s)
     _setint!(s, n, val)

test/intset.jl

@@ -90,6 +90,10 @@ end
     # issue #23099 : these tests should not segfault
     @test_throws ArgumentError symdiff!(IntSet(rand(1:100, 30)), 0)
     @test_throws ArgumentError symdiff!(IntSet(rand(1:100, 30)), [0, 2, 4])
+
+    # issue #23557 :
+    @test_throws MethodError symdiff!(IntSet([1]), ['a']) # should no stack-overflow
+    @test_throws MethodError symdiff!(IntSet([1, 2]),  [[1]]) # should not return IntSet([2])
 end
 
 @testset "copy, copy!, similar" begin