Protect against occupancy calculations with very large numbers. (#2666)

Author: maleadt

lib/cudadrv/occupancy.jl

@@ -57,6 +57,7 @@ function launch_configuration(fun::CuFunction; shmem::Union{Integer,Base.Callabl
                               max_threads::Integer=0)
     blocks_ref = Ref{Cint}()
     threads_ref = Ref{Cint}()
+    max_threads = clamp(max_threads, Cint)
     if isa(shmem, Integer)
         cuOccupancyMaxPotentialBlockSize(blocks_ref, threads_ref, fun, C_NULL, shmem, max_threads)
     elseif Sys.ARCH == :x86 || Sys.ARCH == :x86_64

test/core/cudadrv.jl

@@ -819,6 +819,7 @@ let md = CuModuleFile(joinpath(@__DIR__, "ptx/dummy.ptx"))
     launch_configuration(dummy)
     launch_configuration(dummy; shmem=64)
     launch_configuration(dummy; shmem=64, max_threads=64)
+    launch_configuration(dummy; shmem=64, max_threads=typemax(Cint)+1)
 
     let cb_calls = 0
         launch_configuration(dummy; shmem=threads->(cb_calls += 1; 0))