Merge #518

518: Avoid exponential of positive numbers in softplus implementation r=maleadt a=vargonis

Just use the standard trick to avoid getting `Inf` when broadcasting softplus to GPU arrays containing large positive numbers.

Co-authored-by: Rodrigo Vargas <rodrigo@odd.co>

Author: bors[bot]

src/nnlib.jl

@@ -28,4 +28,4 @@ end
   λ * ifelse(x > 0, x/1, α * (exp(x) - 1))
 end
 
-@cufunc softplus(x) = log1p(exp(x))
+@cufunc softplus(x) = ifelse(x > 0, x + log1p(exp(-x)), log1p(exp(x)))

test/dnn.jl

@@ -78,6 +78,16 @@ end
   @test testf(CuArrays.CUDNN.cudnnAddTensor, cu(rand(Float64, 10, 10, 3, 1)), cu(rand(Float64, 10, 10, 3, 1)))
   @test testf(CuArrays.CUDNN.cudnnActivationForward, cu(rand(Float64, 10, 10, 3, 1)), cu(rand(Float64, 10, 10, 3, 1)))
   @test testf(CuArrays.CUDNN.cudnnActivationBackward, cu(rand(Float64, 10, 10, 3, 1)), cu(rand(Float64, 10, 10, 3, 1)), cu(rand(Float64, 10, 10, 3, 1)), cu(rand(Float64, 10, 10, 3, 1)))
+
+  # activations defined in src/nnlib.jl
+  for dims in ((5,5), (5,))
+    for f in (σ, logσ, elu, swish, gelu, selu, softplus)
+      @test testf(x -> f.(x), rand(Float64, dims))
+    end
+  end
+  # softplus does not give `Inf` for large arguments
+  x = cu([1000.])
+  @test all(softplus.(x) .== x)
 end
 
 @testset "Batchnorm" begin