Avoid exponential of positive numbers in softplus implementation

Rodrigo Vargas · Rodrigo Vargas · commit f4d3243eadd9 · 2019-11-26T22:42:17.000+01:00
diff --git a/src/nnlib.jl b/src/nnlib.jl
@@ -28,4 +28,4 @@ end
   λ * ifelse(x > 0, x/1, α * (exp(x) - 1))
 end
 
-@cufunc softplus(x) = log1p(exp(x))
+@cufunc softplus(x) = ifelse(x > 0, x + log1p(exp(-x)), log1p(exp(x)))
