From 463c80de21e2f5a019a94bedb55dd78dcd689a04 Mon Sep 17 00:00:00 2001 From: Kai Xu Date: Sun, 27 Oct 2019 00:26:22 +0100 Subject: [PATCH 1/4] add deterministic option --- src/dnn/CUDNN.jl | 2 ++ src/dnn/nnlib.jl | 36 +++++++++++++++++++++++++++++------- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/dnn/CUDNN.jl b/src/dnn/CUDNN.jl index f8167ff8..d4b241f9 100644 --- a/src/dnn/CUDNN.jl +++ b/src/dnn/CUDNN.jl @@ -36,6 +36,8 @@ function handle() return _handle[] end +isdeterministic() = true + include("base.jl") include("libcudnn.jl") diff --git a/src/dnn/nnlib.jl b/src/dnn/nnlib.jl index 789935d5..59b3e2ee 100644 --- a/src/dnn/nnlib.jl +++ b/src/dnn/nnlib.jl @@ -53,8 +53,14 @@ function conv!(y::CuArray{T}, x::CuArray{T}, w::CuArray{T}, cdims::DenseConvDims end end +# Reference for deterministic algo: +# https://docs.nvidia.com/deeplearning/sdk/cudnn-developer-guide/index.html#cudnnConvolutionBackwardFilter function ∇conv_filter!(dw::CuArray{T}, x::CuArray{T}, dy::CuArray{T}, - cdims::DenseConvDims; alpha=1, algo=0) where T<:CUDNNFloat + cdims::DenseConvDims; alpha=1, algo=isdeterministic() ? 1 : 0) where T<:CUDNNFloat + if isdeterministic() + @assert algo != 0 "algorithm 0 (CUDNN_CONVOLUTION_BWD_FILTER_ALGO_0) is not deterministic" + end + if version() < v"6" all(x -> x == 1, dilation(cdims)) || error("Only dilation = 1 is supported in cuDNN version < 6") end @@ -67,8 +73,14 @@ function ∇conv_filter!(dw::CuArray{T}, x::CuArray{T}, dy::CuArray{T}, end end +# Reference for deterministic algo: +# https://docs.nvidia.com/deeplearning/sdk/cudnn-developer-guide/index.html#cudnnConvolutionBackwardData function ∇conv_data!(dx::CuArray{T}, dy::CuArray{T}, w::CuArray{T}, - cdims::DenseConvDims; alpha=1, algo=0) where T<:CUDNNFloat + cdims::DenseConvDims; alpha=1, algo=isdeterministic() ? 1 : 0) where T<:CUDNNFloat + if isdeterministic() + @assert algo != 0 "algorithm 0 (CUDNN_CONVOLUTION_BWD_DATA_ALGO_0) is not deterministic" + end + if version() < v"6" all(x -> x == 1, dilation(cdims)) || error("Only dilation = 1 is supported in cuDNN version < 6") end @@ -84,12 +96,22 @@ end ∇conv_bias!(db::CuArray{T}, dy::CuArray{T}; alpha=1, beta=0) where T<:CUDNNFloat = cudnnConvolutionBackwardBias(db, dy, alpha=alpha, beta=beta) -maxpool!(y::CuArray{T}, x::CuArray{T}, pdims::PoolDims) where T<:CUDNNFloat = - cudnnPoolingForward(y, x, pdims; mode=0) +# Reference for deterministic mode: +# https://docs.nvidia.com/deeplearning/sdk/cudnn-developer-guide/index.html#cudnnPoolingMode_t +function maxpool!(y::CuArray{T}, x::CuArray{T}, pdims::PoolDims; mode=isdeterministic() ? 3 : 0) where T<:CUDNNFloat + if isdeterministic() + @assert mode == 3 "only mode 3 (CUDNN_POOLING_MAX_DETERMINISTIC) is deterministic" + end + return cudnnPoolingForward(y, x, pdims; mode=mode) +end -∇maxpool!(dx::CuArray{T}, dy::CuArray{T}, y::CuArray{T}, x::CuArray{T}, - pdims::PoolDims) where T<:CUDNNFloat = - cudnnPoolingBackward(dx, dy, x, y, pdims, mode=0) +function ∇maxpool!(dx::CuArray{T}, dy::CuArray{T}, y::CuArray{T}, x::CuArray{T}, + pdims::PoolDims, mode=isdeterministic() ? 3 : 0) where T<:CUDNNFloat + if isdeterministic() + @assert mode == 3 "only mode 3 (CUDNN_POOLING_MAX_DETERMINISTIC) is deterministic" + end + return cudnnPoolingBackward(dx, dy, x, y, pdims, mode=mode) +end meanpool!(y::CuArray{T}, x::CuArray{T}, pdims::PoolDims) where T<:CUDNNFloat = cudnnPoolingForward(y, x, pdims, mode=1) From 064124a44c61300107c79fc1d4ba41c563238115 Mon Sep 17 00:00:00 2001 From: Kai Xu Date: Tue, 7 Jan 2020 20:52:59 +0000 Subject: [PATCH 2/4] use global variable --- src/dnn/CUDNN.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/dnn/CUDNN.jl b/src/dnn/CUDNN.jl index 709a739f..1d83f70f 100644 --- a/src/dnn/CUDNN.jl +++ b/src/dnn/CUDNN.jl @@ -16,7 +16,10 @@ import ..CuArrays.unsafe_free! import NNlib -isdeterministic() = true +const deterministic = Ref(true) + +isdeterministic() = deterministic[] +deterministic!(flag) = (deterministic[] = flag) const libcudnn = if Sys.iswindows() # no ccall by soname, we need the filename From ea0d5febb1e479afed020d341b5cf6626520c7a1 Mon Sep 17 00:00:00 2001 From: Kai Xu Date: Tue, 7 Jan 2020 20:57:44 +0000 Subject: [PATCH 3/4] warn instead of assert --- src/dnn/nnlib.jl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/dnn/nnlib.jl b/src/dnn/nnlib.jl index e05b2bd9..82403bfc 100644 --- a/src/dnn/nnlib.jl +++ b/src/dnn/nnlib.jl @@ -60,7 +60,10 @@ end function ∇conv_filter!(dw::CuArray{T}, x::CuArray{T}, dy::CuArray{T}, cdims::DenseConvDims; alpha=1, algo=isdeterministic() ? 1 : 0) where T<:CUDNNFloat if isdeterministic() - @assert algo != 0 "algorithm 0 (CUDNN_CONVOLUTION_BWD_FILTER_ALGO_0) is not deterministic" + if algo == 0 + @warn "algorithm 0 (CUDNN_CONVOLUTION_BWD_FILTER_ALGO_0) is not deterministic; algorithm 1 is used instead" + algo = 1 + end end if version() < v"6" @@ -80,7 +83,9 @@ end function ∇conv_data!(dx::CuArray{T}, dy::CuArray{T}, w::CuArray{T}, cdims::DenseConvDims; alpha=1, algo=isdeterministic() ? 1 : 0) where T<:CUDNNFloat if isdeterministic() - @assert algo != 0 "algorithm 0 (CUDNN_CONVOLUTION_BWD_DATA_ALGO_0) is not deterministic" + if algo == 0 + @warn "algorithm 0 (CUDNN_CONVOLUTION_BWD_DATA_ALGO_0) is not deterministic; algorithm 1 is used instead" + end end if version() < v"6" @@ -102,7 +107,9 @@ end # https://docs.nvidia.com/deeplearning/sdk/cudnn-developer-guide/index.html#cudnnPoolingMode_t function maxpool!(y::CuArray{T}, x::CuArray{T}, pdims::PoolDims; mode=isdeterministic() ? 3 : 0) where T<:CUDNNFloat if isdeterministic() - @assert mode == 3 "only mode 3 (CUDNN_POOLING_MAX_DETERMINISTIC) is deterministic" + if model != 3 + @warn "mode $mode is not deterministic; mode 3 (CUDNN_POOLING_MAX_DETERMINISTIC) is used instead" + end end return cudnnPoolingForward(y, x, pdims; mode=mode) end @@ -110,7 +117,9 @@ end function ∇maxpool!(dx::CuArray{T}, dy::CuArray{T}, y::CuArray{T}, x::CuArray{T}, pdims::PoolDims, mode=isdeterministic() ? 3 : 0) where T<:CUDNNFloat if isdeterministic() - @assert mode == 3 "only mode 3 (CUDNN_POOLING_MAX_DETERMINISTIC) is deterministic" + if model != 3 + @warn "mode $mode is not deterministic; mode 3 (CUDNN_POOLING_MAX_DETERMINISTIC) is used instead" + end end return cudnnPoolingBackward(dx, dy, x, y, pdims, mode=mode) end From 3f16d7e2b02088f096dbb04bf4f9ee6d5e1b3699 Mon Sep 17 00:00:00 2001 From: Kai Xu Date: Fri, 10 Jan 2020 14:52:13 +0000 Subject: [PATCH 4/4] Update src/dnn/CUDNN.jl Co-Authored-By: Mike J Innes --- src/dnn/CUDNN.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnn/CUDNN.jl b/src/dnn/CUDNN.jl index 0f56aeb3..70fade6d 100644 --- a/src/dnn/CUDNN.jl +++ b/src/dnn/CUDNN.jl @@ -21,7 +21,7 @@ const libcudnn = Ref("libcudnn") # control of deterministic const deterministic = Ref(true) isdeterministic() = deterministic[] -deterministic!(flag) = (deterministic[] = flag) +deterministic!(flag = true) = (deterministic[] = flag) # core library include("libcudnn_common.jl")