Skip to content

Commit 09347ea

Browse files
ToucheSirToucheSir
authored
Merge pull request #471 from FluxML/nnlibcuda-merge
Merge NNlibCUDA as subpackage
2 parents 1b24915 + 4aa890f commit 09347ea

40 files changed

+2532
-0
lines changed

‎ext/NNlibCUDA/.buildkite/pipeline.yml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
steps:
2+
- label: "GPU integration with julia v1.6"
3+
plugins:
4+
- JuliaCI/julia#v1:
5+
version: "1.6"
6+
- JuliaCI/julia-test#v1: ~
7+
agents:
8+
queue: "juliagpu"
9+
cuda: "*"
10+
timeout_in_minutes: 60
11+
12+
- label: "GPU integration with julia v1"
13+
plugins:
14+
- JuliaCI/julia#v1:
15+
version: "1.8"
16+
- JuliaCI/julia-test#v1: ~
17+
agents:
18+
queue: "juliagpu"
19+
cuda: "*"
20+
timeout_in_minutes: 60

‎ext/NNlibCUDA/.github/workflows/compathelper.yml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: CompatHelper
2+
3+
on:
4+
schedule:
5+
- cron: '00 * * * *'
6+
issues:
7+
types: [opened, reopened]
8+
9+
jobs:
10+
build:
11+
runs-on: ${{ matrix.os }}
12+
strategy:
13+
matrix:
14+
julia-version: [1.2.0]
15+
julia-arch: [x86]
16+
os: [ubuntu-latest]
17+
steps:
18+
- uses: julia-actions/setup-julia@latest
19+
with:
20+
version: ${{ matrix.julia-version }}
21+
- name: Pkg.add("CompatHelper")
22+
run: julia -e 'using Pkg; Pkg.add("CompatHelper")'
23+
- name: CompatHelper.main()
24+
env:
25+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
26+
run: julia -e 'using CompatHelper; CompatHelper.main()'

‎ext/NNlibCUDA/.github/workflows/tagbot.yml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
name: TagBot
2+
on:
3+
issue_comment:
4+
types:
5+
- created
6+
workflow_dispatch:
7+
jobs:
8+
TagBot:
9+
if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot'
10+
runs-on: ubuntu-latest
11+
steps:
12+
- uses: JuliaRegistries/TagBot@v1
13+
with:
14+
token: ${{ secrets.GITHUB_TOKEN }}
15+
ssh: ${{ secrets.DOCUMENTER_KEY }}

‎ext/NNlibCUDA/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Manifest.toml

‎ext/NNlibCUDA/LICENSE.md

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
The NNlibCUDA.jl package is licensed under the MIT "Expat" License:
2+
3+
> Copyright (c) 2021-22: Julia Computing, INc., Dhairya Gandhi and Contributors
4+
>
5+
> Permission is hereby granted, free of charge, to any person obtaining
6+
> a copy of this software and associated documentation files (the
7+
> "Software"), to deal in the Software without restriction, including
8+
> without limitation the rights to use, copy, modify, merge, publish,
9+
> distribute, sublicense, and/or sell copies of the Software, and to
10+
> permit persons to whom the Software is furnished to do so, subject to
11+
> the following conditions:
12+
>
13+
> The above copyright notice and this permission notice shall be
14+
> included in all copies or substantial portions of the Software.
15+
>
16+
> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17+
> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18+
> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
19+
> IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
20+
> CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
21+
> TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
22+
> SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23+

‎ext/NNlibCUDA/Project.toml

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
name = "NNlibCUDA"
2+
uuid = "a00861dc-f156-4864-bf3c-e6376f28a68d"
3+
version = "0.2.7"
4+
5+
[deps]
6+
Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
7+
CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
8+
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
9+
NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
10+
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
11+
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
12+
cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd"
13+
14+
[compat]
15+
Adapt = "3.3"
16+
cuDNN = "1"
17+
CUDA = "4"
18+
NNlib = "0.8.15"
19+
julia = "1.6"
20+
21+
[extras]
22+
CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
23+
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
24+
ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
25+
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
26+
Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
27+
28+
[targets]
29+
test = ["CUDA", "ChainRulesCore", "ForwardDiff", "Test", "Zygote"]

‎ext/NNlibCUDA/README.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# NNlibCUDA.jl
2+
3+
This is a glue package which extends functions from [NNlib.jl](https://github.com/FluxML/NNlib.jl) to work with [CUDA.jl](https://github.com/JuliaGPU/CUDA.jl). It should be loaded automatically when using [Flux.jl](https://github.com/FluxML/Flux.jl), but not when using NNlib.jl by itself.
4+
5+
Julia gpu kernels are in `src/`, while wrappers around `cudnn` are in `src/cudnn/`.

‎ext/NNlibCUDA/src/NNlibCUDA.jl

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
module NNlibCUDA
2+
3+
using NNlib
4+
using CUDA, cuDNN
5+
using Random, Statistics
6+
7+
const IntOrIntTuple = Union{Integer, NTuple{N,<:Integer} where N}
8+
9+
include("upsample.jl")
10+
include("sampling.jl")
11+
include("activations.jl")
12+
include("batchedadjtrans.jl")
13+
include("batchedmul.jl")
14+
include("ctc.jl")
15+
include("fold.jl")
16+
include("scatter.jl")
17+
include("gather.jl")
18+
include("utils.jl")
19+
include("cudnn/cudnn.jl")
20+
include("cudnn/conv.jl")
21+
include("cudnn/pooling.jl")
22+
include("cudnn/softmax.jl")
23+
include("cudnn/activations.jl")
24+
include("cudnn/batchnorm.jl")
25+
26+
end # module

‎ext/NNlibCUDA/src/activations.jl

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Activation functions
2+
3+
# Some of activation functions need a wrapper for GPU support
4+
# https://github.com/JuliaGPU/CuArrays.jl/issues/614
5+
6+
# @cufunc softplus(x::Real) = ifelse(x > 0, x + log1p(exp(-x)), log1p(exp(x)))
7+
8+
# @cufunc logσ(x::Real) = -softplus(-x)
9+
10+
# @cufunc function gelu(x::Real)
11+
# p = oftype(x / 1, π)
12+
# λ = oftype(x / 1, √(2 / p))
13+
# α = oftype(x / 1, 0.044715)
14+
# h = oftype(x / 1, 0.5)
15+
# h * x * (one(x) + tanh(λ * (x + α * x^3)))
16+
# end
17+
18+
# @cufunc lisht(x::Real) = x * tanh(x)
19+
20+
# @cufunc logcosh(x::Real) = x + softplus(-2x) - log(oftype(x, 2))
21+
22+
# @cufunc mish(x::Real) = x * tanh(softplus(x))
23+
24+
# @cufunc tanhshrink(x::Real) = x - tanh(x)

‎ext/NNlibCUDA/src/batchedadjtrans.jl

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
using NNlib: BatchedAdjoint, BatchedTranspose, BatchedAdjOrTrans
2+
using Adapt
3+
using Adapt: WrappedArray
4+
5+
const CuBatchedAdjoint{T} = BatchedAdjoint{T, <: CuArray{T}}
6+
const CuBatchedTranspose{T} = BatchedTranspose{T, <: CuArray{T}}
7+
const CuBatchedAdjOrTrans{T} = Union{CuBatchedAdjoint{T}, CuBatchedTranspose{T}}
8+
const WrappedCuBatchedAdjOrTrans{T, N} = WrappedArray{T, N, CuBatchedAdjOrTrans{T}, CuBatchedAdjOrTrans{T}}
9+
10+
11+
Base.print_array(io::IO, b::Union{CuBatchedAdjOrTrans, WrappedCuBatchedAdjOrTrans}) = Base.print_array(io, adapt(Array, b))
12+
Base._show_nonempty(io::IO, b::Union{CuBatchedAdjOrTrans, WrappedCuBatchedAdjOrTrans}, prefix::String) = Base._show_nonempty(io, adapt(Array, b), prefix)
13+
Base.show_vector(io::IO, b::Union{CuBatchedAdjOrTrans, WrappedCuBatchedAdjOrTrans}, opn, cls) = Base.show_vector(io, adapt(Array, b), opn, cls)
14+
15+
Base.convert(::Type{T}, b::Union{CuBatchedAdjOrTrans, WrappedCuBatchedAdjOrTrans}) where {T<:Array} = Base.convert(T, adapt(Array, b))
16+
Base.Array{T, N}(b::Union{CuBatchedAdjOrTrans, WrappedCuBatchedAdjOrTrans}) where {T, N} = Array{T, N}(adapt(Array, b))
17+
Base.collect(b::Union{CuBatchedAdjOrTrans, WrappedCuBatchedAdjOrTrans}) = collect(adapt(Array, b))

0 commit comments

Comments
 (0)