|
| 1 | +using Test |
| 2 | +using Flux |
| 3 | + |
| 4 | +using Enzyme |
| 5 | +using Functors |
| 6 | +using FiniteDifferences |
| 7 | +using CUDA |
| 8 | + |
| 9 | +Enzyme.API.typeWarning!(false) # suppresses a warning with Bilinear https://github.com/EnzymeAD/Enzyme.jl/issues/1341 |
| 10 | +Enzyme.API.runtimeActivity!(true) # for Enzyme debugging |
| 11 | +# Enzyme.Compiler.bitcode_replacement!(false) |
| 12 | + |
| 13 | +_make_zero(x::Union{Number,AbstractArray}) = zero(x) |
| 14 | +_make_zero(x) = x |
| 15 | +make_zero(model) = fmap(_make_zero, model) |
| 16 | +## make_differential(model) = fmapstructure(make_zero, model) # NOT SUPPORTED, See https://github.com/EnzymeAD/Enzyme.jl/issues/1329 |
| 17 | + |
| 18 | +function gradient_fd(f, x...) |
| 19 | + x = [cpu(x) for x in x] |
| 20 | + ps_and_res = [x isa AbstractArray ? (x, identity) : Flux.destructure(x) for x in x] |
| 21 | + ps = [f64(x[1]) for x in ps_and_res] |
| 22 | + res = [x[2] for x in ps_and_res] |
| 23 | + fdm = FiniteDifferences.central_fdm(5, 1) |
| 24 | + gs = FiniteDifferences.grad(fdm, (ps...) -> f((re(p) for (p,re) in zip(ps, res))...), ps...) |
| 25 | + return ((re(g) for (re, g) in zip(res, gs))...,) |
| 26 | +end |
| 27 | + |
| 28 | +function gradient_ez(f, x...) |
| 29 | + args = [] |
| 30 | + for x in x |
| 31 | + if x isa Number |
| 32 | + push!(args, Active(x)) |
| 33 | + else |
| 34 | + push!(args, Duplicated(x, make_zero(x))) |
| 35 | + end |
| 36 | + end |
| 37 | + ret = Enzyme.autodiff(ReverseWithPrimal, f, Active, args...) |
| 38 | + g = ntuple(i -> x[i] isa Number ? ret[1][i] : args[i].dval, length(x)) |
| 39 | + return g |
| 40 | +end |
| 41 | + |
| 42 | +function test_grad(g1, g2; broken=false) |
| 43 | + fmap_with_path(g1, g2) do kp, x, y |
| 44 | + :state ∈ kp && return # ignore RNN and LSTM state |
| 45 | + if x isa AbstractArray{<:Number} |
| 46 | + # @show kp |
| 47 | + @test x ≈ y rtol=1e-2 atol=1e-6 broken=broken |
| 48 | + end |
| 49 | + return x |
| 50 | + end |
| 51 | +end |
| 52 | + |
| 53 | +function test_enzyme_grad(loss, model, x) |
| 54 | + Flux.trainmode!(model) |
| 55 | + l = loss(model, x) |
| 56 | + @test loss(model, x) == l # Check loss doesn't change with multiple runs |
| 57 | + |
| 58 | + grads_fd = gradient_fd(loss, model, x) |> cpu |
| 59 | + grads_flux = Flux.gradient(loss, model, x) |> cpu |
| 60 | + grads_enzyme = gradient_ez(loss, model, x) |> cpu |
| 61 | + |
| 62 | + # test_grad(grads_flux, grads_enzyme) |
| 63 | + test_grad(grads_fd, grads_enzyme) |
| 64 | +end |
| 65 | + |
| 66 | +@testset "gradient_ez" begin |
| 67 | + @testset "number and arrays" begin |
| 68 | + f(x, y) = sum(x.^2) + y^3 |
| 69 | + x = Float32[1, 2, 3] |
| 70 | + y = 3f0 |
| 71 | + g = gradient_ez(f, x, y) |
| 72 | + @test g[1] isa Array{Float32} |
| 73 | + @test g[2] isa Float32 |
| 74 | + @test g[1] ≈ 2x |
| 75 | + @test g[2] ≈ 3*y^2 |
| 76 | + end |
| 77 | + |
| 78 | + @testset "struct" begin |
| 79 | + struct SimpleDense{W, B, F} |
| 80 | + weight::W |
| 81 | + bias::B |
| 82 | + σ::F |
| 83 | + end |
| 84 | + SimpleDense(in::Integer, out::Integer; σ=identity) = SimpleDense(randn(Float32, out, in), zeros(Float32, out), σ) |
| 85 | + (m::SimpleDense)(x) = m.σ.(m.weight * x .+ m.bias) |
| 86 | + @functor SimpleDense |
| 87 | + |
| 88 | + model = SimpleDense(2, 4) |
| 89 | + x = randn(Float32, 2) |
| 90 | + loss(model, x) = sum(model(x)) |
| 91 | + |
| 92 | + g = gradient_ez(loss, model, x) |
| 93 | + @test g[1] isa SimpleDense |
| 94 | + @test g[2] isa Array{Float32} |
| 95 | + @test g[1].weight isa Array{Float32} |
| 96 | + @test g[1].bias isa Array{Float32} |
| 97 | + @test g[1].weight ≈ ones(Float32, 4, 1) .* x' |
| 98 | + @test g[1].bias ≈ ones(Float32, 4) |
| 99 | + end |
| 100 | +end |
| 101 | + |
| 102 | +@testset "Models" begin |
| 103 | + function loss(model, x) |
| 104 | + Flux.reset!(model) |
| 105 | + sum(model(x)) |
| 106 | + end |
| 107 | + |
| 108 | + models_xs = [ |
| 109 | + (Dense(2, 4), randn(Float32, 2), "Dense"), |
| 110 | + (Chain(Dense(2, 4, relu), Dense(4, 3)), randn(Float32, 2), "Chain(Dense, Dense)"), |
| 111 | + (f64(Chain(Dense(2, 4), Dense(4, 2))), randn(Float64, 2, 1), "f64(Chain(Dense, Dense))"), |
| 112 | + (Flux.Scale([1.0f0 2.0f0 3.0f0 4.0f0], true, abs2), randn(Float32, 2), "Flux.Scale"), |
| 113 | + (Conv((3, 3), 2 => 3), randn(Float32, 3, 3, 2, 1), "Conv"), |
| 114 | + (Chain(Conv((3, 3), 2 => 3, relu), Conv((3, 3), 3 => 1, relu)), rand(Float32, 5, 5, 2, 1), "Chain(Conv, Conv)"), |
| 115 | + (Chain(Conv((4, 4), 2 => 2, pad=SamePad()), MeanPool((5, 5), pad=SamePad())), rand(Float32, 5, 5, 2, 2), "Chain(Conv, MeanPool)"), |
| 116 | + (Maxout(() -> Dense(5 => 4, tanh), 3), randn(Float32, 5, 1), "Maxout"), |
| 117 | + (RNN(3 => 2), randn(Float32, 3, 2), "RNN"), |
| 118 | + (Chain(RNN(3 => 4), RNN(4 => 3)), randn(Float32, 3, 2), "Chain(RNN, RNN)"), |
| 119 | + (LSTM(3 => 5), randn(Float32, 3, 2), "LSTM"), |
| 120 | + (Chain(LSTM(3 => 5), LSTM(5 => 3)), randn(Float32, 3, 2), "Chain(LSTM, LSTM)"), |
| 121 | + (SkipConnection(Dense(2 => 2), vcat), randn(Float32, 2, 3), "SkipConnection"), |
| 122 | + (Flux.Bilinear((2, 2) => 3), randn(Float32, 2, 1), "Bilinear"), |
| 123 | + ] |
| 124 | + |
| 125 | + for (model, x, name) in models_xs |
| 126 | + @testset "check grad $name" begin |
| 127 | + println("testing $name") |
| 128 | + test_enzyme_grad(loss, model, x) |
| 129 | + end |
| 130 | + end |
| 131 | +end |
| 132 | + |
| 133 | +@testset "Recurrence Tests" begin |
| 134 | + function loss(model, x) |
| 135 | + Flux.reset!(model) |
| 136 | + for i in 1:3 |
| 137 | + x = model(x) |
| 138 | + end |
| 139 | + return sum(x) |
| 140 | + end |
| 141 | + |
| 142 | + models_xs = [ |
| 143 | + (RNN(3 => 3), randn(Float32, 3, 2), "RNN"), |
| 144 | + (LSTM(3 => 3), randn(Float32, 3, 2), "LSTM"), |
| 145 | + # TESTS BELOW ARE BROKEN FOR ZYGOTE BUT CORRECT FOR ENZYME! |
| 146 | + (Chain(RNN(3 => 5), RNN(5 => 3)), randn(Float32, 3, 2), "Chain(RNN, RNN)"), |
| 147 | + (Chain(LSTM(3 => 5), LSTM(5 => 3)), randn(Float32, 3, 2), "Chain(LSTM, LSTM)"), |
| 148 | + ] |
| 149 | + |
| 150 | + for (model, x, name) in models_xs |
| 151 | + @testset "check grad $name" begin |
| 152 | + println("testing $name") |
| 153 | + test_enzyme_grad(loss, model, x) |
| 154 | + end |
| 155 | + end |
| 156 | +end |
| 157 | + |
| 158 | +@testset "Broken Models" begin |
| 159 | + function loss(model, x) |
| 160 | + Flux.reset!(model) |
| 161 | + sum(model(x)) |
| 162 | + end |
| 163 | + |
| 164 | + device = Flux.get_device() |
| 165 | + |
| 166 | + models_xs = [ |
| 167 | + (GRU(3 => 5), randn(Float32, 3, 10), "GRU"), |
| 168 | + (ConvTranspose((3, 3), 3 => 2, stride=2), rand(Float32, 5, 5, 3, 1), "ConvTranspose"), |
| 169 | + ] |
| 170 | + |
| 171 | + for (model, x, name) in models_xs |
| 172 | + @testset "check grad $name" begin |
| 173 | + println("testing $name") |
| 174 | + broken = false |
| 175 | + try |
| 176 | + test_enzyme_grad(loss, model, x) |
| 177 | + catch e |
| 178 | + println(e) |
| 179 | + broken = true |
| 180 | + end |
| 181 | + @test broken |
| 182 | + end |
| 183 | + end |
| 184 | +end |
| 185 | + |
0 commit comments