Merge branch 'main' into remove-reduce-code

Author: Benedikt Brantner Admin

Project.toml

@@ -3,6 +3,9 @@ uuid = "aed23131-dcd0-47ca-8090-d21e605652e3"
 authors = ["Michael Kraus"]
 version = "0.2.0"
 
+[weakdeps]
+SymbolicUtils = "d1185830-fcd6-423d-90d6-eec64667417b"
+
 [deps]
 AbstractNeuralNetworks = "60874f82-5ada-4c70-bd1c-fa6be7711c8a"
 Latexify = "23fbe1c1-3f47-55db-b15f-69d7ec21a316"
@@ -14,14 +17,16 @@ Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7"
 SymbolicUtils = "d1185830-fcd6-423d-90d6-eec64667417b"
 
 [compat]
-AbstractNeuralNetworks = "0.3, 0.4, 0.5"
+AbstractNeuralNetworks = "0.3, 0.4, 0.5, 0.6"
 Documenter = "1.8.0"
 ForwardDiff = "0.10.38"
-GeometricMachineLearning = "0.3.7"
+GeometricMachineLearning = "0.4"
 Latexify = "0.16.5"
 RuntimeGeneratedFunctions = "0.5"
 SafeTestsets = "0.1"
 SymbolicUtils = "<3.8.0"
+SafeTestsets = "0.1"
+SymbolicUtils = "<3.8.0"
 Symbolics = "5, 6"
 Zygote = "0.6.73"
 julia = "1.10"

docs/src/double_derivative.md

@@ -18,7 +18,7 @@
 ```@example jacobian_gradient
 using AbstractNeuralNetworks
 using SymbolicNeuralNetworks
-using SymbolicNeuralNetworks: Jacobian, Gradient, derivative, params
+using SymbolicNeuralNetworks: Jacobian, Gradient, derivative
 using Latexify: latexify
 
 c = Chain(Dense(2, 1, tanh; use_bias = false))
@@ -59,7 +59,7 @@ We can easily differentiate a neural network twice by using [`SymbolicNeuralNetw
 ```@example jacobian_gradient
 using AbstractNeuralNetworks
 using SymbolicNeuralNetworks
-using SymbolicNeuralNetworks: Jacobian, Gradient, derivative
+using SymbolicNeuralNetworks: Jacobian, Gradient, derivative, params
 using Latexify: latexify
 
 c = Chain(Dense(2, 1, tanh))

docs/src/hamiltonian_neural_network.md

@@ -4,6 +4,7 @@ Here we build a Hamiltonian neural network as a symbolic neural network.
 
 ```julia hnn
 using SymbolicNeuralNetworks
+using SymbolicNeuralNetworks: params
 using GeometricMachineLearning
 using AbstractNeuralNetworks: Dense, UnknownArchitecture, Model
 using LinearAlgebra: norm
@@ -45,7 +46,7 @@ nothing # hide
 We can now train the network:
 
 ```julia hnn
-ps = NeuralNetwork(c, T).params
+ps = params(NeuralNetwork(c, T))
 dl = DataLoader(z_data, hvf_analytic(z_data))
 o = Optimizer(AdamOptimizer(.01), ps)
 batch = Batch(200)
@@ -59,7 +60,7 @@ We now integrate the vector field:
 
 ```julia hnn
 using GeometricIntegrators
-hvf_closure(input) = build_nn_function(hvf, x, nn)(input, nn_dummy.params)
+hvf_closure(input) = build_nn_function(hvf, x, nn)(input, params(nn_dummy))
 function v(v, t, q, params)
     v .= hvf_closure(q)
 end

scripts/pullback_comparison.jl

@@ -1,7 +1,7 @@
 using SymbolicNeuralNetworks
 using AbstractNeuralNetworks
 using GeometricMachineLearning
-using AbstractNeuralNetworks: FeedForwardLoss
+using AbstractNeuralNetworks: FeedForwardLoss, params
 using GeometricMachineLearning: ZygotePullback
 import Random
 Random.seed!(123)
@@ -25,9 +25,9 @@ _do = 1.
 # @time zpb_evaluated = zpb(params(nn_cpu), nn.model, (input, output))[2](_do)[1].params
 # @assert values(spb_evaluated) .≈ values(zpb_evaluated)
 
-function timenn(pb, params, model, input, output, _do = 1.)
-    pb(params, model, (input, output))[2](_do)
-    @time pb(params, model, (input, output))[2](_do)
+function timenn(pb, _params, model, input, output, _do = 1.)
+    pb(_params, model, (input, output))[2](_do)
+    @time pb(_params, model, (input, output))[2](_do)
 end
 
 timenn(spb, params(nn_cpu), nn.model, input, output)

src/build_function/build_function.jl

@@ -54,7 +54,7 @@ built_function([1. 2.; 3. 4.], params(nn), 1)
 # output
 
 1-element Vector{Float64}:
- -0.9999967113439513
+ 0.9912108161055604
 ```
 
 Note that we have to supply an extra argument (index) to `_build_nn_function` that we do not have to supply to [`build_nn_function`](@ref).

src/build_function/build_function_arrays.jl

@@ -22,8 +22,8 @@ funcs_evaluated = funcs(input, params(nn))
 # output
 
 2-element Vector{NamedTuple}:
- (a = [-0.9999386280616135], b = [0.9998772598897417])
- (c = [-0.9998158954841537],)
+ (a = [0.985678060655224], b = [0.9715612392570434])
+ (c = [0.9576465981186686],)
 ```
 """
 function build_nn_function(eqs::AbstractArray{<:Union{NamedTuple, NeuralNetworkParameters}}, sparams::NeuralNetworkParameters, sinput::Symbolics.Arr...)
@@ -58,7 +58,7 @@ funcs_evaluated = funcs(input, params(nn))
 
 # output
 
-(a = [-0.9999386280616135], b = [0.9998772598897417])
+(a = [0.985678060655224], b = [0.9715612392570434])
 ```
 
 # Implementation

src/custom_definitions_and_extensions/latexraw.jl

@@ -1,3 +1,7 @@
 function _latexraw(args::AbstractNeuralNetworks.GenericActivation; kwargs...)
     _latexraw(args.σ; kwargs...)
+end
+
+function _latexraw(args::AbstractNeuralNetworks.TanhActivation; kwargs...)
+    _latexraw(tanh; kwargs...)
 end

src/derivatives/gradient.jl

@@ -18,7 +18,6 @@ Compute the symbolic output of `nn` and differentiate it with respect to the par
 ```julia
 using SymbolicNeuralNetworks: SymbolicNeuralNetwork, Gradient, derivative
 using AbstractNeuralNetworks
-using Latexify: latexify
 
 c = Chain(Dense(2, 1, tanh))
 nn = SymbolicNeuralNetwork(c)
@@ -81,7 +80,6 @@ using SymbolicNeuralNetworks: SymbolicNeuralNetwork, symbolic_pullback
 using AbstractNeuralNetworks
 using AbstractNeuralNetworks: params
 using LinearAlgebra: norm
-using Latexify: latexify
 
 c = Chain(Dense(2, 1, tanh))
 nn = SymbolicNeuralNetwork(c)

src/derivatives/jacobian.jl

@@ -45,7 +45,7 @@ We can use `Jacobian` together with [`build_nn_function`](@ref):
 ```jldoctest
 using SymbolicNeuralNetworks
 using SymbolicNeuralNetworks: Jacobian, derivative
-using AbstractNeuralNetworks: Dense, Chain, NeuralNetwork
+using AbstractNeuralNetworks: Dense, Chain, NeuralNetwork, params
 using Symbolics
 import Random
 
@@ -59,7 +59,7 @@ nn = SymbolicNeuralNetwork(c)
 □ = SymbolicNeuralNetworks.Jacobian(nn)
 # here we need to access the derivative and convert it into a function
 jacobian1 = build_nn_function(derivative(□), nn)
-ps = NeuralNetwork(c, Float64).params
+ps = params(NeuralNetwork(c, Float64))
 input = rand(input_dim)
 #derivative
 Dtanh(x::Real) = 4 * exp(2 * x) / (1 + exp(2x)) ^ 2

src/derivatives/pullback.jl

@@ -108,8 +108,11 @@ SymbolicPullback(nn::SymbolicNeuralNetwork) = SymbolicPullback(nn, AbstractNeura
 Return the `NamedTuple` that's equivalent to the `NeuralNetworkParameters`.
 """
 _get_params(nt::NamedTuple) = nt
+function _get_params(nt::NamedTuple{(:params,), Tuple{AT}}) where {AT <: Any}
+    @warn "This function was most likely called because @adjoint for `NeuralNetworkParameters` hasn't been implemented."
+    nt.params
+end
 _get_params(ps::NeuralNetworkParameters) = params(ps)
-_get_params(ps::NamedTuple{(:params,), Tuple{NT}}) where {NT<:NamedTuple} = ps.params
 _get_params(ps::AbstractArray{<:Union{NamedTuple, NeuralNetworkParameters}}) = [_get_params(nt) for nt in ps]
 
 """