Merge branch 'main' into fix-reshape

Author: benedict-96

Project.toml

@@ -1,7 +1,7 @@
 name = "SymbolicNeuralNetworks"
 uuid = "aed23131-dcd0-47ca-8090-d21e605652e3"
 authors = ["Michael Kraus"]
-version = "0.2.0"
+version = "0.3.0"
 
 [deps]
 AbstractNeuralNetworks = "60874f82-5ada-4c70-bd1c-fa6be7711c8a"
@@ -10,25 +10,30 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 RuntimeGeneratedFunctions = "7e49a35a-f44a-4d26-94aa-eba1b4ca6b47"
 Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7"
 
+[weakdeps]
+SymbolicUtils = "d1185830-fcd6-423d-90d6-eec64667417b"
+
 [compat]
-AbstractNeuralNetworks = "0.3, 0.4"
+AbstractNeuralNetworks = "0.3, 0.4, 0.5, 0.6"
 Documenter = "1.8.0"
 ForwardDiff = "0.10.38"
+GeometricMachineLearning = "0.4"
 Latexify = "0.16.5"
 RuntimeGeneratedFunctions = "0.5"
-SafeTestsets = "0.1"
+SymbolicUtils = "<3.8.0"
 Symbolics = "5, 6"
 Zygote = "0.6.73"
-julia = "1.6"
+julia = "1.10"
 
 [extras]
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+GeometricMachineLearning = "194d25b2-d3f5-49f0-af24-c124f4aa80cc"
 Latexify = "23fbe1c1-3f47-55db-b15f-69d7ec21a316"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [targets]
-test = ["Test", "ForwardDiff", "Random", "Documenter", "Latexify", "SafeTestsets", "Zygote"]
+test = ["Test", "ForwardDiff", "Random", "Documenter", "Latexify", "SafeTestsets", "Zygote", "GeometricMachineLearning"]

docs/src/double_derivative.md

@@ -59,7 +59,7 @@ We can easily differentiate a neural network twice by using [`SymbolicNeuralNetw
 ```@example jacobian_gradient
 using AbstractNeuralNetworks
 using SymbolicNeuralNetworks
-using SymbolicNeuralNetworks: Jacobian, Gradient, derivative
+using SymbolicNeuralNetworks: Jacobian, Gradient, derivative, params
 using Latexify: latexify
 
 c = Chain(Dense(2, 1, tanh))
@@ -92,7 +92,7 @@ x = \begin{pmatrix} 1 \\ 0 \end{pmatrix}, \quad W = \begin{bmatrix} 1 & 0 \\ 0 &
 ```
 
 ```@example jacobian_gradient
-built_function = build_nn_function(derivative(g), nn.params, nn.input)
+built_function = build_nn_function(derivative(g), params(nn), nn.input)
 
 x = [1., 0.]
 ps = NeuralNetworkParameters((L1 = (W = [1. 0.; 0. 1.], b = [0., 0.]), ))

docs/src/symbolic_neural_networks.md

@@ -6,7 +6,7 @@ We first call the symbolic neural network that only consists of one layer:
 
 ```@example snn
 using SymbolicNeuralNetworks
-using AbstractNeuralNetworks: Chain, Dense, initialparameters
+using AbstractNeuralNetworks: Chain, Dense, params
 
 input_dim = 2
 output_dim = 1
@@ -23,7 +23,7 @@ using Symbolics
 using Latexify: latexify
 
 @variables sinput[1:input_dim]
-soutput = nn.model(sinput, nn.params)
+soutput = nn.model(sinput, params(nn))
 
 soutput
 ```
@@ -101,7 +101,7 @@ We now compare the neural network-approximated curve to the original one:
 fig = Figure()
 ax = Axis3(fig[1, 1])
 
-surface!(x_vec, y_vec, [c([x, y], nn_cpu.params)[1] for x in x_vec, y in y_vec]; alpha = .8, colormap = :darkterrain, transparency = true)
+surface!(x_vec, y_vec, [c([x, y], params(nn_cpu))[1] for x in x_vec, y in y_vec]; alpha = .8, colormap = :darkterrain, transparency = true)
 fig
 ```
 

scripts/pullback_comparison.jl

@@ -1,7 +1,7 @@
 using SymbolicNeuralNetworks
 using AbstractNeuralNetworks
 using GeometricMachineLearning
-using AbstractNeuralNetworks: FeedForwardLoss
+using AbstractNeuralNetworks: FeedForwardLoss, params
 using GeometricMachineLearning: ZygotePullback
 import Random
 Random.seed!(123)
@@ -19,16 +19,16 @@ output = rand(1, batch_size)
 # output sensitivities
 _do = 1.
 
-# spb(nn_cpu.params, nn.model, (input, output))[2](_do)
-# zpb(nn_cpu.params, nn.model, (input, output))[2](_do)
-# @time spb_evaluated = spb(nn_cpu.params, nn.model, (input, output))[2](_do)
-# @time zpb_evaluated = zpb(nn_cpu.params, nn.model, (input, output))[2](_do)[1].params
+# spb(params(nn_cpu), nn.model, (input, output))[2](_do)
+# zpb(params(nn_cpu), nn.model, (input, output))[2](_do)
+# @time spb_evaluated = spb(params(nn_cpu), nn.model, (input, output))[2](_do)
+# @time zpb_evaluated = zpb(params(nn_cpu), nn.model, (input, output))[2](_do)[1].params
 # @assert values(spb_evaluated) .≈ values(zpb_evaluated)
 
-function timenn(pb, params, model, input, output, _do = 1.)
-    pb(params, model, (input, output))[2](_do)
-    @time pb(params, model, (input, output))[2](_do)
+function timenn(pb, _params, model, input, output, _do = 1.)
+    pb(_params, model, (input, output))[2](_do)
+    @time pb(_params, model, (input, output))[2](_do)
 end
 
-timenn(spb, nn_cpu.params, nn.model, input, output)
-timenn(zpb, nn_cpu.params, nn.model, input, output)
+timenn(spb, params(nn_cpu), nn.model, input, output)
+timenn(zpb, params(nn_cpu), nn.model, input, output)

src/SymbolicNeuralNetworks.jl

@@ -15,44 +15,28 @@ module SymbolicNeuralNetworks
 
     RuntimeGeneratedFunctions.init(@__MODULE__)
 
-    include("equation_types.jl")
+    include("custom_definitions_and_extensions/equation_types.jl")
 
-    export symbolize
-    include("utils/symbolize.jl")
+    include("symbolic_neuralnet/symbolize.jl")
 
     include("utils/create_array.jl")
 
     export AbstractSymbolicNeuralNetwork
-    export SymbolicNeuralNetwork, SymbolicModel
-    export HamiltonianSymbolicNeuralNetwork, HNNLoss
-    export architecture, model, params, equations, functions
+    export SymbolicNeuralNetwork
 
-    # make symbolic parameters (`NeuralNetworkParameters`)
-    export symbolicparameters
-    include("layers/abstract.jl")
-    include("layers/dense.jl")
-    include("layers/linear.jl")
-    include("chain.jl")
-
-    export evaluate_equations
-    include("symbolic_neuralnet.jl")
-
-    export symbolic_hamiltonian
-    include("hamiltonian.jl")
+    include("symbolic_neuralnet/symbolic_neuralnet.jl")
 
     export build_nn_function
-    include("utils/build_function.jl")
-    include("utils/build_function2.jl")
-    include("utils/build_function_arrays.jl")
+    include("build_function/build_function.jl")
+    include("build_function/build_function_double_input.jl")
+    include("build_function/build_function_arrays.jl")
 
     export SymbolicPullback
-    include("pullback.jl")
+    include("derivatives/pullback.jl")
 
     include("derivatives/derivative.jl")
     include("derivatives/jacobian.jl")
     include("derivatives/gradient.jl")
 
-    include("custom_equation.jl")
-
-    include("utils/latexraw.jl")
+    include("custom_definitions_and_extensions/latexraw.jl")
 end

src/utils/build_function.jl renamed to src/build_function/build_function.jl

@@ -19,7 +19,7 @@ The functions mentioned in the implementation section were adjusted ad-hoc to de
 Other problems may occur. In case you bump into one please [open an issue on github](https://github.com/JuliaGNI/SymbolicNeuralNetworks.jl/issues).
 """
 function build_nn_function(eq::EqT, nn::AbstractSymbolicNeuralNetwork)
-    build_nn_function(eq, nn.params, nn.input)
+    build_nn_function(eq, params(nn), nn.input)
 end
 
 function build_nn_function(eq::EqT, sparams::NeuralNetworkParameters, sinput::Symbolics.Arr; reduce = hcat)
@@ -46,25 +46,26 @@ Build a function that can process a matrix. This is used as a starting point for
 # Examples
 
 ```jldoctest
-using SymbolicNeuralNetworks: _build_nn_function, symbolicparameters
-using Symbolics
-using AbstractNeuralNetworks
+using SymbolicNeuralNetworks: _build_nn_function, SymbolicNeuralNetwork
+using AbstractNeuralNetworks: params, Chain, Dense, NeuralNetwork
+import Random
+Random.seed!(123)
 
 c = Chain(Dense(2, 1, tanh))
-params = symbolicparameters(c)
-@variables sinput[1:2]
-eq = c(sinput, params)
-built_function = _build_nn_function(eq, params, sinput)
-ps = initialparameters(c)
-input = rand(2, 2)
-
-(built_function(input, ps, 1), built_function(input, ps, 2)) .≈ (c(input[:, 1], ps), c(input[:, 2], ps))
+nn = NeuralNetwork(c)
+snn = SymbolicNeuralNetwork(nn)
+eq = c(snn.input, params(snn))
+built_function = _build_nn_function(eq, params(snn), snn.input)
+built_function([1. 2.; 3. 4.], params(nn), 1)
 
 # output
 
-(true, true)
+1-element Vector{Float64}:
+ 0.9912108161055604
 ```
 
+Note that we have to supply an extra argument (index) to `_build_nn_function` that we do not have to supply to [`build_nn_function`](@ref).
+
 # Implementation
 
 This first calls `Symbolics.build_function` with the keyword argument `expression = Val{true}` and then modifies the generated code by calling:
@@ -75,31 +76,16 @@ This first calls `Symbolics.build_function` with the keyword argument `expressio
 
 See the docstrings for those functions for details on how the code is modified. 
 """
-function _build_nn_function(eq::EqT, params::NeuralNetworkParameters, sinput::Symbolics.Arr)
+function _build_nn_function(eq::EqT, sparams::NeuralNetworkParameters, sinput::Symbolics.Arr)
     sc_eq = Symbolics.scalarize(eq)
-    code = build_function(sc_eq, sinput, values(params)...; expression = Val{true}) |> _reduce_code
+    code = build_function(sc_eq, sinput, values(sparams)...; expression = Val{true}) |> _reduce
     rewritten_code = fix_map_reduce(modify_input_arguments(rewrite_arguments(fix_create_array(code))))
     parallelized_code = make_kernel(rewritten_code)
     @RuntimeGeneratedFunction(parallelized_code)
 end
 
-"""
-    _reduce_code(code)
-
-Reduce the code.
-
-For some reason `Symbolics.build_function` sometimes returns a tuple and sometimes it doesn't.
-
-This function takes care of this. 
-If `build_function` returns a tuple `reduce_code` checks which of the expressions is in-place and then returns the other (not in-place) expression.
-"""
-function _reduce_code(code::Expr)
-    code
-end
-
-function _reduce_code(code::Tuple{Expr, Expr})
-    contains(string(code[1]), "ˍ₋out") ? code[2] : code[1]
-end
+_reduce(a) = a
+_reduce(a::Tuple) = a[1]
 
 """
     rewrite_arguments(s)

src/utils/build_function_arrays.jl renamed to src/build_function/build_function_arrays.jl

@@ -1,32 +1,29 @@
 """
     build_nn_function(eqs::AbstractArray{<:NeuralNetworkParameters}, sparams, sinput...)
 
-Build an executable function based on `eqs` that potentially also has a symbolic output.
+Build an executable function based on an array of symbolic equations `eqs`.
 
 # Examples
 
 ```jldoctest
 using SymbolicNeuralNetworks: build_nn_function, SymbolicNeuralNetwork
-using AbstractNeuralNetworks: Chain, Dense, initialparameters, NeuralNetworkParameters
+using AbstractNeuralNetworks: Chain, Dense, NeuralNetwork, params
 import Random
 Random.seed!(123)
 
 ch = Chain(Dense(2, 1, tanh))
-nn = SymbolicNeuralNetwork(ch)
-eqs = [(a = ch(nn.input, nn.params), b = ch(nn.input, nn.params).^2), (c = ch(nn.input, nn.params).^3, )]
-funcs = build_nn_function(eqs, nn.params, nn.input)
+nn = NeuralNetwork(ch)
+snn = SymbolicNeuralNetwork(nn)
+eqs = [(a = ch(snn.input, params(snn)), b = ch(snn.input, params(snn)).^2), (c = ch(snn.input, params(snn)).^3, )]
+funcs = build_nn_function(eqs, params(snn), snn.input)
 input = [1., 2.]
-ps = initialparameters(ch) |> NeuralNetworkParameters
-a = ch(input, ps)
-b = ch(input, ps).^2
-c = ch(input, ps).^3
-funcs_evaluated = funcs(input, ps)
-
-(funcs_evaluated[1].a, funcs_evaluated[1].b, funcs_evaluated[2].c) .≈ (a, b, c)
+funcs_evaluated = funcs(input, params(nn))
 
 # output
 
-(true, true, true)
+2-element Vector{NamedTuple}:
+ (a = [0.985678060655224], b = [0.9715612392570434])
+ (c = [0.9576465981186686],)
 ```
 """
 function build_nn_function(eqs::AbstractArray{<:Union{NamedTuple, NeuralNetworkParameters}}, sparams::NeuralNetworkParameters, sinput::Symbolics.Arr...; reduce = hcat)
@@ -47,25 +44,21 @@ Return a function that takes an input, (optionally) an output and neural network
 
 ```jldoctest
 using SymbolicNeuralNetworks: build_nn_function, SymbolicNeuralNetwork
-using AbstractNeuralNetworks: Chain, Dense, initialparameters, NeuralNetworkParameters
+using AbstractNeuralNetworks: Chain, Dense, NeuralNetwork, params
 import Random
 Random.seed!(123)
 
 c = Chain(Dense(2, 1, tanh))
-nn = SymbolicNeuralNetwork(c)
-eqs = (a = c(nn.input, nn.params), b = c(nn.input, nn.params).^2)
-funcs = build_nn_function(eqs, nn.params, nn.input)
+nn = NeuralNetwork(c)
+snn = SymbolicNeuralNetwork(nn)
+eqs = (a = c(snn.input, params(snn)), b = c(snn.input, params(snn)).^2)
+funcs = build_nn_function(eqs, params(snn), snn.input)
 input = [1., 2.]
-ps = initialparameters(c) |> NeuralNetworkParameters
-a = c(input, ps)
-b = c(input, ps).^2
-funcs_evaluated = funcs(input, ps)
-
-(funcs_evaluated.a, funcs_evaluated.b) .≈ (a, b)
+funcs_evaluated = funcs(input, params(nn))
 
 # output
 
-(true, true)
+(a = [0.985678060655224], b = [0.9715612392570434])
 ```
 
 # Implementation
@@ -90,16 +83,17 @@ Return an executable function for each entry in `eqs`. This still has to be proc
 
 ```jldoctest
 using SymbolicNeuralNetworks: function_valued_parameters, SymbolicNeuralNetwork
-using AbstractNeuralNetworks: Chain, Dense, initialparameters, NeuralNetworkParameters
+using AbstractNeuralNetworks: Chain, Dense, NeuralNetwork, params
 import Random
 Random.seed!(123)
 
 c = Chain(Dense(2, 1, tanh))
-nn = SymbolicNeuralNetwork(c)
-eqs = (a = c(nn.input, nn.params), b = c(nn.input, nn.params).^2)
-funcs = function_valued_parameters(eqs, nn.params, nn.input)
+nn = NeuralNetwork(c)
+snn = SymbolicNeuralNetwork(nn)
+eqs = (a = c(snn.input, params(snn)), b = c(snn.input, params(snn)).^2)
+funcs = function_valued_parameters(eqs, params(snn), snn.input)
 input = [1., 2.]
-ps = initialparameters(c) |> NeuralNetworkParameters
+ps = params(nn)
 a = c(input, ps)
 b = c(input, ps).^2
 

src/utils/build_function2.jl renamed to src/build_function/build_function_double_input.jl

@@ -13,7 +13,7 @@ Also compare this to [`build_nn_function(::EqT, ::AbstractSymbolicNeuralNetwork)
 See the *extended help section* of [`build_nn_function(::EqT, ::AbstractSymbolicNeuralNetwork)`](@ref).
 """
 function build_nn_function(eqs, nn::AbstractSymbolicNeuralNetwork, soutput)
-    build_nn_function(eqs, nn.params, nn.input, soutput)
+    build_nn_function(eqs, params(nn), nn.input, soutput)
 end
 
 function build_nn_function(eq::EqT, sparams::NeuralNetworkParameters, sinput::Symbolics.Arr, soutput::Symbolics.Arr; reduce = hcat)
@@ -59,7 +59,7 @@ See the docstrings for those functions for details on how the code is modified.
 """
 function _build_nn_function(eq::EqT, params::NeuralNetworkParameters, sinput::Symbolics.Arr, soutput::Symbolics.Arr)
     sc_eq = Symbolics.scalarize(eq)
-    code = build_function(sc_eq, sinput, soutput, values(params)...; expression = Val{true}) |> _reduce_code
+    code = build_function(sc_eq, sinput, soutput, values(params)...; expression = Val{true}) |> _reduce
     rewritten_code = fix_map_reduce(modify_input_arguments2(rewrite_arguments2(fix_create_array(code))))
     parallelized_code = make_kernel2(rewritten_code)
     @RuntimeGeneratedFunction(parallelized_code)