diff --git a/NeuralNetwork.NET/APIs/Enums/ActivationType.cs b/NeuralNetwork.NET/APIs/Enums/ActivationType.cs index 7981751..e7abe65 100644 --- a/NeuralNetwork.NET/APIs/Enums/ActivationType.cs +++ b/NeuralNetwork.NET/APIs/Enums/ActivationType.cs @@ -1,4 +1,4 @@ -namespace NeuralNetworkNET.APIs.Enums +namespace NeuralNetworkNET.APIs.Enums { /// /// Indicates an activation function to use in a neural network @@ -59,6 +59,14 @@ public enum ActivationType : byte /// /// A linear activation function that just returns the input value /// - Identity + Identity, + + /// + /// The Mish function, proposed by Diganta Misra (https://arxiv.org/abs/1908.08681) + /// Definition: x tanh(ln(1 + e^2)) + /// Implimentation: x * Tanh(Softplus(x)) + /// + Mish + } -} \ No newline at end of file +} diff --git a/NeuralNetwork.NET/Networks/Activations/ActivationFunctionProvider.cs b/NeuralNetwork.NET/Networks/Activations/ActivationFunctionProvider.cs index 41726b0..78ef6c8 100644 --- a/NeuralNetwork.NET/Networks/Activations/ActivationFunctionProvider.cs +++ b/NeuralNetwork.NET/Networks/Activations/ActivationFunctionProvider.cs @@ -1,4 +1,4 @@ -using System; +using System; using JetBrains.Annotations; using NeuralNetworkNET.APIs.Enums; using NeuralNetworkNET.Networks.Activations.Delegates; @@ -28,7 +28,8 @@ public static (ActivationFunction, ActivationFunction) GetActivations(Activation case ActivationType.Softmax: return (ActivationFunctions.Softmax, null); case ActivationType.Softplus: return (ActivationFunctions.Softplus, ActivationFunctions.Sigmoid); case ActivationType.ELU: return (ActivationFunctions.ELU, ActivationFunctions.ELUPrime); - case ActivationType.Identity: return (ActivationFunctions.Identity, ActivationFunctions.Identityprime); + case ActivationType.Identity: return (ActivationFunctions.Identity, ActivationFunctions.IdentityPrime); + case ActivationType.Mish: return (ActivationFunctions.Mish, ActivationFunctions.MishPrime); default: throw new ArgumentOutOfRangeException(nameof(ActivationType), "Unsupported activation function"); } diff --git a/NeuralNetwork.NET/Networks/Activations/ActivationFunctions.cs b/NeuralNetwork.NET/Networks/Activations/ActivationFunctions.cs index 9b21adf..5b32a7d 100644 --- a/NeuralNetwork.NET/Networks/Activations/ActivationFunctions.cs +++ b/NeuralNetwork.NET/Networks/Activations/ActivationFunctions.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Runtime.CompilerServices; using JetBrains.Annotations; @@ -217,6 +217,30 @@ public static float Softplus(float x) [PublicAPI] [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static float Identityprime(float x) => 1; + public static float IdentityPrime(float x) => 1; + + /// + /// Applies the Mish function + /// + /// The input to process + [PublicAPI] + [Pure] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static float Mish(float x) => x * Tanh(Softplus(x)); + + /// + /// Applies the Mish Derivative function + /// + /// The input to process + [PublicAPI] + [Pure] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static float MishPrime(float x) + { + float + s = 2 * (float)Math.Exp(x) + (float)Math.Exp(2 * x) + 2, + w = 4 * (x + 1) + (4 * ((float)Math.Exp(2 * x))) + (float)Math.Exp(3 * x) + (float)Math.Exp(x) * (4 * x + 6); + return (float)Math.Exp(x) * w / (s * s); + } } }