Softmax layer refactoring

Author: Sergio0694

NeuralNetwork.NET/APIs/TrainingAlgorithmInfo.cs

@@ -15,7 +15,7 @@ public static class TrainingAlgorithmsInfo
         /// <param name="lambda">The lambda regularization parameter</param>
         [PublicAPI]
         [Pure, NotNull]
-        public static StochasticGradientDescentInfo CreateForStochasticGradientDescent(float eta = 0.1f, float lambda = 0f) => new StochasticGradientDescentInfo(eta, lambda);
+        public static StochasticGradientDescentInfo StochasticGradientDescent(float eta = 0.1f, float lambda = 0f) => new StochasticGradientDescentInfo(eta, lambda);
 
         /// <summary>
         /// Gets an instance implementing <see cref="Interfaces.ITrainingAlgorithmInfo"/> for the <see cref="SupervisedLearning.Algorithms.TrainingAlgorithmType.Adadelta"/> algorithm
@@ -25,6 +25,6 @@ public static class TrainingAlgorithmsInfo
         /// <param name="l2">An optional L2 regularization parameter</param>
         [PublicAPI]
         [Pure, NotNull]
-        public static AdadeltaInfo CreateForAdadelta(float rho = 0.95f, float epsilon = 1e-8f, float l2 = 0f) => new AdadeltaInfo(rho, epsilon, l2);
+        public static AdadeltaInfo Adadelta(float rho = 0.95f, float epsilon = 1e-8f, float l2 = 0f) => new AdadeltaInfo(rho, epsilon, l2);
     }
 }

NeuralNetwork.NET/Extensions/MatrixExtensions.cs

@@ -13,43 +13,6 @@ namespace NeuralNetworkNET.Extensions
     /// </summary>
     public static class MatrixExtensions
     {
-        #region Activation
-
-        /// <summary>
-        /// Performs the softmax normalization on the input matrix, dividing every value by the sum of all the values
-        /// </summary>
-        /// <param name="m">The matrix to normalize</param>
-        internal static unsafe void InPlaceSoftmaxNormalization(in this Tensor m)
-        {
-            // Setup
-            int h = m.Entities, w = m.Length;
-            Tensor.New(1, h, out Tensor partials);
-            float* pp = partials, pm = m;
-
-            // Partial sum
-            void PartialSum(int i)
-            {
-                int offset = i * w;
-                float sum = 0;
-                for (int j = 0; j < w; j++)
-                    sum += pm[offset + j];
-                pp[i] = sum;
-            }
-            Parallel.For(0, h, PartialSum).AssertCompleted();
-
-            // Normalization of the matrix values
-            void NormalizationKernel(int i)
-            {
-                int offset = i * w;
-                for (int j = 0; j < w; j++)
-                    pm[offset + j] /= pp[i];
-            }
-            Parallel.For(0, h, NormalizationKernel).AssertCompleted();
-            partials.Free();
-        }
-
-        #endregion
-
         #region Misc
 
         /// <summary>

NeuralNetwork.NET/Networks/Layers/Cpu/SoftmaxLayer.cs

@@ -3,6 +3,7 @@
 using NeuralNetworkNET.APIs.Enums;
 using NeuralNetworkNET.APIs.Interfaces;
 using NeuralNetworkNET.APIs.Structs;
+using NeuralNetworkNET.cpuDNN;
 using NeuralNetworkNET.Extensions;
 using NeuralNetworkNET.Networks.Activations;
 using NeuralNetworkNET.Networks.Cost;
@@ -22,10 +23,17 @@ public SoftmaxLayer(in TensorInfo input, int outputs, WeightsInitializationMode
             : base(input, outputs, ActivationFunctionType.Softmax, CostFunctionType.LogLikelyhood, weightsMode, biasMode) { }
 
         /// <inheritdoc/>
-        public override void Forward(in Tensor x, out Tensor z, out Tensor a)
+        public override unsafe void Forward(in Tensor x, out Tensor z, out Tensor a)
         {
-            base.Forward(x, out z, out a);
-            a.InPlaceSoftmaxNormalization();
+            fixed (float* pw = Weights, pb = Biases)
+            {
+                Tensor.Reshape(pw, InputInfo.Size, OutputInfo.Size, out Tensor w);
+                Tensor.Reshape(pb, 1, Biases.Length, out Tensor b);
+                Tensor.New(x.Entities, OutputInfo.Size, out z);
+                CpuDnn.FullyConnectedForward(x, w, b, z);
+                Tensor.New(z.Entities, z.Length, out a);
+                CpuDnn.SoftmaxForward(z, a);
+            }
         }
 
         public SoftmaxLayer(in TensorInfo input, int outputs, [NotNull] float[] weights, [NotNull] float[] biases)

Samples/DigitsCudaTest/Program.cs

@@ -36,7 +36,7 @@ static async Task Main()
             CancellationTokenSource cts = new CancellationTokenSource();
             Console.CancelKeyPress += (s, e) => cts.Cancel();
             TrainingSessionResult result = await NetworkManager.TrainNetworkAsync(network, (training.X, training.Y), 20, 400,
-                TrainingAlgorithmsInfo.CreateForAdadelta(), 0.5f,
+                TrainingAlgorithmsInfo.Adadelta(), 0.5f,
                 new Progress<BatchProgress>(p =>
                 {
                     Console.SetCursorPosition(0, Console.CursorTop);

Samples/DigitsTest/Program.cs

@@ -21,7 +21,7 @@ static async Task Main()
                 NetworkLayers.FullyConnected(100, ActivationFunctionType.Sigmoid),
                 NetworkLayers.FullyConnected(10, ActivationFunctionType.Sigmoid, CostFunctionType.CrossEntropy));
             TrainingSessionResult result = await NetworkManager.TrainNetworkAsync(network, (training.X, training.Y), 60, 10,
-                TrainingAlgorithmsInfo.CreateForStochasticGradientDescent(), 0.5f,
+                TrainingAlgorithmsInfo.StochasticGradientDescent(), 0.5f,
                 testParameters: new TestParameters(test, new Progress<BackpropagationProgressEventArgs>(p =>
                 {
                     Printf($"Epoch {p.Iteration}, cost: {p.Result.Cost}, accuracy: {p.Result.Accuracy}");

Unit/NeuralNetwork.NET.Unit/NetworkTest.cs

@@ -91,7 +91,7 @@ public void GradientDescentTest1()
             NeuralNetwork network = NetworkManager.NewSequential(TensorInfo.CreateForGrayscaleImage(28, 28),
                 NetworkLayers.FullyConnected(100, ActivationFunctionType.Sigmoid),
                 NetworkLayers.Softmax(10)).To<INeuralNetwork, NeuralNetwork>();
-            TrainingSessionResult result = NetworkTrainer.TrainNetwork(network, batches, 4, 0, TrainingAlgorithmsInfo.CreateForStochasticGradientDescent(), null, null, null, null, default);
+            TrainingSessionResult result = NetworkTrainer.TrainNetwork(network, batches, 4, 0, TrainingAlgorithmsInfo.StochasticGradientDescent(), null, null, null, null, default);
             Assert.IsTrue(result.StopReason == TrainingStopReason.EpochsCompleted);
             (_, _, float accuracy) = network.Evaluate(testSet);
             Assert.IsTrue(accuracy > 80);