Fully connected cpu layer code refactored and tested

Author: Sergio0694

NeuralNetwork.NET/Extensions/MatrixExtensions.cs

@@ -716,7 +716,14 @@ public static unsafe bool ContentEquals(in this Tensor m, in Tensor o,float abso
             float* pm = m, po = o;
             int items = m.Size;
             for (int i = 0; i < items; i++)
-                if (!pm[i].EqualsWithDelta(po[i], absolute, relative)) return false;
+                if (!pm[i].EqualsWithDelta(po[i], absolute, relative))
+                {
+                    #if DEBUG
+                    if (System.Diagnostics.Debugger.IsAttached)
+                        System.Diagnostics.Debug.WriteLine($"[DEBUG] {pm[i]} | {po[i]} | Threshold exceeded");
+                    #endif
+                    return false;
+                }
             return true;
         }
 

NeuralNetwork.NET/Networks/Layers/Cpu/FullyConnectedLayer.cs

@@ -4,6 +4,7 @@
 using NeuralNetworkNET.APIs.Enums;
 using NeuralNetworkNET.APIs.Interfaces;
 using NeuralNetworkNET.APIs.Structs;
+using NeuralNetworkNET.cpuDNN;
 using NeuralNetworkNET.Extensions;
 using NeuralNetworkNET.Networks.Activations;
 using NeuralNetworkNET.Networks.Activations.Delegates;
@@ -35,11 +36,14 @@ public FullyConnectedLayer(in TensorInfo input, int neurons, [NotNull] float[] w
         /// <inheritdoc/>
         public override unsafe void Forward(in Tensor x, out Tensor z, out Tensor a)
         {
-            fixed (float* pw = Weights)
+            fixed (float* pw = Weights, pb = Biases)
             {
-                Tensor.Reshape(pw, InputInfo.Size, OutputInfo.Size, out Tensor wTensor);
-                x.MultiplyWithSum(wTensor, Biases, out z);
-                z.Activation(ActivationFunctions.Activation, out a);
+                Tensor.Reshape(pw, InputInfo.Size, OutputInfo.Size, out Tensor w);
+                Tensor.Reshape(pb, 1, Biases.Length, out Tensor b);
+                Tensor.New(x.Entities, OutputInfo.Size, out z);
+                CpuDnn.FullyConnectedForward(x, w, b, z);
+                Tensor.New(z.Entities, z.Length, out a);
+                CpuDnn.ActivationForward(z, ActivationFunctions.Activation, a);
             }
         }
 
@@ -48,21 +52,19 @@ public override unsafe void Backpropagate(in Tensor dy, in Tensor z, ActivationF
         {
             fixed (float* pw = Weights)
             {
-                Tensor.Reshape(pw, InputInfo.Size, OutputInfo.Size, out Tensor wTensor);
-                wTensor.Transpose(out Tensor wt);
-                z.InPlaceMultiplyAndHadamardProductWithActivationPrime(dy, wt, activationPrime);
-                wt.Free();
+                Tensor.Reshape(pw, InputInfo.Size, OutputInfo.Size, out Tensor w);
+                CpuDnn.FullyConnectedBackwardData(z, w, dy, activationPrime, z);
             }
         }
 
         /// <inheritdoc/>
         public override void ComputeGradient(in Tensor a, in Tensor delta, out Tensor dJdw, out Tensor dJdb)
         {
-            a.Transpose(out Tensor at);
-            at.Multiply(delta, out Tensor dJdwM);
-            dJdwM.Reshape(1, Weights.Length, out dJdw);
-            at.Free();
-            delta.CompressVertically(out dJdb);
+            Tensor.New(InputInfo.Size, OutputInfo.Size, out Tensor dw);
+            CpuDnn.FullyConnectedBackwardFilter(a, delta, dw);
+            dw.Reshape(1, dw.Size, out dJdw); // Flatten the result
+            Tensor.New(1, Biases.Length, out dJdb);
+            CpuDnn.FullyConnectedBackwardBias(delta, dJdb);
         }
 
         /// <inheritdoc/>

NeuralNetwork.NET/cpuDNN/CpuDnn{Main}.cs

@@ -7,10 +7,19 @@
 
 namespace NeuralNetworkNET.cpuDNN
 {
+    /// <summary>
+    /// A static class that contains primitives to implement a CNN running on CPU
+    /// </summary>
     public static partial class CpuDnn
     {
         #region Activation
 
+        /// <summary>
+        /// Executes the input activation function on the target <see cref="Tensor"/>
+        /// </summary>
+        /// <param name="x">The layer input <see cref="Tensor"/></param>
+        /// <param name="f">The activation function to apply to the input</param>
+        /// <param name="y">The output <see cref="Tensor"/> - it can be the same as the input</param>
         public static unsafe void ActivationForward(in Tensor x, [NotNull] ActivationFunction f, in Tensor y)
         {
             // Setup
@@ -31,15 +40,22 @@ void Kernel(int i)
             Parallel.For(0, n, Kernel).AssertCompleted();
         }
 
-        public static unsafe void ActivationBackward(in Tensor x, in Tensor y, [NotNull] ActivationFunction f_, in Tensor dx)
+        /// <summary>
+        /// Executes the backward activation function on the target <see cref="Tensor"/>, with the given error delta
+        /// </summary>
+        /// <param name="x">The activity on the input layer</param>
+        /// <param name="dy">The current error delta to backpropagate</param>
+        /// <param name="f_">The derivative of the activation function used in the forward pass</param>
+        /// <param name="dx">The resulting input error delta - it can be the same as the input <see cref="Tensor"/></param>
+        public static unsafe void ActivationBackward(in Tensor x, in Tensor dy, [NotNull] ActivationFunction f_, in Tensor dx)
         {
             // Check
-            if (!y.MatchShape(x)) throw new ArgumentException("The input tensors must have the same shape", nameof(y));
-            if (!dx.MatchShape(x)) throw new ArgumentException("The output tensor must have the same shape as the input", nameof(y));
+            if (!dy.MatchShape(x)) throw new ArgumentException("The input tensors must have the same shape", nameof(dy));
+            if (!dx.MatchShape(x)) throw new ArgumentException("The output tensor must have the same shape as the input", nameof(dy));
             int
                 n = x.Entities,
                 l = x.Length;
-            float* px = x, py = y, pdx = dx;
+            float* px = x, pdy = dy, pdx = dx;
 
             // Loop in parallel
             void Kernel(int i)
@@ -48,7 +64,7 @@ void Kernel(int i)
                 for (int j = 0; j < l; j++)
                 {
                     int target = offset + j;
-                    pdx[target] = f_(px[target]) * py[target];
+                    pdx[target] = f_(px[target]) * pdy[target];
                 }
             }
             Parallel.For(0, n, Kernel).AssertCompleted();
@@ -58,6 +74,13 @@ void Kernel(int i)
 
         #region Fully connected
 
+        /// <summary>
+        /// Executes the forward pass on a fully connected layer
+        /// </summary>
+        /// <param name="x">The input <see cref="Tensor"/> to process</param>
+        /// <param name="w">The layer weights</param>
+        /// <param name="b">The layer biases</param>
+        /// <param name="y">The output <see cref="Tensor"/> for the current layer</param>
         public static unsafe void FullyConnectedForward(in Tensor x, in Tensor w, in Tensor b, in Tensor y)
         {
             // Initialize the parameters and the result matrix
@@ -89,6 +112,14 @@ void Kernel(int i)
             Parallel.For(0, h, Kernel).AssertCompleted();
         }
 
+        /// <summary>
+        /// Executes the backward pass on a fully connected layer
+        /// </summary>
+        /// <param name="x">The activity on the layer inputs</param>
+        /// <param name="w">The layer weights</param>
+        /// <param name="dy">The output error delta</param>
+        /// <param name="f_">The derivative of the activation function used in the forward pass</param>
+        /// <param name="dx">The resulting input error delta</param>
         public static unsafe void FullyConnectedBackwardData(in Tensor x, in Tensor w, in Tensor dy, [NotNull] ActivationFunction f_, in Tensor dx)
         {
             if (w.Entities != x.Length) throw new ArgumentException("The weights tensor doesn't have a valid shape", nameof(w));
@@ -127,6 +158,12 @@ void Kernel(int i)
             wt.Free();
         }
 
+        /// <summary>
+        /// Executes the backward pass on a fully connected layer to calculate the gradient with respect to the weights
+        /// </summary>
+        /// <param name="x">The layer inputs</param>
+        /// <param name="dy">The layer output error delta</param>
+        /// <param name="dw">The resulting weights gradient <see cref="Tensor"/></param>
         public static void FullyConnectedBackwardFilter(in Tensor x, in Tensor dy, in Tensor dw)
         {
             if (x.Entities != dy.Entities) throw new ArgumentException("The input tensor doesn't match the number of samples from the delta", nameof(x));
@@ -136,10 +173,15 @@ public static void FullyConnectedBackwardFilter(in Tensor x, in Tensor dy, in Te
             xt.Free();
         }
 
+        /// <summary>
+        /// Executes the backward pass on a fully connected layer to calculate the gradient with respect to the biases
+        /// </summary>
+        /// <param name="dy">The layer output error delta</param>
+        /// <param name="db">The resulting biases gradient <see cref="Tensor"/></param>
         public static unsafe void FullyConnectedBackwardBias(in Tensor dy, in Tensor db)
         {
             // Preliminary checks and declarations
-            if (!db.MatchShape(1, dy.Length)) throw new ArgumentException("The output tensor doesn't have the right shape", nameof(db));
+            if (!db.MatchShape(1, dy.Length)) throw new ArgumentException("Invalid result tensor shape", nameof(db));
             int
                 n = dy.Entities,
                 l = dy.Length;
@@ -150,8 +192,8 @@ void Kernel(int j)
             {
                 float sum = 0;
                 for (int i = 0; i < n; i++)
-                    sum += pdb[i * l + j];
-                pdy[j] = sum;
+                    sum += pdy[i * l + j];
+                pdb[j] = sum;
             }
             Parallel.For(0, l, Kernel).AssertCompleted();
         }