Removed unnecessary code, Unit tests fixed

Author: Sergio0694

NeuralNetwork.NET/Extensions/ConvolutionExtensions.cs

@@ -2,6 +2,7 @@
 using System.Threading.Tasks;
 using JetBrains.Annotations;
 using NeuralNetworkNET.APIs.Structs;
+using NeuralNetworkNET.cpuDNN;
 using NeuralNetworkNET.Extensions;
 using NeuralNetworkNET.Networks.Activations.Delegates;
 
@@ -161,7 +162,7 @@ public static unsafe void QuadraticCostPrime(in Tensor yHat, in Tensor y, in Ten
 
             // Calculate (yHat - y) * activation'(z)
             float* pyHat = yHat, py = y, pz = z;
-            unsafe void Kernel(int i)
+            void Kernel(int i)
             {
                 // Save the index and iterate for each column
                 int offset = i * w;
@@ -192,7 +193,7 @@ public static void CrossEntropyCostPrime(in Tensor yHat, in Tensor y, in Tensor
             if (h != y.Entities || w != y.Length) throw new ArgumentException("The two matrices must have the same size");
 
             // Calculate (yHat - y)
-            yHat.Subtract(y);
+            CpuBlas.Subtract(yHat, y, yHat);
         }
 
         #endregion

NeuralNetwork.NET/Extensions/MatrixExtensions.cs

@@ -10,6 +10,7 @@
 using NeuralNetworkNET.APIs.Interfaces;
 using NeuralNetworkNET.APIs.Enums;
 using NeuralNetworkNET.APIs.Structs;
+using NeuralNetworkNET.cpuDNN;
 using NeuralNetworkNET.Extensions;
 using NeuralNetworkNET.Helpers;
 using NeuralNetworkNET.Networks.Activations;
@@ -229,7 +230,7 @@ internal unsafe void Backpropagate(in TrainingBatch batch, float dropout, [NotNu
                     if (_Layers[i].LayerType == LayerType.FullyConnected && dropout > 0)
                     {
                         ThreadSafeRandom.NextDropoutMask(aList[i].Entities, aList[i].Length, dropout, out dropoutMasks[i]);
-                        aList[i].InPlaceHadamardProduct(dropoutMasks[i]);
+                        CpuBlas.MultiplyElementwise(aList[i], dropoutMasks[i], aList[i]);
                     }
                 }
 
@@ -252,7 +253,7 @@ internal unsafe void Backpropagate(in TrainingBatch batch, float dropout, [NotNu
                      * Multiply the previous delta with the transposed weights of the following layer
                      * Compute d(l), the Hadamard product of z'(l) and delta(l + 1) * W(l + 1)T */
                     _Layers[l + 1].Backpropagate(*deltas[l + 1], zList[l], _Layers[l].ActivationFunctions.ActivationPrime);
-                    if (!dropoutMasks[l].IsNull) zList[l].InPlaceHadamardProduct(dropoutMasks[l]);
+                    if (!dropoutMasks[l].IsNull) CpuBlas.MultiplyElementwise(zList[l], dropoutMasks[l], zList[l]);
                     deltas[l] = zList + l;
                 }
 

NeuralNetwork.NET/Extensions/PoolingExtensions.cs

@@ -4,6 +4,7 @@
 using NeuralNetworkNET.APIs.Enums;
 using NeuralNetworkNET.APIs.Interfaces;
 using NeuralNetworkNET.APIs.Structs;
+using NeuralNetworkNET.cpuDNN;
 using NeuralNetworkNET.Extensions;
 using NeuralNetworkNET.Networks.Activations;
 using NeuralNetworkNET.Networks.Activations.Delegates;
@@ -68,7 +69,8 @@ public override void ComputeGradient(in Tensor a, in Tensor delta, out Tensor dJ
                 DnnInstance.FullyConnectedBackwardFilter(a.Entities, a.Length, delta.Length, a_gpu.Ptr, delta_gpu.Ptr, w_gpu.Ptr);
                 w_gpu.CopyToHost(1, Weights.Length, out dJdw);
             }
-            delta.CompressVertically(out dJdb); // Doing this on CPU is generally faster than launching the kernels
+            Tensor.New(1, Biases.Length, out dJdb);
+            CpuDnn.FullyConnectedBackwardBias(delta, dJdb); // Doing this on CPU is generally faster than launching the kernels
         }
 
         #endregion

NeuralNetwork.NET/Networks/Cost/CostFunctions.cs

@@ -65,6 +65,35 @@ void Kernel(int i)
             Parallel.For(0, n, Kernel).AssertCompleted();
         }
 
+        /// <summary>
+        /// Performs the in place multiplication (Hadamard product) product between two <see cref="Tensor"/> instances
+        /// </summary>
+        /// <param name="x1">The first <see cref="Tensor"/></param>
+        /// <param name="x2">The second <see cref="Tensor"/></param>
+        /// <param name="y">The resulting <see cref="Tensor"/></param>
+        public static unsafe void MultiplyElementwise(in Tensor x1, in Tensor x2, in Tensor y)
+        {
+            // Check
+            int
+                n = x1.Entities,
+                l = x1.Length;
+            if (!x1.MatchShape(x2)) throw new ArgumentException("The two input tensors must be of equal shape");
+            if (!x1.MatchShape(y)) throw new ArgumentException("The output tensor must have the same shape as the input tensors", nameof(y));
+            float* px1 = x1, px2 = x2, py = y;
+
+            // Loop in parallel
+            void Kernel(int i)
+            {
+                int offset = i * l;
+                for (int j = 0; j < l; j++)
+                {
+                    int position = offset + j;
+                    py[position] = px1[position] * px2[position];
+                }
+            }
+            Parallel.For(0, n, Kernel).AssertCompleted();
+        }
+
         /// <summary>
         /// Subtracts two <see cref="Tensor"/> instances, element wise
         /// </summary>
@@ -93,7 +122,6 @@ void Kernel(int i)
             Parallel.For(0, n, Kernel).AssertCompleted();
         }
 
-
         /// <summary>
         /// Compresses a <see cref="Tensor"/> into a row by summing the components column by column
         /// </summary>