Skip to content

Commit faf3af7

Browse files
Sergio0694Sergio0694
committed
Added fully connected CpuDnn methods (WIP)
1 parent b858921 commit faf3af7

File tree

3 files changed

+180
-2
lines changed

3 files changed

+180
-2
lines changed

‎NeuralNetwork.NET/APIs/Structs/Tensor.cs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,14 @@ public void Reshape(int n, int chw, out Tensor tensor)
226226
[MethodImpl(MethodImplOptions.AggressiveInlining)]
227227
public bool MatchShape(in Tensor tensor) => Entities == tensor.Entities && Length == tensor.Length;
228228

229+
/// <summary>
230+
/// Checks whether or not the current instance has the same shape as the input arguments
231+
/// </summary>
232+
/// <param name="entities">The expected number of entities</param>
233+
/// <param name="length">The expected length of each entity</param>
234+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
235+
public bool MatchShape(int entities, int length) => Entities == entities && Length == length;
236+
229237
/// <summary>
230238
/// Frees the memory associated with the current instance
231239
/// </summary>

‎NeuralNetwork.NET/cpuDNN/CpuBlas.cs

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
using System;
2+
using System.Threading.Tasks;
3+
using NeuralNetworkNET.APIs.Structs;
4+
using NeuralNetworkNET.Extensions;
5+
6+
namespace NeuralNetworkNET.cpuDNN
7+
{
8+
public static class CpuBlas
9+
{
10+
/// <summary>
11+
/// Transposes the input <see cref="Tensor"/>
12+
/// </summary>
13+
/// <param name="x">The <see cref="Tensor"/> to transpose</param>
14+
/// <param name="y">The output <see cref="Tensor"/></param>
15+
public static unsafe void Transpose(in this Tensor x, in Tensor y)
16+
{
17+
// Setup
18+
if (!y.MatchShape(x.Length, x.Entities)) throw new ArgumentException("The output tensor doesn't have the right shape");
19+
int n = x.Entities, l = x.Length;
20+
float* px = x, py = y;
21+
22+
// Execute the transposition in parallel
23+
void Kernel(int i)
24+
{
25+
int offset = i * l;
26+
for (int j = 0; j < l; j++)
27+
py[j * n + i] = px[offset + j];
28+
}
29+
Parallel.For(0, n, Kernel).AssertCompleted();
30+
}
31+
32+
/// <summary>
33+
/// Performs the multiplication between two matrices
34+
/// </summary>
35+
/// <param name="m1">The first matrix to multiply</param>
36+
/// <param name="m2">The second matrix to multiply</param>
37+
/// <param name="result">The resulting matrix</param>
38+
public static unsafe void Multiply(in this Tensor x1, in Tensor x2, in Tensor y)
39+
{
40+
// Initialize the parameters and the result matrix
41+
if (x1.Length != x2.Entities) throw new ArgumentOutOfRangeException("Invalid matrices sizes");
42+
if (!y.MatchShape(x1.Entities, x2.Length)) throw new ArgumentException("The output tensor doesn't have the right shape", nameof(y));
43+
int
44+
n = x1.Entities,
45+
l = x1.Length,
46+
k = x2.Length;
47+
float* px1 = x1, px2 = x2, py = y;
48+
49+
// Execute the multiplication in parallel
50+
void Kernel(int i)
51+
{
52+
int i1 = i * l;
53+
for (int j = 0; j < k; j++)
54+
{
55+
// Perform the multiplication
56+
int i2 = j;
57+
float res = 0;
58+
for (int q = 0; q < l; q++, i2 += k)
59+
{
60+
res += px1[i1 + q] * px2[i2];
61+
}
62+
py[i * k + j] = res;
63+
}
64+
}
65+
Parallel.For(0, n, Kernel).AssertCompleted();
66+
}
67+
}
68+
}

‎NeuralNetwork.NET/cpuDNN/CpuDnn{Main}.cs

Lines changed: 104 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@ void Kernel(int i)
3434
public static unsafe void ActivationBackward(in Tensor x, in Tensor y, [NotNull] ActivationFunction f_, in Tensor dx)
3535
{
3636
// Check
37-
if (!y.MatchShape(x)) throw new ArgumentException(nameof(y), "The input tensors must have the same shape");
38-
if (!dx.MatchShape(x)) throw new ArgumentException(nameof(y), "The output tensor must have the same shape as the input");
37+
if (!y.MatchShape(x)) throw new ArgumentException("The input tensors must have the same shape", nameof(y));
38+
if (!dx.MatchShape(x)) throw new ArgumentException("The output tensor must have the same shape as the input", nameof(y));
3939
int
4040
n = x.Entities,
4141
l = x.Length;
@@ -55,5 +55,107 @@ void Kernel(int i)
5555
}
5656

5757
#endregion
58+
59+
#region Fully connected
60+
61+
public static unsafe void FullyConnectedForward(in Tensor x, in Tensor w, in Tensor b, in Tensor y)
62+
{
63+
// Initialize the parameters and the result matrix
64+
if (x.Length != w.Entities) throw new ArgumentOutOfRangeException("Invalid tensors shapes");
65+
if (!b.MatchShape(1, w.Length)) throw new ArgumentException("Invalid biases shape", nameof(b));
66+
if (!y.MatchShape(x.Entities, w.Length)) throw new ArgumentException("The output tensor doesn't have the right shape", nameof(y));
67+
int
68+
h = x.Entities,
69+
l = x.Length,
70+
k = w.Length;
71+
float* pdy = y, px = x, pw = w, pb = b;
72+
73+
// Execute the multiplication in parallel
74+
void Kernel(int i)
75+
{
76+
int offset = i * l;
77+
for (int j = 0; j < k; j++)
78+
{
79+
// Perform the multiplication
80+
int start = j;
81+
float res = 0;
82+
for (int q = 0; q < l; q++, start += k)
83+
{
84+
res += px[offset + q] * pw[start];
85+
}
86+
pdy[i * k + j] = res + pb[j]; // Sum the input vector to each component
87+
}
88+
}
89+
Parallel.For(0, h, Kernel).AssertCompleted();
90+
}
91+
92+
public static unsafe void FullyConnectedBackwardData(in Tensor x, in Tensor w, in Tensor dy, [NotNull] ActivationFunction f_, in Tensor dx)
93+
{
94+
if (w.Entities != x.Length) throw new ArgumentException("The weights tensor doesn't have a valid shape", nameof(w));
95+
if (!x.MatchShape(dy.Entities, w.Entities)) throw new ArgumentException("The input tensor doesn't have the right shape", nameof(x));
96+
if (!dx.MatchShape(x)) throw new ArgumentException("The output tensor doesn't have the right shape", nameof(dx));
97+
Tensor.New(w.Length, w.Entities, out Tensor wt);
98+
CpuBlas.Transpose(w, wt);
99+
100+
// Initialize the parameters and the result matrix
101+
int
102+
h = dy.Entities,
103+
l = dy.Length,
104+
k = wt.Length;
105+
float* pdx = dx, px = x, pdy = dy, pwt = wt;
106+
107+
// Execute the multiplication in parallel
108+
void Kernel(int i)
109+
{
110+
int i1 = i * l;
111+
for (int j = 0; j < k; j++)
112+
{
113+
// Perform the multiplication
114+
int i2 = j;
115+
float res = 0;
116+
for (int q = 0; q < l; q++, i2 += k)
117+
{
118+
res += pdy[i1 + q] * pwt[i2];
119+
}
120+
121+
// res has now the matrix multiplication result for position [i, j]
122+
int zIndex = i * k + j;
123+
pdx[zIndex] = f_(px[zIndex]) * res;
124+
}
125+
}
126+
Parallel.For(0, h, Kernel).AssertCompleted();
127+
wt.Free();
128+
}
129+
130+
public static void FullyConnectedBackwardFilter(in Tensor x, in Tensor dy, in Tensor dw)
131+
{
132+
if (x.Entities != dy.Entities) throw new ArgumentException("The input tensor doesn't match the number of samples from the delta", nameof(x));
133+
Tensor.New(x.Length, x.Entities, out Tensor xt);
134+
CpuBlas.Transpose(x, xt);
135+
CpuBlas.Multiply(xt, dy, dw);
136+
xt.Free();
137+
}
138+
139+
public static unsafe void FullyConnectedBackwardBias(in Tensor dy, in Tensor db)
140+
{
141+
// Preliminary checks and declarations
142+
if (!db.MatchShape(1, dy.Length)) throw new ArgumentException("The output tensor doesn't have the right shape", nameof(db));
143+
int
144+
n = dy.Entities,
145+
l = dy.Length;
146+
float* pdy = dy, pdb = db;
147+
148+
// Compress the matrix
149+
void Kernel(int j)
150+
{
151+
float sum = 0;
152+
for (int i = 0; i < n; i++)
153+
sum += pdb[i * l + j];
154+
pdy[j] = sum;
155+
}
156+
Parallel.For(0, l, Kernel).AssertCompleted();
157+
}
158+
159+
#endregion
58160
}
59161
}

0 commit comments

Comments
 (0)