Skip to content

Commit 054fa90

Browse files
Sergio0694Sergio0694
authored
Merge pull request #45 from Sergio0694/dev
Serialization and other changes
2 parents a6c808d + a7e494e commit 054fa90

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+1489
-1205
lines changed

‎NeuralNetwork.NET.Cuda/APIS/CuDnnNetworkLayers.cs

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,26 +45,27 @@ public static INetworkLayer Softmax(
4545
/// Creates a convolutional layer with the desired number of kernels
4646
/// </summary>
4747
/// <param name="input">The input volume to process</param>
48+
/// <param name="info">The info on the convolution operation to perform</param>
4849
/// <param name="kernel">The volume information of the kernels used in the layer</param>
4950
/// <param name="kernels">The number of convolution kernels to apply to the input volume</param>
5051
/// <param name="activation">The desired activation function to use in the network layer</param>
51-
/// <param name="mode">The desired convolution mode to use</param>
5252
/// <param name="biasMode">Indicates the desired initialization mode to use for the layer bias values</param>
5353
[PublicAPI]
5454
[Pure, NotNull]
5555
public static INetworkLayer Convolutional(
56-
TensorInfo input, (int X, int Y) kernel, int kernels, ActivationFunctionType activation,
57-
ConvolutionMode mode = ConvolutionMode.Convolution,
56+
in TensorInfo input,
57+
in ConvolutionInfo info, (int X, int Y) kernel, int kernels, ActivationFunctionType activation,
5858
BiasInitializationMode biasMode = BiasInitializationMode.Zero)
59-
=> new CuDnnConvolutionalLayer(input, kernel, kernels, activation, mode, biasMode);
59+
=> new CuDnnConvolutionalLayer(input, info, kernel, kernels, activation, biasMode);
6060

6161
/// <summary>
6262
/// Creates a pooling layer with a window of size 2 and a stride of 2
6363
/// </summary>
6464
/// <param name="input">The input volume to pool</param>
65+
/// <param name="info">The info on the pooling operation to perform</param>
6566
/// <param name="activation">The desired activation function to use in the network layer</param>
6667
[PublicAPI]
6768
[Pure, NotNull]
68-
public static INetworkLayer Pooling(TensorInfo input, ActivationFunctionType activation) => new CuDnnPoolingLayer(input, activation);
69+
public static INetworkLayer Pooling(in TensorInfo input, in PoolingInfo info, ActivationFunctionType activation) => new CuDnnPoolingLayer(input, info, activation);
6970
}
7071
}

‎NeuralNetwork.NET.Cuda/APIs/CuDnnNetworkLayersDeserializer.cs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
using JetBrains.Annotations;
2+
using NeuralNetworkNET.APIs.Delegates;
3+
using NeuralNetworkNET.APIs.Enums;
4+
using NeuralNetworkNET.APIs.Interfaces;
5+
using NeuralNetworkNET.Cuda.Layers;
6+
using System.IO;
7+
8+
namespace NeuralNetworkNET.APIs
9+
{
10+
/// <summary>
11+
/// A static class that exposes a single deserialization method that can be used to load a saved network using the cuDNN layers
12+
/// </summary>
13+
public static class CuDnnNetworkLayersDeserializer
14+
{
15+
/// <summary>
16+
/// Gets the <see cref="LayerDeserializer"/> instance to load cuDNN network layers
17+
/// </summary>
18+
[PublicAPI]
19+
public static LayerDeserializer Deserializer { get; } = Deserialize;
20+
21+
/// <summary>
22+
/// Deserializes a layer of the given type from the input <see cref="Stream"/>
23+
/// </summary>
24+
/// <param name="stream">The <see cref="Stream"/> to use to load the layer data</param>
25+
/// <param name="type">The type of network layer to return</param>
26+
private static INetworkLayer Deserialize([NotNull] Stream stream, LayerType type)
27+
{
28+
switch (type)
29+
{
30+
case LayerType.FullyConnected: return CuDnnFullyConnectedLayer.Deserialize(stream);
31+
case LayerType.Convolutional: return CuDnnConvolutionalLayer.Deserialize(stream);
32+
case LayerType.Pooling: return CuDnnPoolingLayer.Deserialize(stream);
33+
case LayerType.Softmax: return CuDnnSoftmaxLayer.Deserialize(stream);
34+
default: return null;
35+
}
36+
}
37+
}
38+
}

‎NeuralNetwork.NET.Cuda/Layers/CuDnnConvolutionalLayer.cs

Lines changed: 44 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
using NeuralNetworkNET.Networks.Implementations.Layers;
1010
using NeuralNetworkNET.APIs.Structs;
1111
using NeuralNetworkNET.APIs.Enums;
12+
using NeuralNetworkNET.Extensions;
13+
using NeuralNetworkNET.APIs.Interfaces;
1214

1315
namespace NeuralNetworkNET.Cuda.Layers
1416
{
@@ -48,35 +50,35 @@ internal sealed class CuDnnConvolutionalLayer : ConvolutionalLayer
4850
/// <summary>
4951
/// Sets the cuDNN fields that will be used during future forward/backwards operations
5052
/// </summary>
51-
/// <param name="mode">The desired convolution mode</param>
52-
private void SetupCuDnnInfo(APIs.Enums.ConvolutionMode mode)
53+
private void SetupCuDnnInfo()
5354
{
54-
ConvolutionDescription.Set2D(0, 0, 1, 1, 1, 1, (Alea.cuDNN.ConvolutionMode)mode);
55+
ConvolutionDescription.Set2D(OperationInfo.VerticalPadding, OperationInfo.HorizontalPadding, OperationInfo.VerticalStride, OperationInfo.HorizontalStride, 1, 1, (Alea.cuDNN.ConvolutionMode)OperationInfo.Mode);
5556
FilterDescription.Set4D(DataType.FLOAT, TensorFormat.CUDNN_TENSOR_NCHW, OutputInfo.Channels, KernelInfo.Channels, KernelInfo.Height, KernelInfo.Width);
5657
BiasDescription.Set4D(DataType.FLOAT, TensorFormat.CUDNN_TENSOR_NCHW, 1, OutputInfo.Channels, 1, 1);
5758
}
5859

5960
#endregion
6061

6162
public CuDnnConvolutionalLayer(
62-
TensorInfo input, (int X, int Y) kernelSize, int kernels,
63-
ActivationFunctionType activation, APIs.Enums.ConvolutionMode mode, BiasInitializationMode biasMode)
64-
: base(input, kernelSize, kernels, activation, biasMode)
65-
=> SetupCuDnnInfo(mode);
63+
in TensorInfo input, in ConvolutionInfo operation, (int X, int Y) kernelSize, int kernels,
64+
ActivationFunctionType activation, BiasInitializationMode biasMode)
65+
: base(input, operation, kernelSize, kernels, activation, biasMode)
66+
=> SetupCuDnnInfo();
6667

6768
public CuDnnConvolutionalLayer(
68-
TensorInfo input, TensorInfo kernels, TensorInfo output,
69-
[NotNull] float[,] weights, [NotNull] float[] biases,
70-
ActivationFunctionType activation, APIs.Enums.ConvolutionMode mode)
71-
: base(input, kernels, output, weights, biases, activation)
72-
=> SetupCuDnnInfo(mode);
69+
in TensorInfo input, in ConvolutionInfo operation, TensorInfo kernels, TensorInfo output,
70+
[NotNull] float[] weights, [NotNull] float[] biases, ActivationFunctionType activation)
71+
: base(input, operation, kernels, output, weights, biases, activation)
72+
=> SetupCuDnnInfo();
73+
74+
#region Implementation
7375

7476
/// <inheritdoc/>
7577
public override unsafe void Forward(in Tensor x, out Tensor z, out Tensor a)
7678
{
7779
fixed (float* pw = Weights)
7880
{
79-
Tensor.Fix(pw, OutputInfo.Channels, KernelInfo.Size, out Tensor wTensor);
81+
Tensor.Reshape(pw, OutputInfo.Channels, KernelInfo.Size, out Tensor wTensor);
8082
using (DeviceMemory<float> z_gpu = DnnInstance.Gpu.AllocateDevice<float>(x.Entities * OutputInfo.Size))
8183
{
8284
// Tensors info setup
@@ -117,7 +119,7 @@ public override unsafe void Backpropagate(in Tensor delta_1, in Tensor z, Activa
117119
{
118120
fixed (float* pw = Weights)
119121
{
120-
Tensor.Fix(pw, OutputInfo.Channels, KernelInfo.Size, out Tensor wTensor);
122+
Tensor.Reshape(pw, OutputInfo.Channels, KernelInfo.Size, out Tensor wTensor);
121123
DnnInstance.GetConvolutionBackwardDataAlgorithm(FilterDescription, OutputDescription, ConvolutionDescription, InputDescription, ConvolutionBwdDataPreference.PREFER_FASTEST, IntPtr.Zero, out ConvolutionBwdDataAlgo algorithm);
122124
DnnInstance.GetConvolutionBackwardDataWorkspaceSize(FilterDescription, OutputDescription, ConvolutionDescription, InputDescription, algorithm, out IntPtr size);
123125
using (DeviceMemory<float> delta_gpu = DnnInstance.Gpu.AllocateDevice<float>(z.Size))
@@ -168,5 +170,33 @@ public override void ComputeGradient(in Tensor a, in Tensor delta, out Tensor dJ
168170
}
169171
}
170172
}
173+
174+
#endregion
175+
176+
#region Misc
177+
178+
/// <inheritdoc/>
179+
public override INetworkLayer Clone() => new CuDnnConvolutionalLayer(InputInfo, OperationInfo, KernelInfo, OutputInfo, Weights.BlockCopy(), Biases.BlockCopy(), ActivationFunctionType);
180+
181+
/// <summary>
182+
/// Tries to deserialize a new <see cref="CuDnnConvolutionalLayer"/> from the input <see cref="System.IO.Stream"/>
183+
/// </summary>
184+
/// <param name="stream">The input <see cref="System.IO.Stream"/> to use to read the layer data</param>
185+
[MustUseReturnValue, CanBeNull]
186+
public new static INetworkLayer Deserialize([NotNull] System.IO.Stream stream)
187+
{
188+
if (!stream.TryRead(out TensorInfo input)) return null;
189+
if (!stream.TryRead(out TensorInfo output)) return null;
190+
if (!stream.TryRead(out ActivationFunctionType activation)) return null;
191+
if (!stream.TryRead(out int wLength)) return null;
192+
float[] weights = stream.ReadUnshuffled(wLength);
193+
if (!stream.TryRead(out int bLength)) return null;
194+
float[] biases = stream.ReadUnshuffled(bLength);
195+
if (!stream.TryRead(out ConvolutionInfo operation)) return null;
196+
if (!stream.TryRead(out TensorInfo kernels)) return null;
197+
return new CuDnnConvolutionalLayer(input, operation, kernels, output, weights, biases, activation);
198+
}
199+
200+
#endregion
171201
}
172202
}

‎NeuralNetwork.NET.Cuda/Layers/CuDnnFullyConnectedLayer.cs

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
using NeuralNetworkNET.Networks.Implementations.Layers;
1010
using NeuralNetworkNET.APIs.Structs;
1111
using NeuralNetworkNET.APIs.Enums;
12+
using NeuralNetworkNET.APIs.Interfaces;
1213

1314
namespace NeuralNetworkNET.Cuda.Layers
1415
{
@@ -20,18 +21,20 @@ internal class CuDnnFullyConnectedLayer : FullyConnectedLayer
2021
[NotNull]
2122
private readonly Dnn DnnInstance = DnnService.Instance;
2223

23-
public CuDnnFullyConnectedLayer(in TensorInfo input, int outputs, ActivationFunctionType activation, WeightsInitializationMode weightsMode, BiasInitializationMode biasMode)
24-
: base(input, outputs, activation, weightsMode, biasMode) { }
24+
public CuDnnFullyConnectedLayer(in TensorInfo input, int neurons, ActivationFunctionType activation, WeightsInitializationMode weightsMode, BiasInitializationMode biasMode)
25+
: base(input, neurons, activation, weightsMode, biasMode) { }
2526

26-
public CuDnnFullyConnectedLayer([NotNull] float[,] weights, [NotNull] float[] biases, ActivationFunctionType activation)
27-
: base(weights, biases, activation) { }
27+
public CuDnnFullyConnectedLayer(in TensorInfo input, int neurons, [NotNull] float[] weights, [NotNull] float[] biases, ActivationFunctionType activation)
28+
: base(input, neurons, weights, biases, activation) { }
29+
30+
#region Implementation
2831

2932
/// <inheritdoc/>
3033
public override unsafe void Forward(in Tensor x, out Tensor z, out Tensor a)
3134
{
3235
fixed (float* pw = Weights)
3336
{
34-
Tensor.Fix(pw, InputInfo.Size, OutputInfo.Size, out Tensor wTensor);
37+
Tensor.Reshape(pw, InputInfo.Size, OutputInfo.Size, out Tensor wTensor);
3538
using (DeviceMemory<float>
3639
x_gpu = DnnInstance.Gpu.AllocateDevice(x),
3740
w_gpu = DnnInstance.Gpu.AllocateDevice(wTensor),
@@ -51,7 +54,7 @@ public override unsafe void Backpropagate(in Tensor delta_1, in Tensor z, Activa
5154
{
5255
fixed (float* pw = Weights)
5356
{
54-
Tensor.Fix(pw, InputInfo.Size, OutputInfo.Size, out Tensor wTensor);
57+
Tensor.Reshape(pw, InputInfo.Size, OutputInfo.Size, out Tensor wTensor);
5558
using (DeviceMemory<float>
5659
delta_1_gpu = DnnInstance.Gpu.AllocateDevice(delta_1),
5760
w_gpu = DnnInstance.Gpu.AllocateDevice(wTensor),
@@ -76,5 +79,24 @@ public override void ComputeGradient(in Tensor a, in Tensor delta, out Tensor dJ
7679
}
7780
delta.CompressVertically(out dJdb); // Doing this on CPU is generally faster than launching the kernels
7881
}
82+
83+
#endregion
84+
85+
/// <summary>
86+
/// Tries to deserialize a new <see cref="CuDnnFullyConnectedLayer"/> from the input <see cref="System.IO.Stream"/>
87+
/// </summary>
88+
/// <param name="stream">The input <see cref="System.IO.Stream"/> to use to read the layer data</param>
89+
[MustUseReturnValue, CanBeNull]
90+
public new static INetworkLayer Deserialize([NotNull] System.IO.Stream stream)
91+
{
92+
if (!stream.TryRead(out TensorInfo input)) return null;
93+
if (!stream.TryRead(out TensorInfo output)) return null;
94+
if (!stream.TryRead(out ActivationFunctionType activation)) return null;
95+
if (!stream.TryRead(out int wLength)) return null;
96+
float[] weights = stream.ReadUnshuffled(wLength);
97+
if (!stream.TryRead(out int bLength)) return null;
98+
float[] biases = stream.ReadUnshuffled(bLength);
99+
return new CuDnnFullyConnectedLayer(input, output.Size, weights, biases, activation);
100+
}
79101
}
80102
}

‎NeuralNetwork.NET.Cuda/Layers/CuDnnPoolingLayer.cs

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,9 @@ internal sealed class CuDnnPoolingLayer : PoolingLayer
4141

4242
#endregion
4343

44-
public CuDnnPoolingLayer(TensorInfo input, ActivationFunctionType activation) : base(input, activation)
44+
public CuDnnPoolingLayer(in TensorInfo input, in PoolingInfo operation, ActivationFunctionType activation) : base(input, operation, activation)
4545
{
46-
PoolingDescription.Set2D(PoolingMode.MAX, NanPropagation.PROPAGATE_NAN, 2, 2, 0, 0, 2, 2);
46+
PoolingDescription.Set2D((PoolingMode)operation.Mode, NanPropagation.PROPAGATE_NAN, operation.WindowHeight, operation.WindowWidth, operation.VerticalPadding, operation.HorizontalPadding, operation.VerticalStride, operation.HorizontalStride);
4747
}
4848

4949
/// <inheritdoc/>
@@ -69,6 +69,20 @@ public override void Forward(in Tensor x, out Tensor z, out Tensor a)
6969
public override void Backpropagate(in Tensor delta_1, in Tensor z, ActivationFunction activationPrime) => z.UpscalePool2x2(delta_1, InputInfo.Channels);
7070

7171
/// <inheritdoc/>
72-
public override INetworkLayer Clone() => new PoolingLayer(InputInfo, ActivationFunctionType);
72+
public override INetworkLayer Clone() => new CuDnnPoolingLayer(InputInfo, OperationInfo, ActivationFunctionType);
73+
74+
/// <summary>
75+
/// Tries to deserialize a new <see cref="CuDnnPoolingLayer"/> from the input <see cref="System.IO.Stream"/>
76+
/// </summary>
77+
/// <param name="stream">The input <see cref="System.IO.Stream"/> to use to read the layer data</param>
78+
[MustUseReturnValue, CanBeNull]
79+
public new static INetworkLayer Deserialize([NotNull] System.IO.Stream stream)
80+
{
81+
if (!stream.TryRead(out TensorInfo input)) return null;
82+
if (!stream.TryRead(out TensorInfo _)) return null;
83+
if (!stream.TryRead(out ActivationFunctionType activation)) return null;
84+
if (!stream.TryRead(out PoolingInfo operation)) return null;
85+
return new CuDnnPoolingLayer(input, operation, activation);
86+
}
7387
}
7488
}

‎NeuralNetwork.NET.Cuda/Layers/CuDnnSoftmaxLayer.cs

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
using Alea;
22
using Alea.cuDNN;
33
using JetBrains.Annotations;
4+
using NeuralNetworkNET.Extensions;
45
using NeuralNetworkNET.Cuda.Services;
56
using NeuralNetworkNET.Cuda.Extensions;
67
using NeuralNetworkNET.Networks.Implementations.Layers;
78
using NeuralNetworkNET.APIs.Structs;
89
using NeuralNetworkNET.APIs.Enums;
10+
using NeuralNetworkNET.APIs.Interfaces;
11+
using NeuralNetworkNET.Networks.Activations;
12+
using NeuralNetworkNET.Networks.Cost;
913

1014
namespace NeuralNetworkNET.Cuda.Layers
1115
{
@@ -30,7 +34,7 @@ internal sealed class CuDnnSoftmaxLayer : SoftmaxLayer
3034

3135
public CuDnnSoftmaxLayer(in TensorInfo input, int outputs, WeightsInitializationMode weightsMode, BiasInitializationMode biasMode) : base(input, outputs, weightsMode, biasMode) { }
3236

33-
public CuDnnSoftmaxLayer([NotNull] float[,] weights, [NotNull] float[] biases) : base(weights, biases) { }
37+
public CuDnnSoftmaxLayer(in TensorInfo input, int outputs, [NotNull] float[] weights, [NotNull] float[] biases) : base(input, outputs, weights, biases) { }
3438

3539
/// <inheritdoc/>
3640
public override unsafe void Forward(in Tensor x, out Tensor z, out Tensor a)
@@ -40,7 +44,7 @@ public override unsafe void Forward(in Tensor x, out Tensor z, out Tensor a)
4044
// Linear pass
4145
fixed (float* pw = Weights)
4246
{
43-
Tensor.Fix(pw, InputInfo.Size, OutputInfo.Size, out Tensor wTensor);
47+
Tensor.Reshape(pw, InputInfo.Size, OutputInfo.Size, out Tensor wTensor);
4448
using (DeviceMemory<float>
4549
x_gpu = DnnInstance.Gpu.AllocateDevice(x),
4650
w_gpu = DnnInstance.Gpu.AllocateDevice(wTensor),
@@ -60,5 +64,23 @@ public override unsafe void Forward(in Tensor x, out Tensor z, out Tensor a)
6064
}
6165
}
6266
}
67+
68+
/// <summary>
69+
/// Tries to deserialize a new <see cref="CuDnnSoftmaxLayer"/> from the input <see cref="System.IO.Stream"/>
70+
/// </summary>
71+
/// <param name="stream">The input <see cref="Stream"/> to use to read the layer data</param>
72+
[MustUseReturnValue, CanBeNull]
73+
public new static INetworkLayer Deserialize([NotNull] System.IO.Stream stream)
74+
{
75+
if (!stream.TryRead(out TensorInfo input)) return null;
76+
if (!stream.TryRead(out TensorInfo output)) return null;
77+
if (!stream.TryRead(out ActivationFunctionType activation) && activation == ActivationFunctionType.Softmax) return null;
78+
if (!stream.TryRead(out int wLength)) return null;
79+
float[] weights = stream.ReadUnshuffled(wLength);
80+
if (!stream.TryRead(out int bLength)) return null;
81+
float[] biases = stream.ReadUnshuffled(bLength);
82+
if (!stream.TryRead(out CostFunctionType cost) && cost == CostFunctionType.LogLikelyhood) return null;
83+
return new CuDnnSoftmaxLayer(input, output.Size, weights, biases);
84+
}
6385
}
6486
}

‎NeuralNetwork.NET/APIs/Delegates/LayerDeserializer.cs

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
using System.IO;
2+
using JetBrains.Annotations;
3+
using NeuralNetworkNET.APIs.Enums;
4+
using NeuralNetworkNET.APIs.Interfaces;
5+
6+
namespace NeuralNetworkNET.APIs.Delegates
7+
{
8+
/// <summary>
9+
/// A <see cref="delegate"/> that tries to deserialize a network layer from the input <see cref="Stream"/>, assuming the layer is of the given <see cref="LayerType"/>
10+
/// </summary>
11+
/// <param name="stream">The source <see cref="Stream"/> to load data from. If the layer type is not supported, the <see cref="Stream"/> should not be read at all</param>
12+
/// <param name="type">The type of network layer to deserialize from the <see cref="Stream"/></param>
13+
[CanBeNull]
14+
public delegate INetworkLayer LayerDeserializer([NotNull] Stream stream, LayerType type);
15+
}

‎NeuralNetwork.NET.Cuda/APIs/Enums/ConvolutionMode.cs renamed to ‎NeuralNetwork.NET/APIs/Enums/ConvolutionMode.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
namespace NeuralNetworkNET.APIs.Enums
22
{
33
/// <summary>
4-
/// A simple wrapper over the <see cref="Alea.cuDNN.ConvolutionMode"/> <see cref="enum"/>
4+
/// A simple <see cref="enum"/> indicating the type of convolution operation to perform
55
/// </summary>
66
public enum ConvolutionMode
77
{

‎NeuralNetwork.NET/APIs/Enums/LayerType.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
namespace NeuralNetworkNET.APIs.Misc
1+
namespace NeuralNetworkNET.APIs.Enums
22
{
33
/// <summary>
44
/// Indicates the type of a neural network layer (for serialization purposes only)

0 commit comments

Comments
 (0)