Skip to content

Commit a7e494e

Browse files
Sergio0694Sergio0694
authored
Merge pull request #44 from Sergio0694/feature_improved-serialization
Feature improved serialization
2 parents 1785522 + 024f45f commit a7e494e

37 files changed

+948
-929
lines changed

‎NeuralNetwork.NET.Cuda/APIs/CuDnnNetworkLayersDeserializer.cs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
using JetBrains.Annotations;
2+
using NeuralNetworkNET.APIs.Delegates;
3+
using NeuralNetworkNET.APIs.Enums;
4+
using NeuralNetworkNET.APIs.Interfaces;
5+
using NeuralNetworkNET.Cuda.Layers;
6+
using System.IO;
7+
8+
namespace NeuralNetworkNET.APIs
9+
{
10+
/// <summary>
11+
/// A static class that exposes a single deserialization method that can be used to load a saved network using the cuDNN layers
12+
/// </summary>
13+
public static class CuDnnNetworkLayersDeserializer
14+
{
15+
/// <summary>
16+
/// Gets the <see cref="LayerDeserializer"/> instance to load cuDNN network layers
17+
/// </summary>
18+
[PublicAPI]
19+
public static LayerDeserializer Deserializer { get; } = Deserialize;
20+
21+
/// <summary>
22+
/// Deserializes a layer of the given type from the input <see cref="Stream"/>
23+
/// </summary>
24+
/// <param name="stream">The <see cref="Stream"/> to use to load the layer data</param>
25+
/// <param name="type">The type of network layer to return</param>
26+
private static INetworkLayer Deserialize([NotNull] Stream stream, LayerType type)
27+
{
28+
switch (type)
29+
{
30+
case LayerType.FullyConnected: return CuDnnFullyConnectedLayer.Deserialize(stream);
31+
case LayerType.Convolutional: return CuDnnConvolutionalLayer.Deserialize(stream);
32+
case LayerType.Pooling: return CuDnnPoolingLayer.Deserialize(stream);
33+
case LayerType.Softmax: return CuDnnSoftmaxLayer.Deserialize(stream);
34+
default: return null;
35+
}
36+
}
37+
}
38+
}

‎NeuralNetwork.NET.Cuda/Layers/CuDnnConvolutionalLayer.cs

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ public CuDnnConvolutionalLayer(
6767

6868
public CuDnnConvolutionalLayer(
6969
in TensorInfo input, in ConvolutionInfo operation, TensorInfo kernels, TensorInfo output,
70-
[NotNull] float[,] weights, [NotNull] float[] biases, ActivationFunctionType activation)
70+
[NotNull] float[] weights, [NotNull] float[] biases, ActivationFunctionType activation)
7171
: base(input, operation, kernels, output, weights, biases, activation)
7272
=> SetupCuDnnInfo();
7373

@@ -78,7 +78,7 @@ public override unsafe void Forward(in Tensor x, out Tensor z, out Tensor a)
7878
{
7979
fixed (float* pw = Weights)
8080
{
81-
Tensor.Fix(pw, OutputInfo.Channels, KernelInfo.Size, out Tensor wTensor);
81+
Tensor.Reshape(pw, OutputInfo.Channels, KernelInfo.Size, out Tensor wTensor);
8282
using (DeviceMemory<float> z_gpu = DnnInstance.Gpu.AllocateDevice<float>(x.Entities * OutputInfo.Size))
8383
{
8484
// Tensors info setup
@@ -119,7 +119,7 @@ public override unsafe void Backpropagate(in Tensor delta_1, in Tensor z, Activa
119119
{
120120
fixed (float* pw = Weights)
121121
{
122-
Tensor.Fix(pw, OutputInfo.Channels, KernelInfo.Size, out Tensor wTensor);
122+
Tensor.Reshape(pw, OutputInfo.Channels, KernelInfo.Size, out Tensor wTensor);
123123
DnnInstance.GetConvolutionBackwardDataAlgorithm(FilterDescription, OutputDescription, ConvolutionDescription, InputDescription, ConvolutionBwdDataPreference.PREFER_FASTEST, IntPtr.Zero, out ConvolutionBwdDataAlgo algorithm);
124124
DnnInstance.GetConvolutionBackwardDataWorkspaceSize(FilterDescription, OutputDescription, ConvolutionDescription, InputDescription, algorithm, out IntPtr size);
125125
using (DeviceMemory<float> delta_gpu = DnnInstance.Gpu.AllocateDevice<float>(z.Size))
@@ -173,7 +173,30 @@ public override void ComputeGradient(in Tensor a, in Tensor delta, out Tensor dJ
173173

174174
#endregion
175175

176+
#region Misc
177+
176178
/// <inheritdoc/>
177179
public override INetworkLayer Clone() => new CuDnnConvolutionalLayer(InputInfo, OperationInfo, KernelInfo, OutputInfo, Weights.BlockCopy(), Biases.BlockCopy(), ActivationFunctionType);
180+
181+
/// <summary>
182+
/// Tries to deserialize a new <see cref="CuDnnConvolutionalLayer"/> from the input <see cref="System.IO.Stream"/>
183+
/// </summary>
184+
/// <param name="stream">The input <see cref="System.IO.Stream"/> to use to read the layer data</param>
185+
[MustUseReturnValue, CanBeNull]
186+
public new static INetworkLayer Deserialize([NotNull] System.IO.Stream stream)
187+
{
188+
if (!stream.TryRead(out TensorInfo input)) return null;
189+
if (!stream.TryRead(out TensorInfo output)) return null;
190+
if (!stream.TryRead(out ActivationFunctionType activation)) return null;
191+
if (!stream.TryRead(out int wLength)) return null;
192+
float[] weights = stream.ReadUnshuffled(wLength);
193+
if (!stream.TryRead(out int bLength)) return null;
194+
float[] biases = stream.ReadUnshuffled(bLength);
195+
if (!stream.TryRead(out ConvolutionInfo operation)) return null;
196+
if (!stream.TryRead(out TensorInfo kernels)) return null;
197+
return new CuDnnConvolutionalLayer(input, operation, kernels, output, weights, biases, activation);
198+
}
199+
200+
#endregion
178201
}
179202
}

‎NeuralNetwork.NET.Cuda/Layers/CuDnnFullyConnectedLayer.cs

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
using NeuralNetworkNET.Networks.Implementations.Layers;
1010
using NeuralNetworkNET.APIs.Structs;
1111
using NeuralNetworkNET.APIs.Enums;
12+
using NeuralNetworkNET.APIs.Interfaces;
1213

1314
namespace NeuralNetworkNET.Cuda.Layers
1415
{
@@ -20,18 +21,20 @@ internal class CuDnnFullyConnectedLayer : FullyConnectedLayer
2021
[NotNull]
2122
private readonly Dnn DnnInstance = DnnService.Instance;
2223

23-
public CuDnnFullyConnectedLayer(in TensorInfo input, int outputs, ActivationFunctionType activation, WeightsInitializationMode weightsMode, BiasInitializationMode biasMode)
24-
: base(input, outputs, activation, weightsMode, biasMode) { }
24+
public CuDnnFullyConnectedLayer(in TensorInfo input, int neurons, ActivationFunctionType activation, WeightsInitializationMode weightsMode, BiasInitializationMode biasMode)
25+
: base(input, neurons, activation, weightsMode, biasMode) { }
2526

26-
public CuDnnFullyConnectedLayer([NotNull] float[,] weights, [NotNull] float[] biases, ActivationFunctionType activation)
27-
: base(weights, biases, activation) { }
27+
public CuDnnFullyConnectedLayer(in TensorInfo input, int neurons, [NotNull] float[] weights, [NotNull] float[] biases, ActivationFunctionType activation)
28+
: base(input, neurons, weights, biases, activation) { }
29+
30+
#region Implementation
2831

2932
/// <inheritdoc/>
3033
public override unsafe void Forward(in Tensor x, out Tensor z, out Tensor a)
3134
{
3235
fixed (float* pw = Weights)
3336
{
34-
Tensor.Fix(pw, InputInfo.Size, OutputInfo.Size, out Tensor wTensor);
37+
Tensor.Reshape(pw, InputInfo.Size, OutputInfo.Size, out Tensor wTensor);
3538
using (DeviceMemory<float>
3639
x_gpu = DnnInstance.Gpu.AllocateDevice(x),
3740
w_gpu = DnnInstance.Gpu.AllocateDevice(wTensor),
@@ -51,7 +54,7 @@ public override unsafe void Backpropagate(in Tensor delta_1, in Tensor z, Activa
5154
{
5255
fixed (float* pw = Weights)
5356
{
54-
Tensor.Fix(pw, InputInfo.Size, OutputInfo.Size, out Tensor wTensor);
57+
Tensor.Reshape(pw, InputInfo.Size, OutputInfo.Size, out Tensor wTensor);
5558
using (DeviceMemory<float>
5659
delta_1_gpu = DnnInstance.Gpu.AllocateDevice(delta_1),
5760
w_gpu = DnnInstance.Gpu.AllocateDevice(wTensor),
@@ -76,5 +79,24 @@ public override void ComputeGradient(in Tensor a, in Tensor delta, out Tensor dJ
7679
}
7780
delta.CompressVertically(out dJdb); // Doing this on CPU is generally faster than launching the kernels
7881
}
82+
83+
#endregion
84+
85+
/// <summary>
86+
/// Tries to deserialize a new <see cref="CuDnnFullyConnectedLayer"/> from the input <see cref="System.IO.Stream"/>
87+
/// </summary>
88+
/// <param name="stream">The input <see cref="System.IO.Stream"/> to use to read the layer data</param>
89+
[MustUseReturnValue, CanBeNull]
90+
public new static INetworkLayer Deserialize([NotNull] System.IO.Stream stream)
91+
{
92+
if (!stream.TryRead(out TensorInfo input)) return null;
93+
if (!stream.TryRead(out TensorInfo output)) return null;
94+
if (!stream.TryRead(out ActivationFunctionType activation)) return null;
95+
if (!stream.TryRead(out int wLength)) return null;
96+
float[] weights = stream.ReadUnshuffled(wLength);
97+
if (!stream.TryRead(out int bLength)) return null;
98+
float[] biases = stream.ReadUnshuffled(bLength);
99+
return new CuDnnFullyConnectedLayer(input, output.Size, weights, biases, activation);
100+
}
79101
}
80102
}

‎NeuralNetwork.NET.Cuda/Layers/CuDnnPoolingLayer.cs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,5 +70,19 @@ public override void Forward(in Tensor x, out Tensor z, out Tensor a)
7070

7171
/// <inheritdoc/>
7272
public override INetworkLayer Clone() => new CuDnnPoolingLayer(InputInfo, OperationInfo, ActivationFunctionType);
73+
74+
/// <summary>
75+
/// Tries to deserialize a new <see cref="CuDnnPoolingLayer"/> from the input <see cref="System.IO.Stream"/>
76+
/// </summary>
77+
/// <param name="stream">The input <see cref="System.IO.Stream"/> to use to read the layer data</param>
78+
[MustUseReturnValue, CanBeNull]
79+
public new static INetworkLayer Deserialize([NotNull] System.IO.Stream stream)
80+
{
81+
if (!stream.TryRead(out TensorInfo input)) return null;
82+
if (!stream.TryRead(out TensorInfo _)) return null;
83+
if (!stream.TryRead(out ActivationFunctionType activation)) return null;
84+
if (!stream.TryRead(out PoolingInfo operation)) return null;
85+
return new CuDnnPoolingLayer(input, operation, activation);
86+
}
7387
}
7488
}

‎NeuralNetwork.NET.Cuda/Layers/CuDnnSoftmaxLayer.cs

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
using Alea;
22
using Alea.cuDNN;
33
using JetBrains.Annotations;
4+
using NeuralNetworkNET.Extensions;
45
using NeuralNetworkNET.Cuda.Services;
56
using NeuralNetworkNET.Cuda.Extensions;
67
using NeuralNetworkNET.Networks.Implementations.Layers;
78
using NeuralNetworkNET.APIs.Structs;
89
using NeuralNetworkNET.APIs.Enums;
10+
using NeuralNetworkNET.APIs.Interfaces;
11+
using NeuralNetworkNET.Networks.Activations;
12+
using NeuralNetworkNET.Networks.Cost;
913

1014
namespace NeuralNetworkNET.Cuda.Layers
1115
{
@@ -30,7 +34,7 @@ internal sealed class CuDnnSoftmaxLayer : SoftmaxLayer
3034

3135
public CuDnnSoftmaxLayer(in TensorInfo input, int outputs, WeightsInitializationMode weightsMode, BiasInitializationMode biasMode) : base(input, outputs, weightsMode, biasMode) { }
3236

33-
public CuDnnSoftmaxLayer([NotNull] float[,] weights, [NotNull] float[] biases) : base(weights, biases) { }
37+
public CuDnnSoftmaxLayer(in TensorInfo input, int outputs, [NotNull] float[] weights, [NotNull] float[] biases) : base(input, outputs, weights, biases) { }
3438

3539
/// <inheritdoc/>
3640
public override unsafe void Forward(in Tensor x, out Tensor z, out Tensor a)
@@ -40,7 +44,7 @@ public override unsafe void Forward(in Tensor x, out Tensor z, out Tensor a)
4044
// Linear pass
4145
fixed (float* pw = Weights)
4246
{
43-
Tensor.Fix(pw, InputInfo.Size, OutputInfo.Size, out Tensor wTensor);
47+
Tensor.Reshape(pw, InputInfo.Size, OutputInfo.Size, out Tensor wTensor);
4448
using (DeviceMemory<float>
4549
x_gpu = DnnInstance.Gpu.AllocateDevice(x),
4650
w_gpu = DnnInstance.Gpu.AllocateDevice(wTensor),
@@ -60,5 +64,23 @@ public override unsafe void Forward(in Tensor x, out Tensor z, out Tensor a)
6064
}
6165
}
6266
}
67+
68+
/// <summary>
69+
/// Tries to deserialize a new <see cref="CuDnnSoftmaxLayer"/> from the input <see cref="System.IO.Stream"/>
70+
/// </summary>
71+
/// <param name="stream">The input <see cref="Stream"/> to use to read the layer data</param>
72+
[MustUseReturnValue, CanBeNull]
73+
public new static INetworkLayer Deserialize([NotNull] System.IO.Stream stream)
74+
{
75+
if (!stream.TryRead(out TensorInfo input)) return null;
76+
if (!stream.TryRead(out TensorInfo output)) return null;
77+
if (!stream.TryRead(out ActivationFunctionType activation) && activation == ActivationFunctionType.Softmax) return null;
78+
if (!stream.TryRead(out int wLength)) return null;
79+
float[] weights = stream.ReadUnshuffled(wLength);
80+
if (!stream.TryRead(out int bLength)) return null;
81+
float[] biases = stream.ReadUnshuffled(bLength);
82+
if (!stream.TryRead(out CostFunctionType cost) && cost == CostFunctionType.LogLikelyhood) return null;
83+
return new CuDnnSoftmaxLayer(input, output.Size, weights, biases);
84+
}
6385
}
6486
}

‎NeuralNetwork.NET/APIs/Delegates/LayerDeserializer.cs

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
using System.IO;
2+
using JetBrains.Annotations;
3+
using NeuralNetworkNET.APIs.Enums;
4+
using NeuralNetworkNET.APIs.Interfaces;
5+
6+
namespace NeuralNetworkNET.APIs.Delegates
7+
{
8+
/// <summary>
9+
/// A <see cref="delegate"/> that tries to deserialize a network layer from the input <see cref="Stream"/>, assuming the layer is of the given <see cref="LayerType"/>
10+
/// </summary>
11+
/// <param name="stream">The source <see cref="Stream"/> to load data from. If the layer type is not supported, the <see cref="Stream"/> should not be read at all</param>
12+
/// <param name="type">The type of network layer to deserialize from the <see cref="Stream"/></param>
13+
[CanBeNull]
14+
public delegate INetworkLayer LayerDeserializer([NotNull] Stream stream, LayerType type);
15+
}

‎NeuralNetwork.NET/APIs/Enums/LayerType.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
namespace NeuralNetworkNET.APIs.Misc
1+
namespace NeuralNetworkNET.APIs.Enums
22
{
33
/// <summary>
44
/// Indicates the type of a neural network layer (for serialization purposes only)

‎NeuralNetwork.NET/APIs/Enums/TrainingStopReason.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
namespace NeuralNetworkNET.APIs.Misc
1+
namespace NeuralNetworkNET.APIs.Enums
22
{
33
/// <summary>
44
/// Indicates the reason why a network training session has stopped

‎NeuralNetwork.NET/APIs/Interfaces/INetworkLayer.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
using NeuralNetworkNET.APIs.Misc;
1+
using NeuralNetworkNET.APIs.Enums;
22
using NeuralNetworkNET.APIs.Structs;
33
using System;
44

@@ -17,11 +17,11 @@ public interface INetworkLayer : IEquatable<INetworkLayer>, IClonable<INetworkLa
1717
/// <summary>
1818
/// Gets the info on the layer inputs
1919
/// </summary>
20-
TensorInfo InputInfo { get; }
20+
ref readonly TensorInfo InputInfo { get; }
2121

2222
/// <summary>
2323
/// Gets the info on the layer outputs
2424
/// </summary>
25-
TensorInfo OutputInfo { get; }
25+
ref readonly TensorInfo OutputInfo { get; }
2626
}
2727
}

‎NeuralNetwork.NET/APIs/Interfaces/INeuralNetwork.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ public interface INeuralNetwork : IEquatable<INeuralNetwork>, IClonable<INeuralN
9898
/// <summary>
9999
/// Saves the network to the target file
100100
/// </summary>
101-
/// <param name="FileInfo">The <see cref="FileInfo"/> instance for the target file (it may not exist yet)</param>
101+
/// <param name="target">The <see cref="FileInfo"/> instance for the target file (it may not exist yet)</param>
102102
void Save([NotNull] FileInfo target);
103103

104104
/// <summary>

0 commit comments

Comments
 (0)