Added input parameter in backward method

Sergio0694 · Sergio0694 · commit 46d2b64c6b54 · 2017-12-30T23:11:53.000+01:00
diff --git a/NeuralNetwork.NET/Networks/Implementations/NeuralNetwork.cs b/NeuralNetwork.NET/Networks/Implementations/NeuralNetwork.cs
@@ -252,7 +252,7 @@ internal unsafe void Backpropagate(in TrainingBatch batch, float dropout, [NotNu
                      * Perform the sigmoid prime of z(l), the activity on the previous layer
                      * Multiply the previous delta with the transposed weights of the following layer
                      * Compute d(l), the Hadamard product of z'(l) and delta(l + 1) * W(l + 1)T */
-                    _Layers[l + 1].Backpropagate(*deltas[l + 1], zList[l], _Layers[l].ActivationFunctions.ActivationPrime);
+                    _Layers[l + 1].Backpropagate(aList[l], *deltas[l + 1], zList[l], _Layers[l].ActivationFunctions.ActivationPrime);
                     if (!dropoutMasks[l].IsNull) CpuBlas.MultiplyElementwise(zList[l], dropoutMasks[l], zList[l]);
                     deltas[l] = zList + l;
                 }
diff --git a/NeuralNetwork.NET/Networks/Layers/Abstract/NetworkLayerBase.cs b/NeuralNetwork.NET/Networks/Layers/Abstract/NetworkLayerBase.cs
@@ -75,10 +75,11 @@ protected NetworkLayerBase(in TensorInfo input, in TensorInfo output, Activation
         /// <summary>
         /// Backpropagates the error to compute the delta for the inputs of the layer
         /// </summary>
+        /// <param name="x">The input <see cref="Tensor"/> used in the forward pass</param>
         /// <param name="dy">The output error delta</param>
         /// <param name="z">The activity on the inputs of the layer. It will be modified to become the computed delta</param>
         /// <param name="activationPrime">The activation prime function performed by the previous layer</param>
-        public abstract void Backpropagate(in Tensor dy, in Tensor z, ActivationFunction activationPrime);
+        public abstract void Backpropagate(in Tensor x, in Tensor dy, in Tensor z, ActivationFunction activationPrime);
 
         #region Equality check
 
diff --git a/NeuralNetwork.NET/Networks/Layers/Cpu/ConvolutionalLayer.cs b/NeuralNetwork.NET/Networks/Layers/Cpu/ConvolutionalLayer.cs
@@ -94,7 +94,7 @@ public override unsafe void Forward(in Tensor x, out Tensor z, out Tensor a)
         }
 
         /// <inheritdoc/>
-        public override unsafe void Backpropagate(in Tensor dy, in Tensor z, ActivationFunction activationPrime)
+        public override unsafe void Backpropagate(in Tensor x, in Tensor dy, in Tensor z, ActivationFunction activationPrime)
         {
             fixed (float* pw = Weights)
             {
diff --git a/NeuralNetwork.NET/Networks/Layers/Cpu/FullyConnectedLayer.cs b/NeuralNetwork.NET/Networks/Layers/Cpu/FullyConnectedLayer.cs
@@ -48,7 +48,7 @@ public override unsafe void Forward(in Tensor x, out Tensor z, out Tensor a)
         }
 
         /// <inheritdoc/>
-        public override unsafe void Backpropagate(in Tensor dy, in Tensor z, ActivationFunction activationPrime)
+        public override unsafe void Backpropagate(in Tensor x, in Tensor dy, in Tensor z, ActivationFunction activationPrime)
         {
             fixed (float* pw = Weights)
             {
diff --git a/NeuralNetwork.NET/Networks/Layers/Cpu/PoolingLayer.cs b/NeuralNetwork.NET/Networks/Layers/Cpu/PoolingLayer.cs
@@ -48,7 +48,7 @@ public override void Forward(in Tensor x, out Tensor z, out Tensor a)
         }
 
         /// <inheritdoc/>
-        public override void Backpropagate(in Tensor dy, in Tensor z, ActivationFunction activationPrime) => CpuDnn.PoolingBackward(z, InputInfo, dy, z);
+        public override void Backpropagate(in Tensor x, in Tensor dy, in Tensor z, ActivationFunction activationPrime) => CpuDnn.PoolingBackward(x, InputInfo, dy, z);
 
         /// <inheritdoc/>
         public override INetworkLayer Clone() => new PoolingLayer(InputInfo, OperationInfo, ActivationFunctionType);
diff --git a/NeuralNetwork.NET/Networks/Layers/Cuda/CuDnnConvolutionalLayer.cs b/NeuralNetwork.NET/Networks/Layers/Cuda/CuDnnConvolutionalLayer.cs
@@ -108,7 +108,7 @@ public override void Forward(in Tensor x, out Tensor z, out Tensor a)
         }
 
         /// <inheritdoc/>
-        public override void Backpropagate(in Tensor dy, in Tensor z, ActivationFunction activationPrime)
+        public override void Backpropagate(in Tensor x, in Tensor dy, in Tensor z, ActivationFunction activationPrime)
         {
             using (DeviceMemory<float> delta_gpu = DnnInstance.Gpu.AllocateDevice<float>(z.Size))
             {
diff --git a/NeuralNetwork.NET/Networks/Layers/Cuda/CuDnnFullyConnectedLayer.cs b/NeuralNetwork.NET/Networks/Layers/Cuda/CuDnnFullyConnectedLayer.cs
@@ -46,7 +46,7 @@ public override void Forward(in Tensor x, out Tensor z, out Tensor a)
         }
 
         /// <inheritdoc/>
-        public override void Backpropagate(in Tensor dy, in Tensor z, ActivationFunction activationPrime)
+        public override void Backpropagate(in Tensor x, in Tensor dy, in Tensor z, ActivationFunction activationPrime)
         {
             using (DeviceMemory<float>
                 delta_1_gpu = DnnInstance.Gpu.AllocateDevice(dy),
diff --git a/NeuralNetwork.NET/Networks/Layers/Cuda/CuDnnInceptionLayer.cs b/NeuralNetwork.NET/Networks/Layers/Cuda/CuDnnInceptionLayer.cs
@@ -82,9 +82,6 @@ private int Secondary1x1Weights
             get => InputInfo.Channels * OperationInfo.Secondary1x1AfterPoolingConvolutionKernels;
         }
 
-        // A copy of the forward layer inputs
-        private Tensor _Inputs;
-
         // 3x3 reduction 1x1 convolution activity
         private Tensor _3x3Reduce1x1Z;
 
@@ -300,8 +297,6 @@ internal CuDnnInceptionLayer(in TensorInfo input, in InceptionInfo info, [NotNul
         /// <inheritdoc/>
         public override void Forward(in Tensor x, out Tensor z, out Tensor a)
         {
-            _Inputs.TryFree();
-            x.Duplicate(out _Inputs);
             Tensor.New(x.Entities, OutputInfo.Size, out z);
             Tensor.New(x.Entities, OutputInfo.Size, out a);
             using (DeviceMemory<float>
@@ -443,7 +438,7 @@ public override void Forward(in Tensor x, out Tensor z, out Tensor a)
         }
 
         /// <inheritdoc/>
-        public override void Backpropagate(in Tensor dy, in Tensor z, ActivationFunction activationPrime)
+        public override void Backpropagate(in Tensor x, in Tensor dy, in Tensor z, ActivationFunction activationPrime)
         {
             using (DeviceMemory<float> 
                 dx_gpu = DnnInstance.Gpu.AllocateDevice<float>(z.Size),
@@ -534,7 +529,7 @@ public override void Backpropagate(in Tensor dy, in Tensor z, ActivationFunction
 
                     // Pooling backward
                     using (DeviceMemory<float> 
-                        x_gpu = DnnInstance.Gpu.AllocateDevice(_Inputs),
+                        x_gpu = DnnInstance.Gpu.AllocateDevice(x),
                         poolZ_gpu = DnnInstance.Gpu.AllocateDevice(_PoolingZ))
                     {
                         DnnInstance.PoolingBackward(PoolingDescription, 1, PoolingOutputDescription, poolZ_gpu.Ptr, PoolingOutputDescription, pooldy_gpu.Ptr, InputDescription, x_gpu.Ptr, 1, InputDescription, dx_gpu.Ptr); // TODO: finish pooling backward
@@ -733,7 +728,6 @@ void IDisposable.Dispose()
         // Private Dispose method
         private void Dispose()
         {
-            _Inputs.TryFree();
             _3x3Reduce1x1Z.TryFree();
             _3x3Reduce1x1A.TryFree();
             _3x3Reduce1x1Delta.TryFree();
diff --git a/NeuralNetwork.NET/Networks/Layers/Cuda/CuDnnPoolingLayer.cs b/NeuralNetwork.NET/Networks/Layers/Cuda/CuDnnPoolingLayer.cs
@@ -43,9 +43,6 @@ internal sealed class CuDnnPoolingLayer : PoolingLayer, IDisposable
 
         #region Fields
 
-        // A copy of the layer inputs
-        private Tensor _X;
-
         // A copy of the layer output activity
         private Tensor _Z;
 
@@ -59,8 +56,6 @@ public CuDnnPoolingLayer(in TensorInfo input, in PoolingInfo operation, Activati
         /// <inheritdoc/>
         public override void Forward(in Tensor x, out Tensor z, out Tensor a)
         {
-            _X.TryFree();
-            x.Duplicate(out _X);
             using (DeviceMemory<float>
                 x_gpu = DnnInstance.Gpu.AllocateDevice(x),
                 z_gpu = DnnInstance.Gpu.AllocateDevice<float>(x.Entities * OutputInfo.Size))
@@ -80,12 +75,12 @@ public override void Forward(in Tensor x, out Tensor z, out Tensor a)
         }
 
         /// <inheritdoc/>
-        public override void Backpropagate(in Tensor dy, in Tensor z, ActivationFunction activationPrime)
+        public override void Backpropagate(in Tensor x, in Tensor dy, in Tensor z, ActivationFunction activationPrime)
         {
             using (DeviceMemory<float> dx_gpu = DnnInstance.Gpu.AllocateDevice<float>(z.Size))
             {
                 using (DeviceMemory<float>
-                    x_gpu = DnnInstance.Gpu.AllocateDevice(_X),
+                    x_gpu = DnnInstance.Gpu.AllocateDevice(x),
                     y_gpu = DnnInstance.Gpu.AllocateDevice(_Z),
                     dy_gpu = DnnInstance.Gpu.AllocateDevice(dy))
                 {
@@ -130,7 +125,6 @@ void IDisposable.Dispose()
         // Private Dispose method
         private void Dispose()
         {
-            _X.TryFree();
             _Z.TryFree();
         }
 
diff --git a/Unit/NeuralNetwork.NET.Cuda.Unit/CuDnnInceptionLayerTest.cs b/Unit/NeuralNetwork.NET.Cuda.Unit/CuDnnInceptionLayerTest.cs
@@ -49,8 +49,8 @@ public unsafe void Inception1x1()
                 Tensor.New(xTensor.Entities, xTensor.Length, out Tensor z1);
                 KerasWeightsProvider.FillWithHeEtAlUniform(z1, 10);
                 z1.Duplicate(out Tensor z2);
-                conv.Backpropagate(aConv, z1, ActivationFunctions.ReLUPrime);
-                inception.Backpropagate(aInc, z2, ActivationFunctions.ReLUPrime);
+                conv.Backpropagate(Tensor.Null, aConv, z1, ActivationFunctions.ReLUPrime);
+                inception.Backpropagate(xTensor, aInc, z2, ActivationFunctions.ReLUPrime);
                 Assert.IsTrue(z1.ContentEquals(z2));
 
                 // Gradient
@@ -119,9 +119,9 @@ public unsafe void Inception3x3Pipeline()
                 Tensor.New(xTensor.Entities, xTensor.Length, out Tensor z1);
                 KerasWeightsProvider.FillWithHeEtAlUniform(z1, 10);
                 z1.Duplicate(out Tensor z2);
-                conv2.Backpropagate(aConv, zTemp, conv1.ActivationFunctions.ActivationPrime);
-                conv1.Backpropagate(zTemp, z1, ActivationFunctions.ReLUPrime);
-                inception.Backpropagate(aInc, z2, ActivationFunctions.ReLUPrime);
+                conv2.Backpropagate(Tensor.Null, aConv, zTemp, conv1.ActivationFunctions.ActivationPrime);
+                conv1.Backpropagate(Tensor.Null, zTemp, z1, ActivationFunctions.ReLUPrime);
+                inception.Backpropagate(xTensor, aInc, z2, ActivationFunctions.ReLUPrime);
                 Assert.IsTrue(z1.ContentEquals(z2));
 
                 // Gradient
@@ -188,9 +188,9 @@ public unsafe void Inception5x5Pipeline()
                 Tensor.New(xTensor.Entities, xTensor.Length, out Tensor z1);
                 KerasWeightsProvider.FillWithHeEtAlUniform(z1, 10);
                 z1.Duplicate(out Tensor z2);
-                conv2.Backpropagate(aConv, zTemp, conv1.ActivationFunctions.ActivationPrime);
-                conv1.Backpropagate(zTemp, z1, ActivationFunctions.ReLUPrime);
-                inception.Backpropagate(aInc, z2, ActivationFunctions.ReLUPrime);
+                conv2.Backpropagate(Tensor.Null, aConv, zTemp, conv1.ActivationFunctions.ActivationPrime);
+                conv1.Backpropagate(Tensor.Null, zTemp, z1, ActivationFunctions.ReLUPrime);
+                inception.Backpropagate(xTensor, aInc, z2, ActivationFunctions.ReLUPrime);
                 Assert.IsTrue(z1.ContentEquals(z2));
 
                 // Gradient
@@ -253,9 +253,9 @@ public unsafe void InceptionPoolPipeline()
                 Tensor.New(xTensor.Entities, xTensor.Length, out Tensor z1);
                 KerasWeightsProvider.FillWithHeEtAlUniform(z1, 10);
                 z1.Duplicate(out Tensor z2);
-                conv.Backpropagate(aConv, zTemp, pool.ActivationFunctions.ActivationPrime);
-                pool.Backpropagate(zTemp, z1, ActivationFunctions.ReLUPrime);
-                inception.Backpropagate(aInc, z2, ActivationFunctions.ReLUPrime);
+                conv.Backpropagate(Tensor.Null, aConv, zTemp, pool.ActivationFunctions.ActivationPrime);
+                pool.Backpropagate(xTensor, zTemp, z1, ActivationFunctions.ReLUPrime);
+                inception.Backpropagate(xTensor, aInc, z2, ActivationFunctions.ReLUPrime);
                 Assert.IsTrue(z1.ContentEquals(z2));
 
                 // Gradient
diff --git a/Unit/NeuralNetwork.NET.Cuda.Unit/CuDnnLayersTest.cs b/Unit/NeuralNetwork.NET.Cuda.Unit/CuDnnLayersTest.cs
@@ -43,8 +43,8 @@ private static unsafe void TestBackward(NetworkLayerBase cpu, NetworkLayerBase g
                 Tensor.Reshape(pd_1, delta_1.GetLength(0), delta_1.GetLength(1), out Tensor delta_1t);
                 Tensor.Reshape(pz, z.GetLength(0), z.GetLength(1), out Tensor zt);
                 zt.Duplicate(out Tensor zt2);
-                cpu.Backpropagate(delta_1t, zt, ActivationFunctions.LeCunTanhPrime);
-                gpu.Backpropagate(delta_1t, zt2, ActivationFunctions.LeCunTanhPrime);
+                cpu.Backpropagate(Tensor.Null, delta_1t, zt, ActivationFunctions.LeCunTanhPrime);
+                gpu.Backpropagate(Tensor.Null, delta_1t, zt2, ActivationFunctions.LeCunTanhPrime);
                 Assert.IsTrue(zt.ContentEquals(zt2));
             }
         }
@@ -246,17 +246,18 @@ public unsafe void PoolingBackward()
                 gpu = new CuDnnPoolingLayer(cpu.InputInfo, PoolingInfo.Default, ActivationFunctionType.LeakyReLU);
             gpu.Forward(x, out Tensor z, out Tensor a);
             a.Free();
+            x.Duplicate(out Tensor x1);
             x.Duplicate(out Tensor x2);
             Tensor.New(z.Entities, z.Length, out Tensor delta);
             KerasWeightsProvider.FillWithHeEtAlUniform(delta, 10);
 
             // Backward
-            cpu.Backpropagate(delta, x, ActivationFunctions.LeakyReLUPrime);
-            gpu.Backpropagate(delta, x2, ActivationFunctions.LeakyReLUPrime);
+            cpu.Backpropagate(x, delta, x1, ActivationFunctions.LeakyReLUPrime);
+            gpu.Backpropagate(x, delta, x2, ActivationFunctions.LeakyReLUPrime);
             bool valid = true;
-            float* px = (float*)x.Ptr.ToPointer(), px2 = (float*)x2.Ptr.ToPointer();
+            float* px = (float*)x1.Ptr.ToPointer(), px2 = (float*)x2.Ptr.ToPointer();
             int count = 0;
-            for (int i = 0; i < x.Size; i++)
+            for (int i = 0; i < x1.Size; i++)
             {
                 if (px[i].EqualsWithDelta(px2[i], 1e-5f)) continue;
                 if (px[i].EqualsWithDelta(px2[i] * 100f, 1e-5f)) count++;   // The cuDNN pooling backwards method returns a value scaled by 0.01 from time to time for some reason (less than 2% anyways)
@@ -266,8 +267,9 @@ public unsafe void PoolingBackward()
                     break;
                 }
             }
-            Assert.IsTrue(valid && count * 100f / x.Size < 2);
+            Assert.IsTrue(valid && count * 100f / x1.Size < 2);
             x.Free();
+            x1.Free();
             x2.Free();
             z.Free();
             delta.Free();

Original file line number	Diff line number	Diff line change
`@@ -252,7 +252,7 @@ internal unsafe void Backpropagate(in TrainingBatch batch, float dropout, [NotNu`
`252`	`252`	`* Perform the sigmoid prime of z(l), the activity on the previous layer`
`253`	`253`	`* Multiply the previous delta with the transposed weights of the following layer`
`254`	`254`	`* Compute d(l), the Hadamard product of z'(l) and delta(l + 1) * W(l + 1)T */`
`255`		`- _Layers[l + 1].Backpropagate(*deltas[l + 1], zList[l], _Layers[l].ActivationFunctions.ActivationPrime);`
	`255`	`+ _Layers[l + 1].Backpropagate(aList[l], *deltas[l + 1], zList[l], _Layers[l].ActivationFunctions.ActivationPrime);`
`256`	`256`	`if (!dropoutMasks[l].IsNull) CpuBlas.MultiplyElementwise(zList[l], dropoutMasks[l], zList[l]);`
`257`	`257`	`deltas[l] = zList + l;`
`258`	`258`	`}`
Original file line number	Diff line number	Diff line change
`@@ -94,7 +94,7 @@ public override unsafe void Forward(in Tensor x, out Tensor z, out Tensor a)`
`94`	`94`	`}`
`95`	`95`
`96`	`96`	`/// <inheritdoc/>`
`97`		`- public override unsafe void Backpropagate(in Tensor dy, in Tensor z, ActivationFunction activationPrime)`
	`97`	`+ public override unsafe void Backpropagate(in Tensor x, in Tensor dy, in Tensor z, ActivationFunction activationPrime)`
`98`	`98`	`{`
`99`	`99`	`fixed (float* pw = Weights)`
`100`	`100`	`{`
Original file line number	Diff line number	Diff line change
`@@ -48,7 +48,7 @@ public override unsafe void Forward(in Tensor x, out Tensor z, out Tensor a)`
`48`	`48`	`}`
`49`	`49`
`50`	`50`	`/// <inheritdoc/>`
`51`		`- public override unsafe void Backpropagate(in Tensor dy, in Tensor z, ActivationFunction activationPrime)`
	`51`	`+ public override unsafe void Backpropagate(in Tensor x, in Tensor dy, in Tensor z, ActivationFunction activationPrime)`
`52`	`52`	`{`
`53`	`53`	`fixed (float* pw = Weights)`
`54`	`54`	`{`
Original file line number	Diff line number	Diff line change
`@@ -48,7 +48,7 @@ public override void Forward(in Tensor x, out Tensor z, out Tensor a)`
`48`	`48`	`}`
`49`	`49`
`50`	`50`	`/// <inheritdoc/>`
`51`		`- public override void Backpropagate(in Tensor dy, in Tensor z, ActivationFunction activationPrime) => CpuDnn.PoolingBackward(z, InputInfo, dy, z);`
	`51`	`+ public override void Backpropagate(in Tensor x, in Tensor dy, in Tensor z, ActivationFunction activationPrime) => CpuDnn.PoolingBackward(x, InputInfo, dy, z);`
`52`	`52`
`53`	`53`	`/// <inheritdoc/>`
`54`	`54`	`public override INetworkLayer Clone() => new PoolingLayer(InputInfo, OperationInfo, ActivationFunctionType);`
Original file line number	Diff line number	Diff line change
`@@ -108,7 +108,7 @@ public override void Forward(in Tensor x, out Tensor z, out Tensor a)`
`108`	`108`	`}`
`109`	`109`
`110`	`110`	`/// <inheritdoc/>`
`111`		`- public override void Backpropagate(in Tensor dy, in Tensor z, ActivationFunction activationPrime)`
	`111`	`+ public override void Backpropagate(in Tensor x, in Tensor dy, in Tensor z, ActivationFunction activationPrime)`
`112`	`112`	`{`
`113`	`113`	`using (DeviceMemory<float> delta_gpu = DnnInstance.Gpu.AllocateDevice<float>(z.Size))`
`114`	`114`	`{`
Original file line number	Diff line number	Diff line change
`@@ -46,7 +46,7 @@ public override void Forward(in Tensor x, out Tensor z, out Tensor a)`
`46`	`46`	`}`
`47`	`47`
`48`	`48`	`/// <inheritdoc/>`
`49`		`- public override void Backpropagate(in Tensor dy, in Tensor z, ActivationFunction activationPrime)`
	`49`	`+ public override void Backpropagate(in Tensor x, in Tensor dy, in Tensor z, ActivationFunction activationPrime)`
`50`	`50`	`{`
`51`	`51`	`using (DeviceMemory<float>`
`52`	`52`	`delta_1_gpu = DnnInstance.Gpu.AllocateDevice(dy),`
Original file line number	Diff line number	Diff line change
`@@ -43,9 +43,6 @@ internal sealed class CuDnnPoolingLayer : PoolingLayer, IDisposable`
`43`	`43`
`44`	`44`	`#region Fields`
`45`	`45`
`46`		`- // A copy of the layer inputs`
`47`		`- private Tensor _X;`
`48`		`-`
`49`	`46`	`// A copy of the layer output activity`
`50`	`47`	`private Tensor _Z;`
`51`	`48`
`@@ -59,8 +56,6 @@ public CuDnnPoolingLayer(in TensorInfo input, in PoolingInfo operation, Activati`
`59`	`56`	`/// <inheritdoc/>`
`60`	`57`	`public override void Forward(in Tensor x, out Tensor z, out Tensor a)`
`61`	`58`	`{`
`62`		`- _X.TryFree();`
`63`		`- x.Duplicate(out _X);`
`64`	`59`	`using (DeviceMemory<float>`
`65`	`60`	`x_gpu = DnnInstance.Gpu.AllocateDevice(x),`
`66`	`61`	`z_gpu = DnnInstance.Gpu.AllocateDevice<float>(x.Entities * OutputInfo.Size))`
`@@ -80,12 +75,12 @@ public override void Forward(in Tensor x, out Tensor z, out Tensor a)`
`80`	`75`	`}`
`81`	`76`
`82`	`77`	`/// <inheritdoc/>`
`83`		`- public override void Backpropagate(in Tensor dy, in Tensor z, ActivationFunction activationPrime)`
	`78`	`+ public override void Backpropagate(in Tensor x, in Tensor dy, in Tensor z, ActivationFunction activationPrime)`
`84`	`79`	`{`
`85`	`80`	`using (DeviceMemory<float> dx_gpu = DnnInstance.Gpu.AllocateDevice<float>(z.Size))`
`86`	`81`	`{`
`87`	`82`	`using (DeviceMemory<float>`
`88`		`- x_gpu = DnnInstance.Gpu.AllocateDevice(_X),`
	`83`	`+ x_gpu = DnnInstance.Gpu.AllocateDevice(x),`
`89`	`84`	`y_gpu = DnnInstance.Gpu.AllocateDevice(_Z),`
`90`	`85`	`dy_gpu = DnnInstance.Gpu.AllocateDevice(dy))`
`91`	`86`	`{`
`@@ -130,7 +125,6 @@ void IDisposable.Dispose()`
`130`	`125`	`// Private Dispose method`
`131`	`126`	`private void Dispose()`
`132`	`127`	`{`
`133`		`- _X.TryFree();`
`134`	`128`	`_Z.TryFree();`
`135`	`129`	`}`
`136`	`130`