rewrite mlp and backpropagation for regression

Author: sth4nth

chapter05/mlp.m

@@ -0,0 +1,59 @@
+function [model, L] = mlpReg(X,Y,k,lambda)
+% Train a multilayer perceptron neural network
+% Input:
+%   X: d x n data matrix
+%   Y: p x n response matrix
+%   k: T x 1 vector to specify number of hidden nodes in each layer
+%   lambda: regularization parameter
+% Ouput:
+%   model: model structure
+%   L: loss
+% Written by Mo Chen (sth4nth@gmail.com).
+if nargin < 4
+    lambda = 1e-2;
+end
+eta = 1e-3;
+maxiter = 50000;
+L = inf(1,maxiter);
+
+k = [size(X,1);k(:);size(Y,1)];
+T = numel(k)-1;
+W = cell(T,1);
+b = cell(T,1);
+for t = 1:T
+    W{t} = randn(k(t),k(t+1));
+    b{t} = randn(k(t+1),1);
+end
+R = cell(T,1);
+Z = cell(T+1,1);
+Z{1} = X;
+for iter = 2:maxiter
+%     forward
+    for t = 1:T-1
+        Z{t+1} = tanh(W{t}'*Z{t}+b{t});
+    end
+    Z{T+1} = W{T}'*Z{T}+b{T};
+
+%     loss
+    E = Z{T+1}-Y;     
+    Wn = cellfun(@(x) dot(x(:),x(:)),W);            % |W|^2
+    L(iter) = dot(E(:),E(:))+lambda*sum(Wn);
+
+%     backward
+    R{T} = E;                % delta
+    for t = T-1:-1:1
+        df = 1-Z{t+1}.^2;    % h'(a)
+        R{t} = df.*(W{t+1}*R{t+1});    % delta
+    end
+    
+%     gradient descent
+    for t=1:T
+        dW = Z{t}*R{t}'+lambda*W{t};
+        db = sum(R{t},2);
+        W{t} = W{t}-eta*dW;
+        b{t} = b{t}-eta*db;
+    end
+end
+L = L(1,2:iter);
+model.W = W;
+model.b = b;

chapter05/mlpReg.m

@@ -1,4 +1,4 @@
-function Y = mlpPred(model, X)
+function Y = mlpRegPred(model, X)
 % Multilayer perceptron prediction
 % Input:
 %   model: model structure
@@ -7,7 +7,11 @@
 %   Y: p x n response matrix
 % Written by Mo Chen (sth4nth@gmail.com).
 W = model.W;
-Y = X;
-for l = 1:length(W)
-    Y = sigmoid(W{l}'*Y);
-end
+b = model.b;
+T = length(W);
+Z = cell(T+1,1);
+Z{1} = X;
+for t = 1:T-1
+    Z{t+1} = tanh(W{t}'*Z{t}+b{t});
+end
+Y = W{T}'*Z{T}+b{T};

chapter05/mlpPred.m renamed to chapter05/mlpRegPred.m

@@ -1,9 +1,15 @@
 clear; close all;
-h = [4,5];
-X = [0 0 1 1;0 1 0 1];
-T = [0 1 1 0];
-[model,mse] = mlp(X,T,h);
-plot(mse);
-disp(['T = [' num2str(T) ']']);
-Y = mlpPred(model,X);
-disp(['Y = [' num2str(Y) ']']);
+n = 200;
+x = linspace(0,2*pi,n);
+y = sin(x);
+
+k = [3,4];            % two hidden layers with 3 and 4 hidden nodes
+lambda = 1e-2;
+[model, L] = mlpReg(x,y,k);
+t = mlpRegPred(model,x);
+plot(L);
+figure;
+hold on
+plot(x,y,'.');
+plot(x,t);
+hold off