%%% learns logistic regression weights using the NN toolbox - with graphs
%%% *************************************************************
%%% Milos Hauskrecht
%%% CS2750 Machine Learning, University of Pittsburgh
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%% load the data from the train and test - assume they are normalized
load pima_train.txt;
load pima_test.txt;
tr_data = pima_train;
test_data = pima_test;

data_col= size(tr_data,2)
n_features = data_col - 1

%%% create x
x = tr_data(:,1:n_features);
%% create y vector
y=tr_data(:,data_col);
 
%% builds x for the test set
x_test = test_data(:,1:n_features);
%% builds y vector for the test set
y_test=test_data(:,data_col);

%%% builds a one layer neural network with logistic function output
%%% to be trained with the gradient method
%%% option traingd - gradient descent
%%         traingdm - gradient descent with the momentum
%%%variable learning rate backpropagation, traingda; 
%%% resilient backpropagation trainrp
%%% conjugate gradient (traincgf, traincgp, traincgb, trainscg),
%%% quasi-Newton (trainbfg, trainoss), and Levenberg-Marquardt (trainlm).

net=newff(minmax(x'),[1],{'logsig'},'traingd');

%% initialize graphing constants
graph_step=[];
graph_train=[];
graph_test=[];
pause_step=0.2;

%% sets the number of the NN model
%% see the NN toolbox documentation
net.trainParam.epochs = 50;
net.trainParam.show = 10;
net.trainParam.max_fail=1;

TV.P=x_test';
TV.T=y_test';

%%% training of the neural net
%% supplying the test set allows to see curves for train and test LMS errors
step=2000
graph_step=[graph_step 0];
graph_train=[graph_train mean_misclass_error(round(sim(net,x')),y')];
graph_test=[graph_test mean_misclass_error(round(sim(net,x_test')),y_test')];
for i=1:40
    [net, tr] = train(net,x',y',[],[],[],TV);
    graph_step=[graph_step i*net.trainParam.epochs];
    graph_train=[graph_train mean_misclass_error(round(sim(net,x')),y')];
    graph_test=[graph_test mean_misclass_error(round(sim(net,x_test')),y_test')];
end 

%%% plot mean squared errors
        figure;
        plot(graph_step,graph_train,'-b');
        hold on;
        plot(graph_step,graph_test,'--r');
        xlabel('Step');
        ylabel('Average misclassification error');
        title('Single layer network learning (Mean errors)') %title
        legend('Train set','Test set') %label of the graph
        pause(pause_step);
        hold off;
        
        
%% simulates learned network on inputs in x
%% that is, it computes outputs for all inputs in x and current set of weights
res=sim(net,x');

%%% 'Mean training error (logistic)'
error = mean_misclass_error(round(sim(net,x')),y'); %%% mean_misclass_error(round(res'),y);

%% 'Mean training error (logistic)'
error = mean_misclass_error(round(sim(net,x_test')),y_test'); %%% mean_misclass_error(round(res'),y);