001 package com.croftsoft.apps.insight;
002
003 import java.applet.Applet;
004 import java.awt.Graphics;
005 import java.awt.*;
006 import java.awt.Color;
007 import java.awt.Point;
008 import java.awt.Rectangle;
009
010 public class Backprop_Net {
011 //////////////////////////////////////////////////////////////////////
012 //////////////////////////////////////////////////////////////////////
013
014 double learning_rate = 0.1;
015 double momentum_constant = 0.9;
016 int layers; // synaptic weight layers
017 boolean using_bias_inputs;
018 Matrix_Array inputs;
019 Matrix_Array weights;
020 Matrix_Array weighted_sums;
021 Matrix_Array outputs;
022 Matrix_Array local_gradients;
023 Matrix_Array sum_weighted_deltas;
024 Matrix_Array weights_delta;
025 Matrix_Array weights_momentum;
026
027 //////////////////////////////////////////////////////////////////////
028 //////////////////////////////////////////////////////////////////////
029
030 public Backprop_Net (
031 int layer0_inputs,
032 int [ ] neurons_per_layer,
033 boolean add_bias_inputs,
034 boolean randomize_initial_weights ) {
035 //////////////////////////////////////////////////////////////////////
036 // Constructor method.
037 // The first layer, layer 0, is NOT an input layer.
038 // Should return exception if number of layers < 1.
039 //////////////////////////////////////////////////////////////////////
040 int bias = add_bias_inputs ? 1 : 0;
041 //////////////////////////////////////////////////////////////////////
042 this.layers = neurons_per_layer.length;
043 this.using_bias_inputs = add_bias_inputs;
044 this.inputs = new Matrix_Array ( layers );
045 this.weights = new Matrix_Array ( layers );
046 this.weighted_sums = new Matrix_Array ( layers );
047 this.outputs = new Matrix_Array ( layers );
048 this.local_gradients = new Matrix_Array ( layers );
049 this.sum_weighted_deltas = new Matrix_Array ( layers );
050 this.weights_delta = new Matrix_Array ( layers );
051 this.weights_momentum = new Matrix_Array ( layers );
052 for ( int index_layer = 0;
053 index_layer < layers;
054 index_layer++ ) {
055 if ( index_layer == 0 ) {
056 inputs.m [ 0 ] = new Matrix ( layer0_inputs + bias, 1 );
057 if ( add_bias_inputs ) {
058 inputs.m [ 0 ].data [ 0 ] [ 0 ] = 1.0;
059 }
060 weights.m [ 0 ] = new Matrix (
061 layer0_inputs + bias, neurons_per_layer [ 0 ] );
062 weighted_sums.m [ 0 ]
063 = new Matrix ( neurons_per_layer [ 0 ], 1 );
064 outputs.m [ 0 ]
065 = new Matrix ( neurons_per_layer [ 0 ], 1 );
066 local_gradients.m [ 0 ]
067 = new Matrix ( neurons_per_layer [ 0 ], 1 );
068 sum_weighted_deltas.m [ 0 ]
069 = new Matrix ( neurons_per_layer [ 0 ], 1 );
070 weights_delta.m [ 0 ] = new Matrix ( weights.m [ 0 ] );
071 weights_momentum.m [ 0 ] = new Matrix ( weights.m [ 0 ] );
072 } else {
073 inputs.m [ index_layer ] = new Matrix (
074 neurons_per_layer [ index_layer - 1 ] + bias, 1 );
075 if ( add_bias_inputs ) {
076 inputs.m [ index_layer ].data [ 0 ] [ 0 ] = 1.0;
077 }
078 weights.m [ index_layer ] = new Matrix (
079 neurons_per_layer [ index_layer - 1 ] + bias,
080 neurons_per_layer [ index_layer ] );
081 weighted_sums.m [ index_layer ] = new Matrix (
082 neurons_per_layer [ index_layer ], 1 );
083 outputs.m [ index_layer ] = new Matrix (
084 neurons_per_layer [ index_layer ], 1 );
085 local_gradients.m [ index_layer ] = new Matrix (
086 neurons_per_layer [ index_layer ], 1 );
087 sum_weighted_deltas.m [ index_layer ] = new Matrix (
088 neurons_per_layer [ index_layer ], 1 );
089 weights_delta.m [ index_layer ]
090 = new Matrix ( weights.m [ index_layer ] );
091 weights_momentum.m [ index_layer ]
092 = new Matrix ( weights.m [ index_layer ] );
093 }
094 }
095 if ( randomize_initial_weights ) {
096 weights_randomize_uniform ( -1.0, 1.0 );
097 }
098 }
099
100 public Matrix_Array update_forward_backward (
101 Matrix input_vector,
102 Matrix outputs_desired ) {
103 //////////////////////////////////////////////////////////////////////
104 weights_update ( );
105 Matrix outputs_actual = forward_pass ( input_vector );
106 Matrix error_vector = backward_pass ( outputs_desired );
107 Matrix [ ] matrices_array = new Matrix [ 2 ];
108 matrices_array [ 0 ] = outputs_actual;
109 matrices_array [ 1 ] = error_vector;
110 Matrix_Array results = new Matrix_Array ( matrices_array );
111 return results;
112 }
113
114 public Matrix forward_pass ( Matrix input_vector ) {
115 //////////////////////////////////////////////////////////////////////
116 // needs to raise exception if input_vector and inputs [ 0 ] bad size
117 // Returns the output vector.
118 //////////////////////////////////////////////////////////////////////
119 for ( int index_layer = 0; index_layer < layers; index_layer++ ) {
120 if ( using_bias_inputs ) {
121 for ( int index_row = 1;
122 index_row < inputs.m [ index_layer ].rows;
123 index_row++ ) {
124 if ( index_layer == 0 ) {
125 inputs.m [ index_layer ].data [ index_row ] [ 0 ]
126 = input_vector.data [ index_row - 1 ] [ 0 ];
127 } else {
128 inputs.m [ index_layer ].data [ index_row ] [ 0 ]
129 = outputs.m [ index_layer - 1 ].data [ index_row - 1 ] [ 0 ];
130 }
131 }
132 } else {
133 if ( index_layer == 0 ) {
134 inputs.m [ index_layer ] = new Matrix ( input_vector );
135 } else {
136 inputs.m [ index_layer ]
137 = new Matrix ( outputs.m [ index_layer ] );
138 }
139 }
140 weighted_sums.m [ index_layer ] = Matrix.multiply (
141 weights.m [ index_layer ].transpose ( ),
142 inputs.m [ index_layer ] );
143 outputs.m [ index_layer ]
144 = weighted_sums.m [ index_layer ].sigmoid ( );
145 }
146 return new Matrix ( outputs.m [ layers - 1 ] );
147 }
148
149 public Matrix backward_pass ( Matrix desired_outputs ) {
150 //////////////////////////////////////////////////////////////////////
151 // raise exception if desired_outputs wrong size
152 // Returns error vector.
153 //////////////////////////////////////////////////////////////////////
154 Matrix error_vector
155 = desired_outputs.subtract ( outputs.m [ layers - 1 ] );
156 // check this again, get rid of sigmoid_derivative
157 local_gradients.m [ layers - 1 ]
158 = weighted_sums.m [ layers - 1 ].sigmoid_derivative ( );
159 local_gradients.m [ layers - 1 ] = Matrix.multiply_elements (
160 local_gradients.m [ layers - 1 ], error_vector );
161 weights_delta.m [ layers - 1 ]
162 = Matrix.multiply ( inputs.m [ layers - 1 ],
163 local_gradients.m [ layers - 1 ].transpose ( ) );
164 // needs transpose or sum below?
165 for ( int index_layer = layers - 2;
166 index_layer >= 0;
167 index_layer-- ) {
168 // Chopping off the weights from the bias neuron.
169 sum_weighted_deltas.m [ index_layer ]
170 = weights.m [ index_layer + 1 ].sub_matrix (
171 1, weights.m [ index_layer + 1 ].rows - 1,
172 0, weights.m [ index_layer + 1 ].columns - 1 );
173 // sum_weighted_deltas goes from matrix to vector here
174 // Could be big fat error here...
175 sum_weighted_deltas.m [ index_layer ] = Matrix.multiply (
176 sum_weighted_deltas.m [ index_layer ],
177 local_gradients.m [ index_layer + 1 ] );
178 local_gradients.m [ index_layer ]
179 = outputs.m [ index_layer ].subtract (
180 outputs.m [ index_layer ].square_elements ( ) );
181 local_gradients.m [ index_layer ] = Matrix.multiply_elements (
182 local_gradients.m [ index_layer ],
183 sum_weighted_deltas.m [ index_layer ] );
184 weights_delta.m [ index_layer ] = Matrix.multiply (
185 inputs.m [ index_layer ],
186 local_gradients.m [ index_layer ].transpose ( ) );
187 }
188 weights_delta = weights_delta.multiply ( learning_rate );
189 weights_delta = weights_delta.add ( weights_momentum );
190 weights_momentum = weights_delta.multiply ( momentum_constant );
191 return error_vector;
192 }
193
194 public void weights_update ( ) {
195 //////////////////////////////////////////////////////////////////////
196 weights = weights.add ( weights_delta );
197 }
198
199 public void weights_randomize_uniform ( double min, double max ) {
200 //////////////////////////////////////////////////////////////////////
201 for ( int index_layer = 0; index_layer < layers; index_layer++ ) {
202 weights.m [ index_layer ]
203 = weights.m [ index_layer ].randomize_uniform ( min, max );
204 }
205 }
206
207 //////////////////////////////////////////////////////////////////////
208 //////////////////////////////////////////////////////////////////////
209 }