001         package com.croftsoft.apps.insight;
002    
003         import java.applet.Applet;
004         import java.awt.Graphics;
005         import java.awt.*;
006         import java.awt.Color;
007         import java.awt.Point;
008         import java.awt.Rectangle;
009    
010         public class Backprop_Net {
011         //////////////////////////////////////////////////////////////////////
012         //////////////////////////////////////////////////////////////////////
013    
014         double        learning_rate     = 0.1;
015         double        momentum_constant = 0.9;
016         int           layers; // synaptic weight layers
017         boolean       using_bias_inputs;
018         Matrix_Array  inputs;
019         Matrix_Array  weights;
020         Matrix_Array  weighted_sums;
021         Matrix_Array  outputs;
022         Matrix_Array  local_gradients;
023         Matrix_Array  sum_weighted_deltas;
024         Matrix_Array  weights_delta;
025         Matrix_Array  weights_momentum;
026    
027         //////////////////////////////////////////////////////////////////////
028         //////////////////////////////////////////////////////////////////////
029    
030         public Backprop_Net (
031           int      layer0_inputs,
032           int [ ]  neurons_per_layer,
033           boolean  add_bias_inputs,
034           boolean  randomize_initial_weights ) {
035         //////////////////////////////////////////////////////////////////////
036         // Constructor method.
037         // The first layer, layer 0, is NOT an input layer.
038         // Should return exception if number of layers < 1.
039         //////////////////////////////////////////////////////////////////////
040           int  bias   = add_bias_inputs ? 1 : 0;
041         //////////////////////////////////////////////////////////////////////
042           this.layers = neurons_per_layer.length;
043           this.using_bias_inputs = add_bias_inputs;
044           this.inputs              = new Matrix_Array ( layers );
045           this.weights             = new Matrix_Array ( layers );
046           this.weighted_sums       = new Matrix_Array ( layers );
047           this.outputs             = new Matrix_Array ( layers );
048           this.local_gradients     = new Matrix_Array ( layers );
049           this.sum_weighted_deltas = new Matrix_Array ( layers );
050           this.weights_delta       = new Matrix_Array ( layers );
051           this.weights_momentum    = new Matrix_Array ( layers );
052           for ( int index_layer = 0;
053                     index_layer < layers;
054                     index_layer++ ) {
055             if ( index_layer == 0 ) {
056               inputs.m [ 0 ] = new Matrix ( layer0_inputs + bias, 1 );
057               if ( add_bias_inputs ) {
058                 inputs.m [ 0 ].data [ 0 ] [ 0 ] = 1.0;
059               }
060               weights.m [ 0 ] = new Matrix (
061                 layer0_inputs + bias, neurons_per_layer [ 0 ] );
062               weighted_sums.m [ 0 ]
063                 = new Matrix ( neurons_per_layer [ 0 ], 1 );
064               outputs.m [ 0 ]
065                 = new Matrix ( neurons_per_layer [ 0 ], 1 );
066               local_gradients.m [ 0 ]
067                 = new Matrix ( neurons_per_layer [ 0 ], 1 );
068               sum_weighted_deltas.m [ 0 ]
069                 = new Matrix ( neurons_per_layer [ 0 ], 1 );
070               weights_delta.m [ 0 ] = new Matrix ( weights.m [ 0 ] );
071               weights_momentum.m [ 0 ] = new Matrix ( weights.m [ 0 ] );
072             } else {
073               inputs.m [ index_layer ] = new Matrix (
074                 neurons_per_layer [ index_layer - 1 ] + bias, 1 );
075               if ( add_bias_inputs ) {
076                 inputs.m [ index_layer ].data [ 0 ] [ 0 ] = 1.0;
077               }
078               weights.m [ index_layer ] = new Matrix (
079                 neurons_per_layer [ index_layer - 1 ] + bias,
080                 neurons_per_layer [ index_layer ] );
081               weighted_sums.m [ index_layer ] = new Matrix (
082                 neurons_per_layer [ index_layer ], 1 );
083               outputs.m [ index_layer ] = new Matrix (
084                 neurons_per_layer [ index_layer ], 1 );
085               local_gradients.m [ index_layer ] = new Matrix (
086                 neurons_per_layer [ index_layer ], 1 );
087               sum_weighted_deltas.m [ index_layer ] = new Matrix (
088                 neurons_per_layer [ index_layer ], 1 );
089               weights_delta.m [ index_layer ]
090                 = new Matrix ( weights.m [ index_layer ] );
091               weights_momentum.m [ index_layer ]
092                 = new Matrix ( weights.m [ index_layer ] );
093             }
094           }
095           if ( randomize_initial_weights ) {
096             weights_randomize_uniform ( -1.0, 1.0 );
097           }
098         }
099           
100         public Matrix_Array  update_forward_backward (
101           Matrix  input_vector,
102           Matrix  outputs_desired ) {
103         //////////////////////////////////////////////////////////////////////
104           weights_update ( );
105           Matrix  outputs_actual = forward_pass ( input_vector );
106           Matrix  error_vector = backward_pass ( outputs_desired );
107           Matrix [ ]  matrices_array = new Matrix [ 2 ];
108           matrices_array [ 0 ] = outputs_actual;
109           matrices_array [ 1 ] = error_vector;
110           Matrix_Array  results = new Matrix_Array ( matrices_array );
111           return results;
112         }
113    
114         public Matrix  forward_pass ( Matrix  input_vector ) {
115         //////////////////////////////////////////////////////////////////////
116         // needs to raise exception if input_vector and inputs [ 0 ] bad size
117         // Returns the output vector.
118         //////////////////////////////////////////////////////////////////////
119           for ( int index_layer = 0; index_layer < layers; index_layer++ ) {
120             if ( using_bias_inputs ) {
121               for ( int index_row = 1;
122                         index_row < inputs.m [ index_layer ].rows;
123                         index_row++ ) {
124                 if ( index_layer == 0 ) {
125                   inputs.m [ index_layer ].data [ index_row ] [ 0 ]
126                     = input_vector.data [ index_row - 1 ] [ 0 ];
127                 } else {
128                   inputs.m [ index_layer ].data [ index_row ] [ 0 ]
129                     = outputs.m [ index_layer - 1 ].data [ index_row - 1 ] [ 0 ];
130                 }
131               }
132             } else {
133               if ( index_layer == 0 ) {
134                 inputs.m [ index_layer ] = new Matrix ( input_vector );
135               } else {
136                 inputs.m [ index_layer ]
137                   = new Matrix ( outputs.m [ index_layer ] );
138               }
139             }
140             weighted_sums.m [ index_layer ] = Matrix.multiply (
141               weights.m [ index_layer ].transpose ( ),
142               inputs.m [ index_layer ] );
143             outputs.m [ index_layer ]
144               = weighted_sums.m [ index_layer ].sigmoid ( );
145           }
146           return new Matrix ( outputs.m [ layers - 1 ] );
147         }
148    
149         public Matrix  backward_pass ( Matrix  desired_outputs ) {
150         //////////////////////////////////////////////////////////////////////
151         // raise exception if desired_outputs wrong size
152         // Returns error vector.
153         //////////////////////////////////////////////////////////////////////
154           Matrix  error_vector
155             = desired_outputs.subtract ( outputs.m [ layers - 1 ] );
156    // check this again, get rid of sigmoid_derivative
157           local_gradients.m [ layers - 1 ]
158             = weighted_sums.m [ layers - 1 ].sigmoid_derivative ( );
159           local_gradients.m [ layers - 1 ] = Matrix.multiply_elements (
160             local_gradients.m [ layers - 1 ], error_vector );
161           weights_delta.m [ layers - 1 ]
162             = Matrix.multiply ( inputs.m [ layers - 1 ],
163             local_gradients.m [ layers - 1 ].transpose ( ) );
164    // needs transpose or sum below?
165           for ( int index_layer = layers - 2;
166                     index_layer >= 0;
167                     index_layer-- ) {
168             // Chopping off the weights from the bias neuron.
169             sum_weighted_deltas.m [ index_layer ]
170               = weights.m [ index_layer + 1 ].sub_matrix (
171               1, weights.m [ index_layer + 1 ].rows    - 1,
172               0, weights.m [ index_layer + 1 ].columns - 1 );
173             // sum_weighted_deltas goes from matrix to vector here
174    // Could be big fat error here...
175             sum_weighted_deltas.m [ index_layer ] = Matrix.multiply (
176               sum_weighted_deltas.m [ index_layer ],
177               local_gradients.m [ index_layer + 1 ] );
178             local_gradients.m [ index_layer ]
179               = outputs.m [ index_layer ].subtract (
180                 outputs.m [ index_layer ].square_elements ( ) );
181             local_gradients.m [ index_layer ] = Matrix.multiply_elements (
182               local_gradients.m [ index_layer ],
183               sum_weighted_deltas.m [ index_layer ] );
184             weights_delta.m [ index_layer ] = Matrix.multiply (
185               inputs.m [ index_layer ],
186               local_gradients.m [ index_layer ].transpose ( ) );
187           }
188           weights_delta = weights_delta.multiply ( learning_rate );
189           weights_delta = weights_delta.add ( weights_momentum );
190           weights_momentum = weights_delta.multiply ( momentum_constant );
191           return error_vector;
192         }
193    
194         public void  weights_update ( ) {
195         //////////////////////////////////////////////////////////////////////
196           weights = weights.add ( weights_delta );
197         }
198    
199         public void  weights_randomize_uniform ( double  min, double  max ) {
200         //////////////////////////////////////////////////////////////////////
201           for ( int index_layer = 0; index_layer < layers; index_layer++ ) {
202             weights.m [ index_layer ]
203               = weights.m [ index_layer ].randomize_uniform ( min, max );
204           }
205         }
206    
207         //////////////////////////////////////////////////////////////////////
208         //////////////////////////////////////////////////////////////////////
209         }