001 package com.croftsoft.apps.insight; 002 003 import java.applet.Applet; 004 import java.awt.Graphics; 005 import java.awt.*; 006 import java.awt.Color; 007 import java.awt.Point; 008 import java.awt.Rectangle; 009 010 public class Backprop_Net { 011 ////////////////////////////////////////////////////////////////////// 012 ////////////////////////////////////////////////////////////////////// 013 014 double learning_rate = 0.1; 015 double momentum_constant = 0.9; 016 int layers; // synaptic weight layers 017 boolean using_bias_inputs; 018 Matrix_Array inputs; 019 Matrix_Array weights; 020 Matrix_Array weighted_sums; 021 Matrix_Array outputs; 022 Matrix_Array local_gradients; 023 Matrix_Array sum_weighted_deltas; 024 Matrix_Array weights_delta; 025 Matrix_Array weights_momentum; 026 027 ////////////////////////////////////////////////////////////////////// 028 ////////////////////////////////////////////////////////////////////// 029 030 public Backprop_Net ( 031 int layer0_inputs, 032 int [ ] neurons_per_layer, 033 boolean add_bias_inputs, 034 boolean randomize_initial_weights ) { 035 ////////////////////////////////////////////////////////////////////// 036 // Constructor method. 037 // The first layer, layer 0, is NOT an input layer. 038 // Should return exception if number of layers < 1. 039 ////////////////////////////////////////////////////////////////////// 040 int bias = add_bias_inputs ? 1 : 0; 041 ////////////////////////////////////////////////////////////////////// 042 this.layers = neurons_per_layer.length; 043 this.using_bias_inputs = add_bias_inputs; 044 this.inputs = new Matrix_Array ( layers ); 045 this.weights = new Matrix_Array ( layers ); 046 this.weighted_sums = new Matrix_Array ( layers ); 047 this.outputs = new Matrix_Array ( layers ); 048 this.local_gradients = new Matrix_Array ( layers ); 049 this.sum_weighted_deltas = new Matrix_Array ( layers ); 050 this.weights_delta = new Matrix_Array ( layers ); 051 this.weights_momentum = new Matrix_Array ( layers ); 052 for ( int index_layer = 0; 053 index_layer < layers; 054 index_layer++ ) { 055 if ( index_layer == 0 ) { 056 inputs.m [ 0 ] = new Matrix ( layer0_inputs + bias, 1 ); 057 if ( add_bias_inputs ) { 058 inputs.m [ 0 ].data [ 0 ] [ 0 ] = 1.0; 059 } 060 weights.m [ 0 ] = new Matrix ( 061 layer0_inputs + bias, neurons_per_layer [ 0 ] ); 062 weighted_sums.m [ 0 ] 063 = new Matrix ( neurons_per_layer [ 0 ], 1 ); 064 outputs.m [ 0 ] 065 = new Matrix ( neurons_per_layer [ 0 ], 1 ); 066 local_gradients.m [ 0 ] 067 = new Matrix ( neurons_per_layer [ 0 ], 1 ); 068 sum_weighted_deltas.m [ 0 ] 069 = new Matrix ( neurons_per_layer [ 0 ], 1 ); 070 weights_delta.m [ 0 ] = new Matrix ( weights.m [ 0 ] ); 071 weights_momentum.m [ 0 ] = new Matrix ( weights.m [ 0 ] ); 072 } else { 073 inputs.m [ index_layer ] = new Matrix ( 074 neurons_per_layer [ index_layer - 1 ] + bias, 1 ); 075 if ( add_bias_inputs ) { 076 inputs.m [ index_layer ].data [ 0 ] [ 0 ] = 1.0; 077 } 078 weights.m [ index_layer ] = new Matrix ( 079 neurons_per_layer [ index_layer - 1 ] + bias, 080 neurons_per_layer [ index_layer ] ); 081 weighted_sums.m [ index_layer ] = new Matrix ( 082 neurons_per_layer [ index_layer ], 1 ); 083 outputs.m [ index_layer ] = new Matrix ( 084 neurons_per_layer [ index_layer ], 1 ); 085 local_gradients.m [ index_layer ] = new Matrix ( 086 neurons_per_layer [ index_layer ], 1 ); 087 sum_weighted_deltas.m [ index_layer ] = new Matrix ( 088 neurons_per_layer [ index_layer ], 1 ); 089 weights_delta.m [ index_layer ] 090 = new Matrix ( weights.m [ index_layer ] ); 091 weights_momentum.m [ index_layer ] 092 = new Matrix ( weights.m [ index_layer ] ); 093 } 094 } 095 if ( randomize_initial_weights ) { 096 weights_randomize_uniform ( -1.0, 1.0 ); 097 } 098 } 099 100 public Matrix_Array update_forward_backward ( 101 Matrix input_vector, 102 Matrix outputs_desired ) { 103 ////////////////////////////////////////////////////////////////////// 104 weights_update ( ); 105 Matrix outputs_actual = forward_pass ( input_vector ); 106 Matrix error_vector = backward_pass ( outputs_desired ); 107 Matrix [ ] matrices_array = new Matrix [ 2 ]; 108 matrices_array [ 0 ] = outputs_actual; 109 matrices_array [ 1 ] = error_vector; 110 Matrix_Array results = new Matrix_Array ( matrices_array ); 111 return results; 112 } 113 114 public Matrix forward_pass ( Matrix input_vector ) { 115 ////////////////////////////////////////////////////////////////////// 116 // needs to raise exception if input_vector and inputs [ 0 ] bad size 117 // Returns the output vector. 118 ////////////////////////////////////////////////////////////////////// 119 for ( int index_layer = 0; index_layer < layers; index_layer++ ) { 120 if ( using_bias_inputs ) { 121 for ( int index_row = 1; 122 index_row < inputs.m [ index_layer ].rows; 123 index_row++ ) { 124 if ( index_layer == 0 ) { 125 inputs.m [ index_layer ].data [ index_row ] [ 0 ] 126 = input_vector.data [ index_row - 1 ] [ 0 ]; 127 } else { 128 inputs.m [ index_layer ].data [ index_row ] [ 0 ] 129 = outputs.m [ index_layer - 1 ].data [ index_row - 1 ] [ 0 ]; 130 } 131 } 132 } else { 133 if ( index_layer == 0 ) { 134 inputs.m [ index_layer ] = new Matrix ( input_vector ); 135 } else { 136 inputs.m [ index_layer ] 137 = new Matrix ( outputs.m [ index_layer ] ); 138 } 139 } 140 weighted_sums.m [ index_layer ] = Matrix.multiply ( 141 weights.m [ index_layer ].transpose ( ), 142 inputs.m [ index_layer ] ); 143 outputs.m [ index_layer ] 144 = weighted_sums.m [ index_layer ].sigmoid ( ); 145 } 146 return new Matrix ( outputs.m [ layers - 1 ] ); 147 } 148 149 public Matrix backward_pass ( Matrix desired_outputs ) { 150 ////////////////////////////////////////////////////////////////////// 151 // raise exception if desired_outputs wrong size 152 // Returns error vector. 153 ////////////////////////////////////////////////////////////////////// 154 Matrix error_vector 155 = desired_outputs.subtract ( outputs.m [ layers - 1 ] ); 156 // check this again, get rid of sigmoid_derivative 157 local_gradients.m [ layers - 1 ] 158 = weighted_sums.m [ layers - 1 ].sigmoid_derivative ( ); 159 local_gradients.m [ layers - 1 ] = Matrix.multiply_elements ( 160 local_gradients.m [ layers - 1 ], error_vector ); 161 weights_delta.m [ layers - 1 ] 162 = Matrix.multiply ( inputs.m [ layers - 1 ], 163 local_gradients.m [ layers - 1 ].transpose ( ) ); 164 // needs transpose or sum below? 165 for ( int index_layer = layers - 2; 166 index_layer >= 0; 167 index_layer-- ) { 168 // Chopping off the weights from the bias neuron. 169 sum_weighted_deltas.m [ index_layer ] 170 = weights.m [ index_layer + 1 ].sub_matrix ( 171 1, weights.m [ index_layer + 1 ].rows - 1, 172 0, weights.m [ index_layer + 1 ].columns - 1 ); 173 // sum_weighted_deltas goes from matrix to vector here 174 // Could be big fat error here... 175 sum_weighted_deltas.m [ index_layer ] = Matrix.multiply ( 176 sum_weighted_deltas.m [ index_layer ], 177 local_gradients.m [ index_layer + 1 ] ); 178 local_gradients.m [ index_layer ] 179 = outputs.m [ index_layer ].subtract ( 180 outputs.m [ index_layer ].square_elements ( ) ); 181 local_gradients.m [ index_layer ] = Matrix.multiply_elements ( 182 local_gradients.m [ index_layer ], 183 sum_weighted_deltas.m [ index_layer ] ); 184 weights_delta.m [ index_layer ] = Matrix.multiply ( 185 inputs.m [ index_layer ], 186 local_gradients.m [ index_layer ].transpose ( ) ); 187 } 188 weights_delta = weights_delta.multiply ( learning_rate ); 189 weights_delta = weights_delta.add ( weights_momentum ); 190 weights_momentum = weights_delta.multiply ( momentum_constant ); 191 return error_vector; 192 } 193 194 public void weights_update ( ) { 195 ////////////////////////////////////////////////////////////////////// 196 weights = weights.add ( weights_delta ); 197 } 198 199 public void weights_randomize_uniform ( double min, double max ) { 200 ////////////////////////////////////////////////////////////////////// 201 for ( int index_layer = 0; index_layer < layers; index_layer++ ) { 202 weights.m [ index_layer ] 203 = weights.m [ index_layer ].randomize_uniform ( min, max ); 204 } 205 } 206 207 ////////////////////////////////////////////////////////////////////// 208 ////////////////////////////////////////////////////////////////////// 209 }