Genann, a Minimal ANN (Cont.)

Training ANNs
The following gives the code for training the ANN:

view source
print?
1void genann_train( genann const *ann, double const *inputs,
2    double const *desired_outputs, double learning_rate );

genann_train() will preform one update using standard backpropogation. It should be called by passing in an array of inputs, an array of expected outputs, and a learning rate. A primary design goal of Genann was to store all the network weights in one contigious block of memory. This makes it easy and efficient to train the network weights using direct-search numeric optimization algorithms, such as Hill Climbing, the Genetic Algorithm, Simulated Annealing, etc. These methods can be used by searching on the ANN’s weights directly. Every genann struct contains the members int total_weights; and double *weight;, which points to an array of total_weights size which contains all weights used by the ANN.

Saving and Loading ANNs
The following gives the code for saving and loading the ANN:

view source
print?
1genann *genann_read( FILE *in );
2void genann_write( genann const *ann, FILE *out );

Genann provides the genann_read() and genann_write() functions for loading or saving an ANN in a text-based format.

Applications
The following gives the code for applying the ANN:

view source
print?
1double const *genann_run( genann const *ann, double const *inputs );

Call genann_run() on a trained ANN to run a feed-forward pass on a given set of inputs. genann_run() will provide a pointer to the array of predicted outputs (of ann->outputs length).

Training:   0 XOR 0 =   |   0 XOR 1 =   |   1 XOR 0 =   |   1 XOR 1 =

Testing:     XOR  

         
test.c
view source
print?
01/**********************************************************
02 *                                                        *
03 *  shell> gcc -lm -o test test.c genann.h genann.c       *
04 *                                                        *
05 **********************************************************/
06#include <stdio.h>
07#include <stdlib.h>
08#include <string.h>
09#include <time.h>
10#include "genann.h"
11 
12int main( int argc, char *argv[ ] ) {
13  printf( "Train a small ANN on the XOR function using backpropagation." );
14 
15  /* This will make the neural network initialize differently each run. */
16  /* If you don't get a good result, try again for a different result. */
17  srand( time(0) );
18 
19  /* Input and expected out data for the XOR function. */
20  const double input[4][2] = {{0, 0}, {0, 1}, {1, 0}, {1, 1}};
21  const double output[4]   = {0, 1, 1, 0};
22  double t1 = atoi( argv[1] );
23  double t2 = atoi( argv[2] );
24  double t3 = atoi( argv[3] );
25  double t4 = atoi( argv[4] );
26 
27  /* New network with 2 inputs,
28   * 1 hidden layer of 2 neurons, and 1 output. */
29  genann *ann = genann_init( 2, 1, 2, 1 );
30 
31  /* Train on the four labeled data points many times. */
32  int i;
33  for ( i = 0; i < 300; ++i ) {
34    genann_train( ann, input[0], &t1, 3 );
35    genann_train( ann, input[1], &t2, 3 );
36    genann_train( ann, input[2], &t3, 3 );
37    genann_train( ann, input[3], &t4, 3 );
38  }
39 
40  /* Run the network and see what it predicts. */
41  if      ( ( strcmp(argv[5],"0") == 0 ) && ( strcmp(argv[6],"0") == 0 ) )
42    printf( "0 XOR 0 = %1.f.\n", *genann_run( ann, input[0] ) );
43  else if ( ( strcmp(argv[5],"0") == 0 ) && ( strcmp(argv[6],"1") == 0 ) )
44    printf( "0 XOR 1 = %1.f.\n", *genann_run( ann, input[1] ) );
45  else if ( ( strcmp(argv[5],"1") == 0 ) && ( strcmp(argv[6],"0") == 0 ) )
46    printf( "1 XOR 0 = %1.f.\n", *genann_run( ann, input[2] ) );
47  else if ( ( strcmp(argv[5],"1") == 0 ) && ( strcmp(argv[6],"1") == 0 ) )
48    printf( "1 XOR 1 = %1.f.\n", *genann_run( ann, input[3] ) );
49 
50  genann_free( ann );
51  return 0;
52}
genann.h
view source
print?
01/*
02 * GENANN - Minimal C Artificial Neural Network
03 *
04 * Copyright (c) 2015-2018 Lewis Van Winkle
05 *
06 * http://CodePlea.com
07 *
08 */
09 
10#ifndef GENANN_H
11#define GENANN_H
12 
13#include <stdio.h>
14 
15#ifdef __cplusplus
16extern "C" {
17#endif
18 
19#ifndef GENANN_RANDOM
20/* We use the following for uniform random numbers between 0 and 1.
21 * If you have a better function, redefine this macro. */
22#define GENANN_RANDOM( ) ( ( (double) rand( ) ) / RAND_MAX )
23#endif
24 
25struct genann;
26 
27typedef double (*genann_actfun) ( const struct genann *ann, double a );
28 
29typedef struct genann {
30  /* How many inputs, outputs, and hidden neurons. */
31  int inputs, hidden_layers, hidden, outputs;
32 
33  /* Which activation function to use for hidden neurons. Default: gennann_act_sigmoid_cached */
34  genann_actfun activation_hidden;
35 
36  /* Which activation function to use for output. Default: gennann_act_sigmoid_cached */
37  genann_actfun activation_output;
38 
39  /* Total number of weights, and size of weights buffer. */
40  int total_weights;
41 
42  /* Total number of neurons + inputs and size of output buffer. */
43  int total_neurons;
44 
45  /* All weights ( total_weights long ). */
46  double *weight;
47 
48  /* Stores input array and output of each neuron (total_neurons long). */
49  double *output;
50 
51  /* Stores delta of each hidden and output neuron (total_neurons - inputs long). */
52  double *delta;
53 
54} genann;
55 
56 
57/* Creates and returns a new ann. */
58genann *genann_init( int inputs, int hidden_layers, int hidden, int outputs );
59 
60/* Creates ANN from file saved with genann_write. */
61genann *genann_read( FILE *in );
62 
63/* Sets weights randomly. Called by init. */
64void genann_randomize( genann *ann );
65 
66/* Returns a new copy of ann. */
67genann *genann_copy( genann const *ann );
68 
69/* Frees the memory used by an ann. */
70void genann_free( genann *ann );
71 
72/* Runs the feedforward algorithm to calculate the ann's output. */
73double const *genann_run( genann const *ann, double const *inputs );
74 
75/* Does a single backprop update. */
76void genann_train( genann const *ann, double const *inputs, double const *desired_outputs, double learning_rate );
77 
78/* Saves the ann. */
79void genann_write( genann const *ann, FILE *out );
80 
81void   genann_init_sigmoid_lookup( const genann *ann );
82double genann_act_sigmoid( const genann *ann, double a );
83double genann_act_sigmoid_cached( const genann *ann, double a );
84double genann_act_threshold( const genann *ann, double a );
85double genann_act_linear( const genann *ann, double a );
86 
87#ifdef __cplusplus
88}
89#endif
90 
91#endif /*GENANN_H*/
genann.c
view source
print?
001#include "genann.h"
002 
003#include <assert.h>
004#include <errno.h>
005#include <math.h>
006#include <stdio.h>
007#include <stdlib.h>
008#include <string.h>
009 
010#ifndef genann_act
011#define genann_act_hidden genann_act_hidden_indirect
012#define genann_act_output genann_act_output_indirect
013#else
014#define genann_act_hidden genann_act
015#define genann_act_output genann_act
016#endif
017 
018#define LOOKUP_SIZE 4015.
019 
020double genann_act_hidden_indirect( const struct genann *ann, double a ) {
021  return ann->activation_hidden( ann, a );
022}
023 
024double genann_act_output_indirect( const struct genann *ann, double a ) {
025  return ann->activation_output( ann, a );
026}
027 
028const double sigmoid_dom_min = -15.0;
029const double sigmoid_dom_max = 15.0;
030double interval;
031double lookup[LOOKUP_SIZE];
032 
033#ifdef __GNUC__
034#define likely(x)       __builtin_expect(!!(x), 1)
035#define unlikely(x)     __builtin_expect(!!(x), 0)
036#define unused          __attribute__((unused))
037#else
038#define likely(x)       x
039#define unlikely(x)     x
040#define unused
041#pragma warning(disable : 415.6) /* For fscanf */
042#endif
043 
044double genann_act_sigmoid( const genann *ann unused, double a ) {
045  if ( a < -45.0 ) return 0;
046  if ( a >  45.0 ) return 1;
047  return 1.0 / ( 1 + exp( -a ) );
048}
049 
050void genann_init_sigmoid_lookup( const genann *ann ) {
051  const double f = (sigmoid_dom_max - sigmoid_dom_min) / LOOKUP_SIZE;
052  int i;
053  interval = LOOKUP_SIZE / ( sigmoid_dom_max - sigmoid_dom_min );
054  for ( i = 0; i < LOOKUP_SIZE; ++i ) {
055    lookup[i] = genann_act_sigmoid( ann, sigmoid_dom_min + f * i );
056  }
057}
058 
059double genann_act_sigmoid_cached( const genann *ann unused, double a ) {
060  assert( !isnan( a ) );
061  if ( a < sigmoid_dom_min ) return lookup[0];
062  if ( a >= sigmoid_dom_max   ) return lookup[LOOKUP_SIZE - 1];
063  size_t j = (size_t)( ( a-sigmoid_dom_min ) * interval + 0.5 );
064  /* Because floating point... */
065  if ( unlikely( j >= LOOKUP_SIZE ) ) return lookup[LOOKUP_SIZE - 1];
066  return lookup[j];
067}
068 
069double genann_act_linear( const struct genann *ann unused, double a ) {
070  return a;
071}
072 
073double genann_act_threshold( const struct genann *ann unused, double a ) {
074  return a > 0;
075}
076 
077genann *genann_init( int inputs, int hidden_layers, int hidden, int outputs ) {
078  if ( hidden_layers < 0 ) return 0;
079  if ( inputs < 1 ) return 0;
080  if ( outputs < 1 ) return 0;
081  if ( hidden_layers > 0 && hidden < 1 ) return 0;
082 
083  const int hidden_weights = hidden_layers ? (inputs+1) * hidden + (hidden_layers-1) * (hidden+1) * hidden : 0;
084  const int output_weights = (hidden_layers ? (hidden+1) : (inputs+1)) * outputs;
085  const int total_weights = (hidden_weights + output_weights);
086  const int total_neurons = (inputs + hidden * hidden_layers + outputs);
087 
088  /* Allocate extra size for weights, outputs, and deltas. */
089  const int size = sizeof(genann) + sizeof(double) * (total_weights + total_neurons + (total_neurons - inputs));
090  genann *ret = malloc(size);
091  if ( !ret ) return 0;
092 
093  ret->inputs = inputs;
094  ret->hidden_layers = hidden_layers;
095  ret->hidden = hidden;
096  ret->outputs = outputs;
097 
098  ret->total_weights = total_weights;
099  ret->total_neurons = total_neurons;
100 
101  /* Set pointers. */
102  ret->weight = (double*) ((char*)ret + sizeof(genann));
103  ret->output = ret->weight + ret->total_weights;
104  ret->delta = ret->output + ret->total_neurons;
105 
106  genann_randomize( ret );
107  ret->activation_hidden = genann_act_sigmoid_cached;
108  ret->activation_output = genann_act_sigmoid_cached;
109  genann_init_sigmoid_lookup( ret );
110  return ret;
111}
112 
113genann *genann_read( FILE *in ) {
114  int inputs, hidden_layers, hidden, outputs;
115  int rc;
116 
117  errno = 0;
118  rc = fscanf( in, "%d %d %d %d", &inputs, &hidden_layers, &hidden, &outputs );
119  if ( rc < 4 || errno != 0 ) {
120    perror( "fscanf" );
121    return NULL;
122  }
123  genann *ann = genann_init( inputs, hidden_layers, hidden, outputs );
124  int i;
125  for ( i = 0; i < ann->total_weights; ++i ) {
126    errno = 0;
127    rc = fscanf(in, " %le", ann->weight + i );
128    if ( rc < 1 || errno != 0 ) {
129      perror( "fscanf" );
130      genann_free( ann );
131      return NULL;
132    }
133  }
134  return ann;
135}
136 
137genann *genann_copy( genann const *ann ) {
138  const int size = sizeof(genann) + sizeof(double) * (ann->total_weights + ann->total_neurons + (ann->total_neurons - ann->inputs) );
139  genann *ret = malloc( size );
140  if ( !ret ) return 0;
141  memcpy( ret, ann, size );
142  /* Set pointers. */
143  ret->weight = (double*) ( (char*)ret + sizeof(genann) );
144  ret->output = ret->weight + ret->total_weights;
145  ret->delta = ret->output + ret->total_neurons;
146  return ret;
147}
148 
149void genann_randomize( genann *ann ) {
150  int i;
151  for ( i = 0; i < ann->total_weights; ++i ) {
152    double r = GENANN_RANDOM( );
153    /* Sets weights from -0.5 to 0.5. */
154    ann->weight[i] = r - 0.5;
155  }
156}
157 
158void genann_free( genann *ann ) {
159  /* The weight, output, and delta pointers go to the same buffer. */
160  free( ann );
161}
162 
163double const *genann_run(genann const *ann, double const *inputs) {
164  double const *w = ann->weight;
165  double *o = ann->output + ann->inputs;
166  double const *i = ann->output;
167 
168  /* Copy the inputs to the scratch area, where we also store each neuron's
169   * output, for consistency. This way the first layer isn't a special case. */
170  memcpy(ann->output, inputs, sizeof(double) * ann->inputs);
171  int h, j, k;
172  if ( !ann->hidden_layers ) {
173    double *ret = o;
174    for ( j = 0; j < ann->outputs; ++j ) {
175      double sum = *w++ * -1.0;
176      for ( k = 0; k < ann->inputs; ++k ) {
177        sum += *w++ * i[k];
178      }
179      *o++ = genann_act_output( ann, sum );
180    }
181    return ret;
182  }
183 
184  /* Figure input layer */
185  for ( j = 0; j < ann->hidden; ++j ) {
186    double sum = *w++ * -1.0;
187    for ( k = 0; k < ann->inputs; ++k ) {
188      sum += *w++ * i[k];
189    }
190    *o++ = genann_act_hidden( ann, sum );
191  }
192  i += ann->inputs;
193  /* Figure hidden layers, if any. */
194  for ( h = 1; h < ann->hidden_layers; ++h ) {
195    for ( j = 0; j < ann->hidden; ++j ) {
196      double sum = *w++ * -1.0;
197      for ( k = 0; k < ann->hidden; ++k ) {
198        sum += *w++ * i[k];
199      }
200      *o++ = genann_act_hidden( ann, sum );
201    }
202    i += ann->hidden;
203  }
204  double const *ret = o;
205  /* Figure output layer. */
206  for ( j = 0; j < ann->outputs; ++j ) {
207    double sum = *w++ * -1.0;
208    for ( k = 0; k < ann->hidden; ++k ) {
209      sum += *w++ * i[k];
210    }
211    *o++ = genann_act_output( ann, sum );
212  }
213  /* Sanity check that we used all weights and wrote all outputs. */
214  assert( w - ann->weight == ann->total_weights );
215  assert( o - ann->output == ann->total_neurons );
216  return ret;
217}
218 
219 
220void genann_train( genann const *ann, double const *inputs, double const *desired_outputs, double learning_rate ) {
221  /* To begin with, we must run the network forward. */
222  genann_run( ann, inputs );
223  int h, j, k;
224  /* First set the output layer deltas. */
225  {
226    double const *o = ann->output + ann->inputs + ann->hidden * ann->hidden_layers; /* First output. */
227    double *d = ann->delta + ann->hidden * ann->hidden_layers; /* First delta. */
228    double const *t = desired_outputs; /* First desired output. */
229    /* Set output layer deltas. */
230    if ( genann_act_output == genann_act_linear ||
231         ann->activation_output == genann_act_linear ) {
232      for ( j = 0; j < ann->outputs; ++j ) {
233        *d++ = *t++ - *o++;
234      }
235    }
236    else {
237      for ( j = 0; j < ann->outputs; ++j ) {
238        *d++ = ( *t - *o ) * *o * ( 1.0 - *o );
239        ++o; ++t;
240      }
241    }
242  }
243  /* Set hidden layer deltas, start on last layer and work backwards. */
244  /* Note that loop is skipped in the case of hidden_layers == 0. */
245  for ( h = ann->hidden_layers - 1; h >= 0; --h ) {
246    /* Find first output and delta in this layer. */
247    double const *o = ann->output + ann->inputs + ( h * ann->hidden );
248    double *d = ann->delta + ( h * ann->hidden );
249 
250    /* Find first delta in following layer (which may be hidden or output). */
251    double const * const dd = ann->delta + ( (h+1) * ann->hidden );
252 
253    /* Find first weight in following layer (which may be hidden or output). */
254    double const * const ww = ann->weight + ((ann->inputs+1) * ann->hidden) + ((ann->hidden+1) * ann->hidden * (h));
255    for ( j = 0; j < ann->hidden; ++j ) {
256      double delta = 0;
257      for ( k = 0; k < ( h == ann->hidden_layers-1 ? ann->outputs : ann->hidden ); ++k ) {
258        const double forward_delta = dd[k];
259        const int windex = k * ( ann->hidden + 1 ) + ( j + 1 );
260        const double forward_weight = ww[windex];
261        delta += forward_delta * forward_weight;
262      }
263      *d = *o * (1.0-*o) * delta;
264      ++d; ++o;
265    }
266  }
267  /* Train the outputs. */
268  {
269    /* Find first output delta. */
270    double const *d = ann->delta + ann->hidden * ann->hidden_layers; /* First output delta. */
271    /* Find first weight to first output delta. */
272    double *w = ann->weight + (ann->hidden_layers
273      ? ( (ann->inputs+1) * ann->hidden + (ann->hidden+1) * ann->hidden * (ann->hidden_layers-1) )
274      : ( 0 ) );
275    /* Find first output in previous layer. */
276    double const * const i = ann->output + (ann->hidden_layers
277      ? ( ann->inputs + (ann->hidden) * (ann->hidden_layers-1) )
278      : 0 );
279    /* Set output layer weights. */
280    for ( j = 0; j < ann->outputs; ++j ) {
281      *w++ += *d * learning_rate * -1.0;
282      for ( k = 1; k < (ann->hidden_layers ? ann->hidden : ann->inputs) + 1; ++k ) {
283        *w++ += *d * learning_rate * i[k-1];
284      }
285      ++d;
286    }
287    assert( w - ann->weight == ann->total_weights );
288  }
289  /* Train the hidden layers. */
290  for ( h = ann->hidden_layers - 1; h >= 0; --h ) {
291    /* Find first delta in this layer. */
292    double const *d = ann->delta + ( h * ann->hidden );
293    /* Find first input to this layer. */
294    double const *i = ann->output + ( h
295      ? ( ann->inputs + ann->hidden * (h-1) )
296      : 0 );
297    /* Find first weight to this layer. */
298    double *w = ann->weight + ( h
299      ? ( (ann->inputs+1) * ann->hidden + (ann->hidden+1) * (ann->hidden) * (h-1) )
300      : 0 );
301    for ( j = 0; j < ann->hidden; ++j ) {
302      *w++ += *d * learning_rate * -1.0;
303      for ( k = 1; k < ( h == 0 ? ann->inputs : ann->hidden ) + 1; ++k ) {
304        *w++ += *d * learning_rate * i[k-1];
305      }
306      ++d;
307    }
308  }
309}
310 
311void genann_write( genann const *ann, FILE *out ) {
312  fprintf( out, "%d %d %d %d", ann->inputs, ann->hidden_layers, ann->hidden, ann->outputs );
313 
314  int i;
315  for ( i = 0; i < ann->total_weights; ++i ) {
316    fprintf( out, " %.20e", ann->weight[i] );
317  }
318}




      Q: What did the grape do when it got stepped on?    
      A: It let out a little wine!