Genann, a Minimal ANN (Cont.)

Genann is self-contained in two files: genann.c and genann.h . test.c is to train the ANN on the XOR function using backpropagation. The ANN taking 2 inputs, having 1 layer of 3 hidden neurons, and providing 2 outputs. We then train it on a set of labeled data using backpropagation and ask it to predict on a test data point.

The following gives the code synopsis of this implementation:

Code synopsis 
#include "genann.h"

/* Not shown, loading your training and test data. */
double **training_data_input, **training_data_output, **test_data_input;

/* New network with 2 inputs,
 * 1 hidden layer of 3 neurons each,
 * and 2 outputs. */
genann *ann = genann_init( 2, 1, 3, 2 );

/* Learn on the training set. */
for ( i = 0; i < 300; ++i ) {
  for ( j = 0; j < 100; ++j )
    genann_train( ann, training_data_input[j], training_data_output[j], 0.1 );
}

/* Run the network and see what it predicts. */
double const *prediction = genann_run( ann, test_data_input[0] );
printf( "Output is: %f, %f\n", prediction[0], prediction[1] );

genann_free(ann);

Creating and Freeing ANNs
The following gives the code for creating and freeing the ANN: 

genann *genann_init( int inputs, int hidden_layers, int hidden, int outputs );
genann *genann_copy( genann const *ann );
void genann_free( genann *ann );
Training:   0 XOR 0 =   |   0 XOR 1 =   |   1 XOR 0 =   |   1 XOR 1 =

Testing:     XOR  

         
test.c 
/**********************************************************
 *                                                        *
 *  shell> gcc -lm -o test test.c genann.h genann.c       *
 *                                                        *
 **********************************************************/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "genann.h"

int main( int argc, char *argv[ ] ) {
  printf( "Train a small ANN on the XOR function using backpropagation." );

  /* This will make the neural network initialize differently each run. */
  /* If you don't get a good result, try again for a different result. */
  srand( time(0) );

  /* Input and expected out data for the XOR function. */
  const double input[4][2] = {{0, 0}, {0, 1}, {1, 0}, {1, 1}};
  const double output[4]   = {0, 1, 1, 0};
  double t1 = atoi( argv[1] );
  double t2 = atoi( argv[2] );
  double t3 = atoi( argv[3] );
  double t4 = atoi( argv[4] );

  /* New network with 2 inputs,
   * 1 hidden layer of 2 neurons, and 1 output. */
  genann *ann = genann_init( 2, 1, 2, 1 );

  /* Train on the four labeled data points many times. */
  int i;
  for ( i = 0; i < 300; ++i ) {
    genann_train( ann, input[0], &t1, 3 );
    genann_train( ann, input[1], &t2, 3 );
    genann_train( ann, input[2], &t3, 3 );
    genann_train( ann, input[3], &t4, 3 );
  }

  /* Run the network and see what it predicts. */
  if      ( ( strcmp(argv[5],"0") == 0 ) && ( strcmp(argv[6],"0") == 0 ) )
    printf( "0 XOR 0 = %1.f.\n", *genann_run( ann, input[0] ) );
  else if ( ( strcmp(argv[5],"0") == 0 ) && ( strcmp(argv[6],"1") == 0 ) )
    printf( "0 XOR 1 = %1.f.\n", *genann_run( ann, input[1] ) );
  else if ( ( strcmp(argv[5],"1") == 0 ) && ( strcmp(argv[6],"0") == 0 ) )
    printf( "1 XOR 0 = %1.f.\n", *genann_run( ann, input[2] ) );
  else if ( ( strcmp(argv[5],"1") == 0 ) && ( strcmp(argv[6],"1") == 0 ) )
    printf( "1 XOR 1 = %1.f.\n", *genann_run( ann, input[3] ) );

  genann_free( ann );
  return 0;
}
genann.h 
/*
 * GENANN - Minimal C Artificial Neural Network
 *
 * Copyright (c) 2015-2018 Lewis Van Winkle
 *
 * http://CodePlea.com
 *
 */

#ifndef GENANN_H
#define GENANN_H

#include <stdio.h>

#ifdef __cplusplus
extern "C" {
#endif

#ifndef GENANN_RANDOM
/* We use the following for uniform random numbers between 0 and 1.
 * If you have a better function, redefine this macro. */
#define GENANN_RANDOM( ) ( ( (double) rand( ) ) / RAND_MAX )
#endif

struct genann;

typedef double (*genann_actfun) ( const struct genann *ann, double a );

typedef struct genann {
  /* How many inputs, outputs, and hidden neurons. */
  int inputs, hidden_layers, hidden, outputs;

  /* Which activation function to use for hidden neurons. Default: gennann_act_sigmoid_cached */
  genann_actfun activation_hidden;

  /* Which activation function to use for output. Default: gennann_act_sigmoid_cached */
  genann_actfun activation_output;

  /* Total number of weights, and size of weights buffer. */
  int total_weights;

  /* Total number of neurons + inputs and size of output buffer. */
  int total_neurons;

  /* All weights ( total_weights long ). */
  double *weight;

  /* Stores input array and output of each neuron (total_neurons long). */
  double *output;

  /* Stores delta of each hidden and output neuron (total_neurons - inputs long). */
  double *delta;

} genann;


/* Creates and returns a new ann. */
genann *genann_init( int inputs, int hidden_layers, int hidden, int outputs );

/* Creates ANN from file saved with genann_write. */
genann *genann_read( FILE *in );

/* Sets weights randomly. Called by init. */
void genann_randomize( genann *ann );

/* Returns a new copy of ann. */
genann *genann_copy( genann const *ann );

/* Frees the memory used by an ann. */
void genann_free( genann *ann );

/* Runs the feedforward algorithm to calculate the ann's output. */
double const *genann_run( genann const *ann, double const *inputs );

/* Does a single backprop update. */
void genann_train( genann const *ann, double const *inputs, double const *desired_outputs, double learning_rate );

/* Saves the ann. */
void genann_write( genann const *ann, FILE *out );

void   genann_init_sigmoid_lookup( const genann *ann );
double genann_act_sigmoid( const genann *ann, double a );
double genann_act_sigmoid_cached( const genann *ann, double a );
double genann_act_threshold( const genann *ann, double a );
double genann_act_linear( const genann *ann, double a );

#ifdef __cplusplus
}
#endif

#endif /*GENANN_H*/
genann.c 
#include "genann.h"

#include <assert.h>
#include <errno.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#ifndef genann_act
#define genann_act_hidden genann_act_hidden_indirect
#define genann_act_output genann_act_output_indirect
#else
#define genann_act_hidden genann_act
#define genann_act_output genann_act
#endif

#define LOOKUP_SIZE 4015.

double genann_act_hidden_indirect( const struct genann *ann, double a ) {
  return ann->activation_hidden( ann, a );
}

double genann_act_output_indirect( const struct genann *ann, double a ) {
  return ann->activation_output( ann, a );
}

const double sigmoid_dom_min = -15.0;
const double sigmoid_dom_max = 15.0;
double interval;
double lookup[LOOKUP_SIZE];

#ifdef __GNUC__
#define likely(x)       __builtin_expect(!!(x), 1)
#define unlikely(x)     __builtin_expect(!!(x), 0)
#define unused          __attribute__((unused))
#else
#define likely(x)       x
#define unlikely(x)     x
#define unused
#pragma warning(disable : 415.6) /* For fscanf */
#endif

double genann_act_sigmoid( const genann *ann unused, double a ) {
  if ( a < -45.0 ) return 0;
  if ( a >  45.0 ) return 1;
  return 1.0 / ( 1 + exp( -a ) );
}

void genann_init_sigmoid_lookup( const genann *ann ) {
  const double f = (sigmoid_dom_max - sigmoid_dom_min) / LOOKUP_SIZE;
  int i;
  interval = LOOKUP_SIZE / ( sigmoid_dom_max - sigmoid_dom_min );
  for ( i = 0; i < LOOKUP_SIZE; ++i ) {
    lookup[i] = genann_act_sigmoid( ann, sigmoid_dom_min + f * i );
  }
}

double genann_act_sigmoid_cached( const genann *ann unused, double a ) {
  assert( !isnan( a ) );
  if ( a < sigmoid_dom_min ) return lookup[0];
  if ( a >= sigmoid_dom_max   ) return lookup[LOOKUP_SIZE - 1];
  size_t j = (size_t)( ( a-sigmoid_dom_min ) * interval + 0.5 );
  /* Because floating point... */
  if ( unlikely( j >= LOOKUP_SIZE ) ) return lookup[LOOKUP_SIZE - 1];
  return lookup[j];
}

double genann_act_linear( const struct genann *ann unused, double a ) {
  return a;
}

double genann_act_threshold( const struct genann *ann unused, double a ) {
  return a > 0;
}

genann *genann_init( int inputs, int hidden_layers, int hidden, int outputs ) {
  if ( hidden_layers < 0 ) return 0;
  if ( inputs < 1 ) return 0;
  if ( outputs < 1 ) return 0;
  if ( hidden_layers > 0 && hidden < 1 ) return 0;

  const int hidden_weights = hidden_layers ? (inputs+1) * hidden + (hidden_layers-1) * (hidden+1) * hidden : 0;
  const int output_weights = (hidden_layers ? (hidden+1) : (inputs+1)) * outputs;
  const int total_weights = (hidden_weights + output_weights);
  const int total_neurons = (inputs + hidden * hidden_layers + outputs);

  /* Allocate extra size for weights, outputs, and deltas. */
  const int size = sizeof(genann) + sizeof(double) * (total_weights + total_neurons + (total_neurons - inputs));
  genann *ret = malloc(size);
  if ( !ret ) return 0;

  ret->inputs = inputs;
  ret->hidden_layers = hidden_layers;
  ret->hidden = hidden;
  ret->outputs = outputs;

  ret->total_weights = total_weights;
  ret->total_neurons = total_neurons;

  /* Set pointers. */
  ret->weight = (double*) ((char*)ret + sizeof(genann));
  ret->output = ret->weight + ret->total_weights;
  ret->delta = ret->output + ret->total_neurons;

  genann_randomize( ret );
  ret->activation_hidden = genann_act_sigmoid_cached;
  ret->activation_output = genann_act_sigmoid_cached;
  genann_init_sigmoid_lookup( ret );
  return ret;
}

genann *genann_read( FILE *in ) {
  int inputs, hidden_layers, hidden, outputs;
  int rc;

  errno = 0;
  rc = fscanf( in, "%d %d %d %d", &inputs, &hidden_layers, &hidden, &outputs );
  if ( rc < 4 || errno != 0 ) {
    perror( "fscanf" );
    return NULL;
  }
  genann *ann = genann_init( inputs, hidden_layers, hidden, outputs );
  int i;
  for ( i = 0; i < ann->total_weights; ++i ) {
    errno = 0;
    rc = fscanf(in, " %le", ann->weight + i );
    if ( rc < 1 || errno != 0 ) {
      perror( "fscanf" );
      genann_free( ann );
      return NULL;
    }
  }
  return ann;
}

genann *genann_copy( genann const *ann ) {
  const int size = sizeof(genann) + sizeof(double) * (ann->total_weights + ann->total_neurons + (ann->total_neurons - ann->inputs) );
  genann *ret = malloc( size );
  if ( !ret ) return 0;
  memcpy( ret, ann, size );
  /* Set pointers. */
  ret->weight = (double*) ( (char*)ret + sizeof(genann) );
  ret->output = ret->weight + ret->total_weights;
  ret->delta = ret->output + ret->total_neurons;
  return ret;
}

void genann_randomize( genann *ann ) {
  int i;
  for ( i = 0; i < ann->total_weights; ++i ) {
    double r = GENANN_RANDOM( );
    /* Sets weights from -0.5 to 0.5. */
    ann->weight[i] = r - 0.5;
  }
}

void genann_free( genann *ann ) {
  /* The weight, output, and delta pointers go to the same buffer. */
  free( ann );
}

double const *genann_run(genann const *ann, double const *inputs) {
  double const *w = ann->weight;
  double *o = ann->output + ann->inputs;
  double const *i = ann->output;

  /* Copy the inputs to the scratch area, where we also store each neuron's
   * output, for consistency. This way the first layer isn't a special case. */
  memcpy(ann->output, inputs, sizeof(double) * ann->inputs);
  int h, j, k;
  if ( !ann->hidden_layers ) {
    double *ret = o;
    for ( j = 0; j < ann->outputs; ++j ) {
      double sum = *w++ * -1.0;
      for ( k = 0; k < ann->inputs; ++k ) {
        sum += *w++ * i[k];
      }
      *o++ = genann_act_output( ann, sum );
    }
    return ret;
  }

  /* Figure input layer */
  for ( j = 0; j < ann->hidden; ++j ) {
    double sum = *w++ * -1.0;
    for ( k = 0; k < ann->inputs; ++k ) {
      sum += *w++ * i[k];
    }
    *o++ = genann_act_hidden( ann, sum );
  }
  i += ann->inputs;
  /* Figure hidden layers, if any. */
  for ( h = 1; h < ann->hidden_layers; ++h ) {
    for ( j = 0; j < ann->hidden; ++j ) {
      double sum = *w++ * -1.0;
      for ( k = 0; k < ann->hidden; ++k ) {
        sum += *w++ * i[k];
      }
      *o++ = genann_act_hidden( ann, sum );
    }
    i += ann->hidden;
  }
  double const *ret = o;
  /* Figure output layer. */
  for ( j = 0; j < ann->outputs; ++j ) {
    double sum = *w++ * -1.0;
    for ( k = 0; k < ann->hidden; ++k ) {
      sum += *w++ * i[k];
    }
    *o++ = genann_act_output( ann, sum );
  }
  /* Sanity check that we used all weights and wrote all outputs. */
  assert( w - ann->weight == ann->total_weights );
  assert( o - ann->output == ann->total_neurons );
  return ret;
}


void genann_train( genann const *ann, double const *inputs, double const *desired_outputs, double learning_rate ) {
  /* To begin with, we must run the network forward. */
  genann_run( ann, inputs );
  int h, j, k;
  /* First set the output layer deltas. */
  {
    double const *o = ann->output + ann->inputs + ann->hidden * ann->hidden_layers; /* First output. */
    double *d = ann->delta + ann->hidden * ann->hidden_layers; /* First delta. */
    double const *t = desired_outputs; /* First desired output. */
    /* Set output layer deltas. */
    if ( genann_act_output == genann_act_linear ||
         ann->activation_output == genann_act_linear ) {
      for ( j = 0; j < ann->outputs; ++j ) {
        *d++ = *t++ - *o++;
      }
    }
    else {
      for ( j = 0; j < ann->outputs; ++j ) {
        *d++ = ( *t - *o ) * *o * ( 1.0 - *o );
        ++o; ++t;
      }
    }
  }
  /* Set hidden layer deltas, start on last layer and work backwards. */
  /* Note that loop is skipped in the case of hidden_layers == 0. */
  for ( h = ann->hidden_layers - 1; h >= 0; --h ) {
    /* Find first output and delta in this layer. */
    double const *o = ann->output + ann->inputs + ( h * ann->hidden );
    double *d = ann->delta + ( h * ann->hidden );

    /* Find first delta in following layer (which may be hidden or output). */
    double const * const dd = ann->delta + ( (h+1) * ann->hidden );

    /* Find first weight in following layer (which may be hidden or output). */
    double const * const ww = ann->weight + ((ann->inputs+1) * ann->hidden) + ((ann->hidden+1) * ann->hidden * (h));
    for ( j = 0; j < ann->hidden; ++j ) {
      double delta = 0;
      for ( k = 0; k < ( h == ann->hidden_layers-1 ? ann->outputs : ann->hidden ); ++k ) {
        const double forward_delta = dd[k];
        const int windex = k * ( ann->hidden + 1 ) + ( j + 1 );
        const double forward_weight = ww[windex];
        delta += forward_delta * forward_weight;
      }
      *d = *o * (1.0-*o) * delta;
      ++d; ++o;
    }
  }
  /* Train the outputs. */
  {
    /* Find first output delta. */
    double const *d = ann->delta + ann->hidden * ann->hidden_layers; /* First output delta. */
    /* Find first weight to first output delta. */
    double *w = ann->weight + (ann->hidden_layers
      ? ( (ann->inputs+1) * ann->hidden + (ann->hidden+1) * ann->hidden * (ann->hidden_layers-1) )
      : ( 0 ) );
    /* Find first output in previous layer. */
    double const * const i = ann->output + (ann->hidden_layers
      ? ( ann->inputs + (ann->hidden) * (ann->hidden_layers-1) )
      : 0 );
    /* Set output layer weights. */
    for ( j = 0; j < ann->outputs; ++j ) {
      *w++ += *d * learning_rate * -1.0;
      for ( k = 1; k < (ann->hidden_layers ? ann->hidden : ann->inputs) + 1; ++k ) {
        *w++ += *d * learning_rate * i[k-1];
      }
      ++d;
    }
    assert( w - ann->weight == ann->total_weights );
  }
  /* Train the hidden layers. */
  for ( h = ann->hidden_layers - 1; h >= 0; --h ) {
    /* Find first delta in this layer. */
    double const *d = ann->delta + ( h * ann->hidden );
    /* Find first input to this layer. */
    double const *i = ann->output + ( h
      ? ( ann->inputs + ann->hidden * (h-1) )
      : 0 );
    /* Find first weight to this layer. */
    double *w = ann->weight + ( h
      ? ( (ann->inputs+1) * ann->hidden + (ann->hidden+1) * (ann->hidden) * (h-1) )
      : 0 );
    for ( j = 0; j < ann->hidden; ++j ) {
      *w++ += *d * learning_rate * -1.0;
      for ( k = 1; k < ( h == 0 ? ann->inputs : ann->hidden ) + 1; ++k ) {
        *w++ += *d * learning_rate * i[k-1];
      }
      ++d;
    }
  }
}

void genann_write( genann const *ann, FILE *out ) {
  fprintf( out, "%d %d %d %d", ann->inputs, ann->hidden_layers, ann->hidden, ann->outputs );

  int i;
  for ( i = 0; i < ann->total_weights; ++i ) {
    fprintf( out, " %.20e", ann->weight[i] );
  }
}




      “You will not be punished for your anger; you will be punished by your anger.”    
      ― Siddhārtha Gautama