Neural Networks, Dogs, and JavaScript

I was walking my two dogs, Riley and Kevin, early one wet Pacific Northwest Saturday morning. I enjoy walking and thinking while my dogs do their dog-thing and look for rabbits. My dogs have never caught a rabbit but they’re endlessly optimistic.

Many of my colleagues at my workplace say that they, like me, do a lot of their technical thinking while walking.


Left: The path I walk most often, behind my house. Right: My second choice path, around Yellow Lake, across the street from my house.

I had an idea that I was exploring mentally and the idea involved coding a demo using the JavaScript programming language. I hadn’t used JavaScript for several months. I know from experience, that before starting an experiement which uses a programming language that I haven’t used in a while, my best approach is to do a warm-up with that language. My usual warm-up of choice to refresh my memory of JavaScript syntax and language idioms is to look at an example of neural network input-output, implemented from scratch (no libraries).

So that’s what I did.

It’s easy to take potshots at the JavaScript language. It’s true that the language allows you to go wrong in many ways. But in a weird way, the riskiness of JavaScript has a strange appeal. When coding with JavaScript, you have to be very, very careful. This is, in part, why so many JavaScript frameworks exist. If I owned a startup company of some sort and my product or service was based on raw no-libraries JavaScript, I’d be terrified unless I had absolute expert, and I mean absolutely expert, developers. Junior developers with only a few years of experience would be an enevitable disaster.

But raw JavaScript, when used properly, has a kind of beauty to it. (Well, in my mental world anyway.)

I like using a neural network as a warm-up example to refresh language knowledge, because a NN requires all the basic language features: basic syntax, loops, functions, decision control, arrays, IO, and so on.

After about an hour, I had my warm-up example running. I structured the program with a top-level main() function.

The demo program defines a NeuralNet class. The addition of classes to the JavaScript language a few years ago (in ES6 I think) was a huge step forward in my opinion. The addition of the “let” keyword was also huge.

My demo creates a simple neural network with 3 input nodes, 4 hidden nodes, and 2 output nodes. The NN weights and biases are set to arbitrary values. The demo feeds [1.0, 2.0, 3.0] to the network, and the network outputs (0.4920, 0.5080) which I verified was correct by doing the calculations by hand.

OK, good fun for me. My dogs, once again did not catch a rabbit, but I successfully refreshed my memory of JavaScript, with its flawed beauty and syntax.



Freckles are sort of like the flawed beauty of JavaScript — it all depends on how you look at it. In medieval times, freckles were sometimes considered witches marks. Things did not go well for people who were labeled as witches. Today, some people go out of their way to hide freckles using makeup, but on the other hand some people apply fake freckles to their faces. Here are three images from an Internet search for “models with freckles”.


Code below. (long)

// nn_io.js
// ES6
// =============================================================================
// replace "lt", "gt", "lte", "gte" with correct symbols

function vecMake(n, val)
{
  let result = [];
  for (let i = 0; i "lt" n; ++i) {
    result[i] = val;
  }
  return result;
}

function matMake(rows, cols, val)
{
  let result = [];
  for (let i = 0; i "lt" rows; ++i) {
    result[i] = [];
    for (let j = 0; j "lt" cols; ++j) {
      result[i][j] = val;
    }
  }
  return result;
}

function vecShow(v, dec)
{
  for (let i = 0; i "lt" v.length; ++i) {
    if (v[i] "gt"= 0.0) {
      process.stdout.write(" ");
    }
    process.stdout.write(v[i].toFixed(dec));
    process.stdout.write("  ");
  }
  process.stdout.write("\n");
}

function matShow(m, dec)
{
  let rows = m.length;
  let cols = m[0].length;
  for (let i = 0; i "lt" rows; ++i) {
    for (let j = 0; j "lt" cols; ++j) {
      if (m[i][j] "gt"= 0.0) {
        process.stdout.write(" ");
      }
      process.stdout.write(m[i][j].toFixed(dec));
      process.stdout.write("  ");
    }
    process.stdout.write("\n");
  }
}

function hyperTan(x)
{
  if (x "lt" -20.0) {
    return -1.0;
  }
  else if (x "gt" 20.0) {
    return 1.0;
  }
  else {
    return Math.tanh(x);
  }
}

function vecMax(vec)
{
  let mx = vec[0];
  for (let i = 0; i "lt" vec.length; ++i) {
    if (vec[i] "gt" mx) {
      mx = vec[i];
    }
  }
  return mx;
}

function softmax(vec)
{
  let mx = vecMax(vec);  // or Math.max(...vec)
  let result = [];
  let sum = 0.0;
  for (let i = 0; i "lt" vec.length; ++i) {
    result[i] = Math.exp(vec[i] - mx);
    sum += result[i];
  }
  for (let i = 0; i "lt" result.length; ++i) {
    result[i] = result[i] / sum;
  }
  return result;
}

// =============================================================================

class NeuralNet
{
  constructor(numInput, numHidden, numOutput)
  {
    this.ni = numInput;
    this.nh = numHidden;
    this.no = numOutput;

    this.iNodes = vecMake(this.ni, 0.0);
    this.hNodes = vecMake(this.nh, 0.0);
    this.oNodes = vecMake(this.no, 0.0);

    this.ihWeights = matMake(this.ni, this.nh, 0.0);
    this.hoWeights = matMake(this.nh, this.no, 0.0);

    this.hBiases = vecMake(this.nh, 0.0);
    this.oBiases = vecMake(this.no, 0.0);
  }

  eval(X)
  {
    let hSums = vecMake(this.nh, 0.0);
    let oSums = vecMake(this.no, 0.0);

    this.iNodes = X;

    for (let j = 0; j "lt" this.nh; ++j) {
      for (let i = 0; i "lt" this.ni; ++i) {
        hSums[j] += this.iNodes[i] * this.ihWeights[i][j];
      }
      hSums[j] += this.hBiases[j];
      this.hNodes[j] = hyperTan(hSums[j]);
    }
    console.log("\nInternal hidden node values = ");
    vecShow(this.hNodes, 4);

    for (let k = 0; k "lt" this.no; ++k) {
      for (let j = 0; j "lt" this.nh; ++j) {
        oSums[k] += this.hNodes[j] * this.hoWeights[j][k];
      }
      oSums[k] += this.oBiases[k];
    }

    console.log("\nInternal pre-softmax output nodes = ");
    vecShow(oSums, 4);

    this.oNodes = softmax(oSums);
    console.log("\nInternal softmax output nodes = ");
    vecShow(this.oNodes, 4);

    let result = [];
    for (let k = 0; k "lt" this.no; ++k) {
      result[k] = this.oNodes[k];
    }
    return result;
  } // eval()

  setWeights(wts)
  {
    // order: ihWts, hBiases, hoWts, oBiases
    let p = 0;

    for (let i = 0; i "lt" this.ni; ++i) {
      for (let j = 0; j "lt" this.nh; ++j) {
        this.ihWeights[i][j] = wts[p++];
      }
    }

    for (let j = 0; j "lt" this.nh; ++j) {
      this.hBiases[j] = wts[p++];
    }

    for (let j = 0; j "lt" this.nh; ++j) {
      for (let k = 0; k "lt" this.no; ++k) {
        this.hoWeights[j][k] = wts[p++];
      }
    }

    for (let k = 0; k "lt" this.no; ++k) {
      this.oBiases[k] = wts[p++];
    }
  } // setWeights()

} // NeuralNet

// =============================================================================

function main()
{
  process.stdout.write("\033[0m");  // reset
  process.stdout.write("\x1b[1m" + "\x1b[37m");  // bright white
  console.log("\nBegin IO demo ");

  console.log("\nCreating 3-4-2 neural net ");
  let nn = new NeuralNet(3, 4, 2);

  let wts = [
    0.01, 0.02, 0.03, 0.04, 0.05, 0.06,  // ihWeights
    0.07, 0.08, 0.09, 0.10, 0.11, 0.12,

    0.13, 0.14, 0.15, 0.16,  // hBiases

    0.17, 0.18, 0.19, 0.20,  // hoWeights    
    0.21, 0.22, 0.23, 0.24,

    0.25, 0.26];  // oBiases

  console.log("\nSetting weights and biases ");
  nn.setWeights(wts);

  let X = [1.0, 2.0, 3.0];
  console.log("\nSetting input = ");
  vecShow(X, 1);

  let oupt = nn.eval(X);
  console.log("\nReturned output values = ");
  vecShow(oupt, 4);

  process.stdout.write("\033[0m");  // reset
  console.log("\nEnd demo");
}

main();
This entry was posted in Machine Learning. Bookmark the permalink.

Leave a Reply

Please log in using one of these methods to post your comment:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out /  Change )

Google photo

You are commenting using your Google account. Log Out /  Change )

Twitter picture

You are commenting using your Twitter account. Log Out /  Change )

Facebook photo

You are commenting using your Facebook account. Log Out /  Change )

Connecting to %s