From c8a62f1bea372129483e670d0697513f4ef5b889 Mon Sep 17 00:00:00 2001 From: Ashlynne Juniper <160202125+voidvoxel@users.noreply.github.com> Date: Thu, 13 Jun 2024 08:54:36 -0500 Subject: [PATCH] Add a general-purpose autoencoder class `AE` to serve as a building block for other autoencoder models (#932) * Port the AE from `@voidvoxel/auto-encoder` * Rewrite class `AutoEncoder` from scratch * Remove unused property * Use shallow clone * Add unit tests * Replace `shallowClone` with `deepClone` * Rename private properties * Remove the space in the word "autoencoder" [Wikipedia said "Autoencoder" and not "Auto encoder" or "Auto-encoder"](https://en.wikipedia.org/wiki/Autoencoder) * Rename class `Autoencoder` to `AE` The other classes in this library use their respective acronyms with the exceptions of `FeedForward`, `NeuralNetwork`, and `NeuralNetworkGPU`. Furthermore, `AE` variants of existing classes will likely be made, so an acronym equivalent would be desirable. I'm considering the naming conventions for classes such as `LSTMTimeStep`. For example maybe a future `VAE` class could be made to represent variational autoencoders. * Add `AE` usage to README.md * Update references to `AE` * Use Partial instead of nullable properties * Update autoencoder.ts * Minor improvements * Fix "@rollup/plugin-typescript TS2807" * Choose a more accurate name for `includesAnomalies` (`likelyIncludesAnomalies`, as it makes no guarantees that anomalies are truly present and only provides an intuitive guess) --- README.md | 52 ++++- src/autoencoder.test.ts | 79 ++++++++ src/autoencoder.ts | 189 +++++++++++++++++++ src/errors/untrained-neural-network-error.ts | 7 + src/index.ts | 2 + 5 files changed, 328 insertions(+), 1 deletion(-) create mode 100644 src/autoencoder.test.ts create mode 100644 src/autoencoder.ts create mode 100644 src/errors/untrained-neural-network-error.ts diff --git a/README.md b/README.md index e12f0413..2d07e04c 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ GPU accelerated Neural networks in JavaScript for Browsers and Node.js ![CI](https://github.com/BrainJS/brain.js/workflows/CI/badge.svg) [![codecov](https://codecov.io/gh/BrainJS/brain.js/branch/master/graph/badge.svg?token=3SJIBJ1679)](https://codecov.io/gh/BrainJS/brain.js) Twitter - + [![NPM](https://nodei.co/npm/brain.js.png?compact=true)](https://nodei.co/npm/brain.js/)

@@ -43,6 +43,7 @@ GPU accelerated Neural networks in JavaScript for Browsers and Node.js - [For training with NeuralNetwork](#for-training-with-neuralnetwork) - [For training with `RNNTimeStep`, `LSTMTimeStep` and `GRUTimeStep`](#for-training-with-rnntimestep-lstmtimestep-and-grutimestep) - [For training with `RNN`, `LSTM` and `GRU`](#for-training-with-rnn-lstm-and-gru) + - [For training with `AE`](#for-training-with-ae) - [Training Options](#training-options) - [Async Training](#async-training) - [Cross Validation](#cross-validation) @@ -317,6 +318,54 @@ net.train([ const output = net.run('I feel great about the world!'); // 'happy' ``` +#### For training with `AE` + +Each training pattern can either: + +- Be an array of numbers +- Be an array of arrays of numbers + +Training an autoencoder to compress the values of a XOR calculation: + +```javascript +const net = new brain.AE( + { + hiddenLayers: [ 5, 2, 5 ] + } +); + +net.train([ + [ 0, 0, 0 ], + [ 0, 1, 1 ], + [ 1, 0, 1 ], + [ 1, 1, 0 ] +]); +``` + +Encoding/decoding: + +```javascript +const input = [ 0, 1, 1 ]; + +const encoded = net.encode(input); +const decoded = net.decode(encoded); +``` + +Denoise noisy data: + +```javascript +const noisyData = [ 0, 1, 0 ]; + +const data = net.denoise(noisyData); +``` + +Test for anomalies in data samples: + +```javascript +const shouldBeFalse = net.includesAnomalies([0, 1, 1]); +const shouldBeTrue = net.includesAnomalies([0, 1, 0]); +``` + ### Training Options `train()` takes a hash of options as its second argument: @@ -595,6 +644,7 @@ The user interface used: - [`brain.NeuralNetwork`](src/neural-network.ts) - [Feedforward Neural Network](https://en.wikipedia.org/wiki/Feedforward_neural_network) with backpropagation - [`brain.NeuralNetworkGPU`](src/neural-network-gpu.ts) - [Feedforward Neural Network](https://en.wikipedia.org/wiki/Feedforward_neural_network) with backpropagation, GPU version +- [`brain.AE`](src/autoencoder.ts) - [Autoencoder or "AE"](https://en.wikipedia.org/wiki/Autoencoder) with backpropogation and GPU support - [`brain.recurrent.RNNTimeStep`](src/recurrent/rnn-time-step.ts) - [Time Step Recurrent Neural Network or "RNN"](https://en.wikipedia.org/wiki/Recurrent_neural_network) - [`brain.recurrent.LSTMTimeStep`](src/recurrent/lstm-time-step.ts) - [Time Step Long Short Term Memory Neural Network or "LSTM"](https://en.wikipedia.org/wiki/Long_short-term_memory) - [`brain.recurrent.GRUTimeStep`](src/recurrent/gru-time-step.ts) - [Time Step Gated Recurrent Unit or "GRU"](https://en.wikipedia.org/wiki/Gated_recurrent_unit) diff --git a/src/autoencoder.test.ts b/src/autoencoder.test.ts new file mode 100644 index 00000000..7838fe1e --- /dev/null +++ b/src/autoencoder.test.ts @@ -0,0 +1,79 @@ +import AE from "./autoencoder"; + +const trainingData = [ + [0, 0, 0], + [0, 1, 1], + [1, 0, 1], + [1, 1, 0] +]; + +const xornet = new AE( + { + decodedSize: 3, + hiddenLayers: [ 5, 2, 5 ] + } +); + +const errorThresh = 0.011; + +const result = xornet.train( + trainingData, { + iterations: 100000, + errorThresh + } +); + +test( + "denoise a data sample", + async () => { + expect(result.error).toBeLessThanOrEqual(errorThresh); + + function xor(...args: number[]) { + return Math.round(xornet.denoise(args)[2]); + } + + const run1 = xor(0, 0, 0); + const run2 = xor(0, 1, 1); + const run3 = xor(1, 0, 1); + const run4 = xor(1, 1, 0); + + expect(run1).toBe(0); + expect(run2).toBe(1); + expect(run3).toBe(1); + expect(run4).toBe(0); + } +); + +test( + "encode and decode a data sample", + async () => { + expect(result.error).toBeLessThanOrEqual(errorThresh); + + const run1$input = [0, 0, 0]; + const run1$encoded = xornet.encode(run1$input); + const run1$decoded = xornet.decode(run1$encoded); + + const run2$input = [0, 1, 1]; + const run2$encoded = xornet.encode(run2$input); + const run2$decoded = xornet.decode(run2$encoded); + + for (let i = 0; i < 3; i++) expect(Math.round(run1$decoded[i])).toBe(run1$input[i]); + for (let i = 0; i < 3; i++) expect(Math.round(run2$decoded[i])).toBe(run2$input[i]); + } +); + +test( + "test a data sample for anomalies", + async () => { + expect(result.error).toBeLessThanOrEqual(errorThresh); + + function includesAnomalies(...args: number[]) { + expect(xornet.likelyIncludesAnomalies(args)).toBe(false); + } + + includesAnomalies(0, 0, 0); + includesAnomalies(0, 1, 1); + includesAnomalies(1, 0, 1); + includesAnomalies(1, 1, 0); + } +); diff --git a/src/autoencoder.ts b/src/autoencoder.ts new file mode 100644 index 00000000..e799b042 --- /dev/null +++ b/src/autoencoder.ts @@ -0,0 +1,189 @@ +import { KernelOutput, Texture, TextureArrayOutput } from "gpu.js"; +import { IJSONLayer, INeuralNetworkData, INeuralNetworkDatum, INeuralNetworkTrainOptions } from "./neural-network"; +import { INeuralNetworkGPUOptions, NeuralNetworkGPU } from "./neural-network-gpu"; +import { INeuralNetworkState } from "./neural-network-types"; +import { UntrainedNeuralNetworkError } from "./errors/untrained-neural-network-error"; + +export interface IAEOptions { + binaryThresh: number; + decodedSize: number; + hiddenLayers: number[]; +} + +/** + * An autoencoder learns to compress input data down to relevant features and reconstruct input data from its compressed representation. + */ +export class AE { + private decoder?: NeuralNetworkGPU; + private denoiser: NeuralNetworkGPU; + + constructor ( + options?: Partial + ) { + // Create default options for the autoencoder. + options ??= {}; + + // Create default options for the autoencoder's denoiser subnet. + const denoiserOptions: Partial = {}; + + // Inherit the binary threshold of the parent autoencoder. + denoiserOptions.binaryThresh = options.binaryThresh; + // Inherit the hidden layers of the parent autoencoder. + denoiserOptions.hiddenLayers = options.hiddenLayers; + + // Define the denoiser subnet's input and output sizes. + if (options.decodedSize) denoiserOptions.inputSize = denoiserOptions.outputSize = options.decodedSize; + + // Create the denoiser subnet of the autoencoder. + this.denoiser = new NeuralNetworkGPU(options); + } + + /** + * Denoise input data, removing any anomalies from the data. + * @param {DecodedData} input + * @returns {DecodedData} + */ + denoise(input: DecodedData): DecodedData { + // Run the input through the generic denoiser. + // This isn't the best denoiser implementation, but it's efficient. + // Efficiency is important here because training should focus on + // optimizing for feature extraction as quickly as possible rather than + // denoising and anomaly detection; there are other specialized topologies + // better suited for these tasks anyways, many of which can be implemented + // by using an autoencoder. + return this.denoiser.run(input); + } + + /** + * Decode `EncodedData` into an approximation of its original form. + * + * @param {EncodedData} input + * @returns {DecodedData} + */ + decode(input: EncodedData): DecodedData { + // If the decoder has not been trained yet, throw an error. + if (!this.decoder) throw new UntrainedNeuralNetworkError(this); + + // Decode the encoded input. + return this.decoder.run(input); + } + + /** + * Encode data to extract features, reduce dimensionality, etc. + * + * @param {DecodedData} input + * @returns {EncodedData} + */ + encode(input: DecodedData): EncodedData { + // If the decoder has not been trained yet, throw an error. + if (!this.denoiser) throw new UntrainedNeuralNetworkError(this); + + // Process the input. + this.denoiser.run(input); + + // Get the auto-encoded input. + let encodedInput: TextureArrayOutput = this.encodedLayer as TextureArrayOutput; + + // If the encoded input is a `Texture`, convert it into an `Array`. + if (encodedInput instanceof Texture) encodedInput = encodedInput.toArray(); + else encodedInput = encodedInput.slice(0); + + // Return the encoded input. + return encodedInput as EncodedData; + } + + /** + * Test whether or not a data sample likely contains anomalies. + * If anomalies are likely present in the sample, returns `true`. + * Otherwise, returns `false`. + * + * @param {DecodedData} input + * @returns {boolean} + */ + likelyIncludesAnomalies(input: DecodedData, anomalyThreshold: number = 0.2): boolean { + // Create the anomaly vector. + const anomalies: number[] = []; + + // Attempt to denoise the input. + const denoised = this.denoise(input); + + // Calculate the anomaly vector. + for (let i = 0; i < (input.length ?? 0); i++) { + anomalies[i] = Math.abs((input as number[])[i] - (denoised as number[])[i]); + } + + // Calculate the sum of all anomalies within the vector. + const sum = anomalies.reduce( + (previousValue, value) => previousValue + value + ); + + // Calculate the mean anomaly. + const mean = sum / (input as number[]).length; + + // Return whether or not the mean anomaly rate is greater than the anomaly threshold. + return mean > anomalyThreshold; + } + + /** + * Train the auto encoder. + * + * @param {DecodedData[]} data + * @param {Partial} options + * @returns {INeuralNetworkState} + */ + train(data: DecodedData[], options?: Partial): INeuralNetworkState { + const preprocessedData: INeuralNetworkDatum, Partial>[] = []; + + for (let datum of data) { + preprocessedData.push( { input: datum, output: datum } ); + } + + const results = this.denoiser.train(preprocessedData, options); + + this.decoder = this.createDecoder(); + + return results; + } + + /** + * Create a new decoder from the trained denoiser. + * + * @returns {NeuralNetworkGPU} + */ + private createDecoder() { + const json = this.denoiser.toJSON(); + + const layers: IJSONLayer[] = []; + const sizes: number[] = []; + + for (let i = this.encodedLayerIndex; i < this.denoiser.sizes.length; i++) { + layers.push(json.layers[i]); + sizes.push(json.sizes[i]); + } + + json.layers = layers; + json.sizes = sizes; + + json.options.inputSize = json.sizes[0]; + + const decoder = new NeuralNetworkGPU().fromJSON(json); + + return decoder as unknown as NeuralNetworkGPU; + } + + /** + * Get the layer containing the encoded representation. + */ + private get encodedLayer(): KernelOutput { + return this.denoiser.outputs[this.encodedLayerIndex]; + } + + /** + * Get the offset of the encoded layer. + */ + private get encodedLayerIndex(): number { + return Math.round(this.denoiser.outputs.length * 0.5) - 1; + } +} + +export default AE; diff --git a/src/errors/untrained-neural-network-error.ts b/src/errors/untrained-neural-network-error.ts new file mode 100644 index 00000000..a0f87007 --- /dev/null +++ b/src/errors/untrained-neural-network-error.ts @@ -0,0 +1,7 @@ +export class UntrainedNeuralNetworkError extends Error { + constructor ( + neuralNetwork: any + ) { + super(`Cannot run a ${neuralNetwork.constructor.name} before it is trained.`); + } +} diff --git a/src/index.ts b/src/index.ts index edbd05da..1d410f76 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,5 @@ import * as activation from './activation'; +import { AE } from './autoencoder'; import CrossValidate from './cross-validate'; import { FeedForward } from './feed-forward'; import * as layer from './layer'; @@ -53,6 +54,7 @@ const utilities = { export { activation, + AE, CrossValidate, likely, layer,