mirror of
https://github.com/hanzoai/jube-fork.git
synced 2026-10-30 18:33:33 +00:00
richard-churchman
cc0a2bee21
Included open-source Accord.net library code natively to the solution given the deprecated state of that project. Accord.net will be supported inside Jube for own needs (such as upgrading to .Net 7+). Accord.net libraries modified to compile to .Net 6. Modified Accord.net libraries to better support Newtonsoft Json.net serialisation. Included annotations in the Accord.net libraries to house important dependency statistics. Serialised json can be used independently of Jube to recall a Neural Network. Removal of Binary Serializer totally as a precursor to the projects support of .Net 7+. Extensive bug fixing in crucial functions of Neural Network training and recall. While this part of the system remains in Alpha, it is substantially more fit for purpose.
431 lines
15 KiB
C#
431 lines
15 KiB
C#
// Accord Neural Net Library
|
|
// The Accord.NET Framework
|
|
// http://accord-framework.net
|
|
//
|
|
// Copyright © César Souza, 2009-2017
|
|
// cesarsouza at gmail.com
|
|
//
|
|
// This library is free software; you can redistribute it and/or
|
|
// modify it under the terms of the GNU Lesser General Public
|
|
// License as published by the Free Software Foundation; either
|
|
// version 2.1 of the License, or (at your option) any later version.
|
|
//
|
|
// This library is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
// Lesser General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Lesser General Public
|
|
// License along with this library; if not, write to the Free Software
|
|
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
|
//
|
|
|
|
namespace Accord.Neuro.Networks
|
|
{
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Runtime.Serialization.Formatters.Binary;
|
|
using Accord.Neuro.ActivationFunctions;
|
|
using Accord.Neuro.Layers;
|
|
using Accord.Neuro.Neurons;
|
|
using Accord.Compat;
|
|
|
|
/// <summary>
|
|
/// Deep Belief Network.
|
|
/// </summary>
|
|
///
|
|
/// <remarks>
|
|
/// The Deep Belief Network can be seen as a collection of stacked
|
|
/// <see cref="RestrictedBoltzmannMachine">Restricted Boltzmann
|
|
/// Machines</see> disposed as layers of a network. In turn, the
|
|
/// whole network can be seen as an stochastic activation network
|
|
/// in which the neurons activate within some given probability.
|
|
/// </remarks>
|
|
///
|
|
[Serializable]
|
|
public class DeepBeliefNetwork : ActivationNetwork
|
|
{
|
|
|
|
private List<RestrictedBoltzmannMachine> machines;
|
|
|
|
/// <summary>
|
|
/// Gets the number of output neurons in the network
|
|
/// (the size of the computed output vectors).
|
|
/// </summary>
|
|
///
|
|
public int OutputCount
|
|
{
|
|
get
|
|
{
|
|
if (machines.Count == 0) return 0;
|
|
return machines[machines.Count - 1].Hidden.Neurons.Length;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the Restricted Boltzmann Machines
|
|
/// on each layer of this deep network.
|
|
/// </summary>
|
|
///
|
|
public IList<RestrictedBoltzmannMachine> Machines
|
|
{
|
|
get { return machines; }
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates a new <see cref="DeepBeliefNetwork"/>.
|
|
/// </summary>
|
|
///
|
|
/// <param name="inputsCount">The number of inputs for the network.</param>
|
|
/// <param name="hiddenNeurons">The number of hidden neurons in each layer.</param>
|
|
///
|
|
public DeepBeliefNetwork(int inputsCount, params int[] hiddenNeurons)
|
|
: this(new BernoulliFunction(alpha: 1), inputsCount, hiddenNeurons) { }
|
|
|
|
|
|
/// <summary>
|
|
/// Creates a new <see cref="DeepBeliefNetwork"/>.
|
|
/// </summary>
|
|
///
|
|
/// <param name="function">The activation function to be used in the network neurons.</param>
|
|
/// <param name="inputsCount">The number of inputs for the network.</param>
|
|
/// <param name="hiddenNeurons">The number of hidden neurons in each layer.</param>
|
|
///
|
|
public DeepBeliefNetwork(IStochasticFunction function, int inputsCount, params int[] hiddenNeurons)
|
|
: base(function, inputsCount, null,hiddenNeurons)
|
|
{
|
|
machines = new List<RestrictedBoltzmannMachine>();
|
|
|
|
// Create first layer
|
|
machines.Add(new RestrictedBoltzmannMachine(
|
|
hidden: new StochasticLayer(function, hiddenNeurons[0], inputsCount),
|
|
visible: new StochasticLayer(function, inputsCount, hiddenNeurons[0]))
|
|
);
|
|
|
|
// Create other layers
|
|
for (int i = 1; i < hiddenNeurons.Length; i++)
|
|
{
|
|
machines.Add(new RestrictedBoltzmannMachine(
|
|
hidden: new StochasticLayer(function, hiddenNeurons[i], hiddenNeurons[i - 1]),
|
|
visible: new StochasticLayer(function, hiddenNeurons[i - 1], hiddenNeurons[i])));
|
|
}
|
|
|
|
// Override AForge layers
|
|
layers = new Layer[machines.Count];
|
|
for (int i = 0; i < layers.Length; i++)
|
|
layers[i] = machines[i].Hidden;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates a new <see cref="DeepBeliefNetwork"/>.
|
|
/// </summary>
|
|
///
|
|
/// <param name="inputsCount">The number of inputs for the network.</param>
|
|
/// <param name="layers">The layers to add to the deep network.</param>
|
|
///
|
|
public DeepBeliefNetwork(int inputsCount, params RestrictedBoltzmannMachine[] layers)
|
|
: base(null, inputsCount, null,new int[layers.Length])
|
|
{
|
|
machines = new List<RestrictedBoltzmannMachine>(layers);
|
|
|
|
// Override AForge layers
|
|
base.layers = new Layer[machines.Count];
|
|
for (int i = 0; i < layers.Length; i++)
|
|
base.layers[i] = machines[i].Hidden;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Computes the network's outputs for a given input.
|
|
/// </summary>
|
|
///
|
|
/// <param name="input">The input vector.</param>
|
|
///
|
|
/// <returns>
|
|
/// Returns the network's output for the given input.
|
|
/// </returns>
|
|
///
|
|
public override double[] Compute(double[] input)
|
|
{
|
|
double[] output = input;
|
|
|
|
foreach (RestrictedBoltzmannMachine layer in machines)
|
|
output = layer.Hidden.Compute(output);
|
|
|
|
return output;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Computes the network's outputs for a given input.
|
|
/// </summary>
|
|
///
|
|
/// <param name="input">The input vector.</param>
|
|
/// <param name="layerIndex">The index of the layer.</param>
|
|
///
|
|
/// <returns>
|
|
/// Returns the network's output for the given input.
|
|
/// </returns>
|
|
///
|
|
public double[] Compute(double[] input, int layerIndex)
|
|
{
|
|
double[] output = input;
|
|
|
|
for (int i = 0; i <= layerIndex; i++)
|
|
output = machines[i].Hidden.Compute(output);
|
|
|
|
return output;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Reconstructs a input vector for a given output.
|
|
/// </summary>
|
|
///
|
|
/// <param name="output">The output vector.</param>
|
|
///
|
|
/// <returns>
|
|
/// Returns a probable input vector which may
|
|
/// have originated the given output.
|
|
/// </returns>
|
|
///
|
|
public double[] Reconstruct(double[] output)
|
|
{
|
|
double[] input = output;
|
|
|
|
for (int i = machines.Count - 1; i >= 0; i--)
|
|
input = machines[i].Visible.Compute(input);
|
|
|
|
return input;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Reconstructs a input vector using the output
|
|
/// vector of a given layer.
|
|
/// </summary>
|
|
///
|
|
/// <param name="output">The output vector.</param>
|
|
/// <param name="layerIndex">The index of the layer.</param>
|
|
///
|
|
/// <returns>
|
|
/// Returns a probable input vector which may
|
|
/// have originated the given output in the
|
|
/// indicated layer.
|
|
/// </returns>
|
|
///
|
|
public double[] Reconstruct(double[] output, int layerIndex)
|
|
{
|
|
double[] input = output;
|
|
|
|
for (int i = layerIndex; i >= 0; i--)
|
|
input = machines[i].Visible.Compute(input);
|
|
|
|
return input;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Samples an output vector from the network
|
|
/// given an input vector.
|
|
/// </summary>
|
|
///
|
|
/// <param name="input">An input vector.</param>
|
|
///
|
|
/// <returns>
|
|
/// A possible output considering the
|
|
/// stochastic activations of the network.
|
|
/// </returns>
|
|
///
|
|
public double[] GenerateOutput(double[] input)
|
|
{
|
|
double[] output = input;
|
|
|
|
foreach (RestrictedBoltzmannMachine layer in machines)
|
|
output = layer.Hidden.Generate(output);
|
|
|
|
return output;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Samples an output vector from the network
|
|
/// given an input vector.
|
|
/// </summary>
|
|
///
|
|
/// <param name="input">An input vector.</param>
|
|
/// <param name="layerIndex">The index of the layer.</param>
|
|
///
|
|
/// <returns>
|
|
/// A possible output considering the
|
|
/// stochastic activations of the network.
|
|
/// </returns>
|
|
///
|
|
public double[] GenerateOutput(double[] input, int layerIndex)
|
|
{
|
|
double[] output = input;
|
|
|
|
for (int i = 0; i <= layerIndex; i++)
|
|
output = machines[i].Hidden.Generate(output);
|
|
|
|
return output;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Samples an input vector from the network
|
|
/// given an output vector.
|
|
/// </summary>
|
|
///
|
|
/// <param name="output">An output vector.</param>
|
|
///
|
|
/// <returns>
|
|
/// A possible reconstruction considering the
|
|
/// stochastic activations of the network.
|
|
/// </returns>
|
|
///
|
|
public double[] GenerateInput(double[] output)
|
|
{
|
|
double[] input = output;
|
|
|
|
for (int i = layers.Length - 1; i >= 0; i--)
|
|
input = machines[i].Visible.Generate(input);
|
|
|
|
return input;
|
|
}
|
|
|
|
|
|
/// <summary>
|
|
/// Inserts a new layer at the end of this network.
|
|
/// </summary>
|
|
///
|
|
/// <param name="neurons">The number of neurons in the new layer.</param>
|
|
///
|
|
public void Push(int neurons)
|
|
{
|
|
Push(neurons, new BernoulliFunction(alpha: 1));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Inserts a new layer at the end of this network.
|
|
/// </summary>
|
|
///
|
|
/// <param name="neurons">The number of neurons in the new layer.</param>
|
|
/// <param name="function">The activation function which should be used by the neurons.</param>
|
|
///
|
|
public void Push(int neurons, IStochasticFunction function)
|
|
{
|
|
Push(neurons, function, function);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Inserts a new layer at the end of this network.
|
|
/// </summary>
|
|
///
|
|
/// <param name="neurons">The number of neurons in the layer.</param>
|
|
/// <param name="visibleFunction">The activation function which should be used by the visible neurons.</param>
|
|
/// <param name="hiddenFunction">The activation function which should be used by the hidden neurons.</param>
|
|
///
|
|
public void Push(int neurons, IStochasticFunction visibleFunction, IStochasticFunction hiddenFunction)
|
|
{
|
|
int lastLayerNeurons;
|
|
|
|
if (machines.Count > 0)
|
|
lastLayerNeurons = machines[machines.Count - 1].Hidden.Neurons.Length;
|
|
else lastLayerNeurons = inputsCount;
|
|
|
|
machines.Add(new RestrictedBoltzmannMachine(
|
|
hidden: new StochasticLayer(hiddenFunction, neurons, lastLayerNeurons),
|
|
visible: new StochasticLayer(visibleFunction, lastLayerNeurons, neurons)));
|
|
|
|
// Override AForge layers
|
|
layers = new Layer[machines.Count];
|
|
for (int i = 0; i < layers.Length; i++)
|
|
layers[i] = machines[i].Hidden;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Stacks a new Boltzmann Machine at the end of this network.
|
|
/// </summary>
|
|
///
|
|
/// <param name="network">The machine to be added to the network.</param>
|
|
///
|
|
public void Push(RestrictedBoltzmannMachine network)
|
|
{
|
|
int lastLayerNeurons;
|
|
|
|
if (machines.Count > 0)
|
|
lastLayerNeurons = machines[machines.Count - 1].Hidden.Neurons.Length;
|
|
else lastLayerNeurons = inputsCount;
|
|
|
|
machines.Add(network);
|
|
|
|
// Override AForge layers
|
|
layers = new Layer[machines.Count];
|
|
for (int i = 0; i < layers.Length; i++)
|
|
layers[i] = machines[i].Hidden;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Removes the last layer from the network.
|
|
/// </summary>
|
|
///
|
|
public void Pop()
|
|
{
|
|
if (machines.Count == 0)
|
|
return;
|
|
|
|
machines.RemoveAt(machines.Count - 1);
|
|
|
|
// Override AForge layers
|
|
layers = new Layer[machines.Count];
|
|
for (int i = 0; i < layers.Length; i++)
|
|
layers[i] = machines[i].Hidden;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Updates the weights of the visible layers by copying
|
|
/// the reverse of the weights in the hidden layers.
|
|
/// </summary>
|
|
///
|
|
public void UpdateVisibleWeights()
|
|
{
|
|
foreach (var machine in machines)
|
|
machine.UpdateVisibleWeights();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates a Gaussian-Bernoulli network.
|
|
/// </summary>
|
|
///
|
|
/// <param name="inputsCount">The number of inputs for the network.</param>
|
|
/// <param name="hiddenNeurons">The number of hidden neurons in each layer.</param>
|
|
///
|
|
public static DeepBeliefNetwork CreateGaussianBernoulli(int inputsCount, params int[] hiddenNeurons)
|
|
{
|
|
DeepBeliefNetwork network = new DeepBeliefNetwork(inputsCount, hiddenNeurons);
|
|
|
|
GaussianFunction gaussian = new GaussianFunction();
|
|
foreach (StochasticNeuron neuron in network.machines[0].Visible.Neurons)
|
|
neuron.ActivationFunction = gaussian;
|
|
|
|
return network;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates a Mixed-Bernoulli network.
|
|
/// </summary>
|
|
///
|
|
/// <param name="visible">The <see cref="IStochasticFunction"/> to be used in the first visible layer.</param>
|
|
/// <param name="hidden">The <see cref="IStochasticFunction"/> to be used in all other layers.</param>
|
|
///
|
|
/// <param name="inputsCount">The number of inputs for the network.</param>
|
|
/// <param name="hiddenNeurons">The number of hidden neurons in each layer.</param>
|
|
///
|
|
public static DeepBeliefNetwork CreateMixedNetwork(IStochasticFunction visible,
|
|
IStochasticFunction hidden, int inputsCount, params int[] hiddenNeurons)
|
|
{
|
|
DeepBeliefNetwork network = new DeepBeliefNetwork(hidden, inputsCount, hiddenNeurons);
|
|
|
|
foreach (StochasticNeuron neuron in network.machines[0].Visible.Neurons)
|
|
neuron.ActivationFunction = visible;
|
|
|
|
return network;
|
|
}
|
|
}
|
|
}
|