From 3ba20ccfaf9a38e3a10263732ff53964420567d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Old=C5=99ich=20Ko=C5=BEelsk=C3=BD?= Date: Mon, 2 Sep 2019 06:46:36 +0200 Subject: [PATCH] Redesign (final phase) --- Demo/DemoConsoleApp/DemoSettings.xml | 1349 ++++------------- Demo/DemoConsoleApp/Program.cs | 1 - Demo/DemoConsoleApp/Research.cs | 10 + RCNet/Neural/Network/FF/FeedForwardNetwork.cs | 18 +- RCNet/Neural/Network/SM/Neuron/FiringRate.cs | 168 -- .../Neural/Network/SM/Neuron/HiddenNeuron.cs | 88 +- .../SM/Neuron/HiddenNeuronPredictors.cs | 214 +++ .../HiddenNeuronPredictorsSettings.cs} | 20 +- .../HiddenNeuronPredictorsSettings.xsd} | 0 RCNet/Neural/Network/SM/Neuron/InputNeuron.cs | 2 +- .../NeuralPreprocessorSettings.cs | 5 +- .../Network/SM/Preprocessing/PoolSettings.cs | 5 +- .../Network/SM/Preprocessing/Reservoir.cs | 33 +- RCNet/Neural/Network/SM/StateMachine.cs | 2 +- .../Network/SM/Synapse/InternalSynapse.cs | 2 +- RCNet/RCNet.csproj | 6 +- 16 files changed, 611 insertions(+), 1312 deletions(-) delete mode 100644 RCNet/Neural/Network/SM/Neuron/FiringRate.cs create mode 100644 RCNet/Neural/Network/SM/Neuron/HiddenNeuronPredictors.cs rename RCNet/Neural/Network/SM/{Preprocessing/PredictorsSettings.cs => Neuron/HiddenNeuronPredictorsSettings.cs} (91%) rename RCNet/Neural/Network/SM/{Preprocessing/PredictorsSettings.xsd => Neuron/HiddenNeuronPredictorsSettings.xsd} (100%) diff --git a/Demo/DemoConsoleApp/DemoSettings.xml b/Demo/DemoConsoleApp/DemoSettings.xml index 6e7fd8f..27ac388 100644 --- a/Demo/DemoConsoleApp/DemoSettings.xml +++ b/Demo/DemoConsoleApp/DemoSettings.xml @@ -1,6 +1,118 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -57,17 +169,21 @@ - + + + - + + + + + - + @@ -245,9 +363,9 @@ - - - - - - - + + - + + - + - - + + + + + + + + + + + + + + + + + + - - - - + - - - - - + + + + + + @@ -412,19 +546,24 @@ - - - - - - + + + + + + + + + + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + @@ -1587,40 +879,29 @@ - - + - - - - - - - + + - - - - - - + - + - - - + + + - + - + - + @@ -1634,18 +915,24 @@ - - - - - + + - + - + @@ -1654,11 +941,11 @@ - + - + diff --git a/Demo/DemoConsoleApp/Program.cs b/Demo/DemoConsoleApp/Program.cs index 455ae3d..aa88e59 100644 --- a/Demo/DemoConsoleApp/Program.cs +++ b/Demo/DemoConsoleApp/Program.cs @@ -10,7 +10,6 @@ static void Main(string[] args) { //Research - this is not a part of the demo - it is a free playground //(new Research()).Run(); - SMDemo.RunDemo(new ConsoleLog(), @"DemoSettings.xml"); //Standard execution of the Demo try { diff --git a/Demo/DemoConsoleApp/Research.cs b/Demo/DemoConsoleApp/Research.cs index b4bf395..a720c13 100644 --- a/Demo/DemoConsoleApp/Research.cs +++ b/Demo/DemoConsoleApp/Research.cs @@ -34,6 +34,16 @@ public Research() public void Run() { //Activation tests + double fadingSum = 0; + for(int i = 0; i < 1000; i++) + { + fadingSum *= (1d - 0.1); + fadingSum += 1d; + Console.WriteLine(fadingSum); + } + + Console.ReadLine(); + IActivationFunction testAF = ActivationFactory.Create(new SimpleIFSettings(refractoryPeriods:0), new Random(0)); TestActivation(testAF, 100, 3.5, 10, 70); diff --git a/RCNet/Neural/Network/FF/FeedForwardNetwork.cs b/RCNet/Neural/Network/FF/FeedForwardNetwork.cs index b788da8..cbc2e1b 100644 --- a/RCNet/Neural/Network/FF/FeedForwardNetwork.cs +++ b/RCNet/Neural/Network/FF/FeedForwardNetwork.cs @@ -389,12 +389,8 @@ public Layer(int numOfNeurons, IActivationFunction activation) { throw new ArgumentOutOfRangeException("numOfNeurons", $"Invalid parameter value: {numOfNeurons}"); } - if (activation == null) - { - throw new ArgumentException("activation", "Activation can't be null"); - } //Setup - Activation = activation; + Activation = activation ?? throw new ArgumentException("activation", "Activation can't be null"); NumOfLayerNeurons = numOfNeurons; NumOfInputNodes = -1; WeightsStartFlatIdx = 0; @@ -430,11 +426,13 @@ internal void FinalizeStructure(int numOfInputNodes, int neuronsFlatStartIdx, in /// internal Layer DeepClone() { - Layer clone = new Layer(NumOfLayerNeurons, Activation); - clone.NumOfInputNodes = NumOfInputNodes; - clone.WeightsStartFlatIdx = WeightsStartFlatIdx; - clone.BiasesStartFlatIdx = BiasesStartFlatIdx; - clone.NeuronsStartFlatIdx = NeuronsStartFlatIdx; + Layer clone = new Layer(NumOfLayerNeurons, Activation) + { + NumOfInputNodes = NumOfInputNodes, + WeightsStartFlatIdx = WeightsStartFlatIdx, + BiasesStartFlatIdx = BiasesStartFlatIdx, + NeuronsStartFlatIdx = NeuronsStartFlatIdx + }; return clone; } diff --git a/RCNet/Neural/Network/SM/Neuron/FiringRate.cs b/RCNet/Neural/Network/SM/Neuron/FiringRate.cs deleted file mode 100644 index b8b8ac6..0000000 --- a/RCNet/Neural/Network/SM/Neuron/FiringRate.cs +++ /dev/null @@ -1,168 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using RCNet.Extensions; - -namespace RCNet.Neural.Network.SM.Neuron -{ - /// - /// Class converts recent spikes to an exponentially weighted firing rate (0-1) - /// - [Serializable] - class FiringRate - { - //Constants - private const int SpikeBuffLength = sizeof(ulong) * 8; - private const double FadingNumOfFiringsDecay = 0.025; - - //Static members - private static readonly decimal[] _spikeValueCache; - private static readonly decimal[] _sumOfSpikeValues; - private static readonly ulong _highestBitMask; - - //Instance members - //Attribute properties - /// - /// Returns current number of the spikes within the sliding buffer - /// - public int NumOfRecentFirings { get; private set; } - - /// - /// Returns fading count of the firings - /// - public double FadingNumOfFirings { get; private set; } - - //Attributes - private ulong _spikes; - private int _numOfBufferedData; - - /// - /// Static constructor - /// - static FiringRate() - { - _sumOfSpikeValues = new decimal[SpikeBuffLength]; - _sumOfSpikeValues.Populate(0); - _spikeValueCache = new decimal[SpikeBuffLength]; - for(int i = 0; i < SpikeBuffLength; i++) - { - decimal val = (decimal)Math.Exp(-i); - _spikeValueCache[i] = val; - _sumOfSpikeValues[i] = (i > 0 ? _sumOfSpikeValues[i - 1] : 0) + val; - } - - _highestBitMask = 1; - _highestBitMask <<= (SpikeBuffLength - 1); - return; - } - - /// - /// Creates new initialized instance - /// - public FiringRate() - { - Reset(); - return; - } - - //Methods - /// - /// Resets internal state to initial state - /// - public void Reset() - { - _spikes = 0; - _numOfBufferedData = 0; - NumOfRecentFirings = 0; - return; - } - - /// - /// Updates internal state - /// - /// - public void Update(bool spike) - { - NumOfRecentFirings -= (_spikes & _highestBitMask) > 0 ? 1 : 0; - FadingNumOfFirings *= (1d - FadingNumOfFiringsDecay); - _spikes <<= 1; - if (spike) - { - _spikes |= 1; - ++NumOfRecentFirings; - ++FadingNumOfFirings; - } - if (_numOfBufferedData < SpikeBuffLength) - { - ++_numOfBufferedData; - } - return; - } - - /// - /// Returns recent exponentially weighted firing rate - /// - /// Average exponentially weighted firing rate between 0 and 1 - public double GetRecentExpWRate() - { - if(_numOfBufferedData == 0) - { - return 0; - } - decimal rate = 0; - ulong localCopy = _spikes; - for(int i = 0; i < _numOfBufferedData; i++) - { - if((localCopy & 1) > 0) - { - rate += _spikeValueCache[i]; - } - localCopy >>= 1; - } - //Average rate - return (double)(rate / _sumOfSpikeValues[_numOfBufferedData - 1]); - } - - /// - /// Returns recent firing frequency - /// - /// Firing frequency between 0 and 1 - public double GetRecentFrequency() - { - if (_numOfBufferedData == 0) - { - return 0; - } - return (double)NumOfRecentFirings / _numOfBufferedData; - } - - /// - /// Returns last spikes as an integer number where bits are in order that the higher bit represents the more recent spike - /// - /// Last spikes history as an integer - public ulong GetLastSpikes(int histLength) - { - if (_numOfBufferedData == 0 || histLength < 1) - { - return 0; - } - if(histLength > _numOfBufferedData) - { - histLength = _numOfBufferedData; - } - ulong localCopy = _spikes; - ulong result = 0; - for (int i = 0; i < histLength; i++) - { - result <<= 1; - result += localCopy & 1; - localCopy >>= 1; - } - return result; - } - - }//FiringRate - -}//Namespace diff --git a/RCNet/Neural/Network/SM/Neuron/HiddenNeuron.cs b/RCNet/Neural/Network/SM/Neuron/HiddenNeuron.cs index fbde70b..da788b6 100644 --- a/RCNet/Neural/Network/SM/Neuron/HiddenNeuron.cs +++ b/RCNet/Neural/Network/SM/Neuron/HiddenNeuron.cs @@ -56,11 +56,9 @@ public class HiddenNeuron : INeuron public bool AfterFirstSpike { get; private set; } /// - /// Configuration of the predictors (enabling/disabling). - /// It can be null. + /// Swithes enabling/disabling neuron's predictors /// - public PredictorsSettings PredictorsCfg { get; private set; } - + public HiddenNeuronPredictorsSettings PredictorsCfg { get; } //Attributes /// @@ -72,7 +70,6 @@ public class HiddenNeuron : INeuron /// Firing /// private readonly double _analogFiringThreshold; - private readonly FiringRate _firingRate; /// /// Stimulation @@ -97,6 +94,11 @@ public class HiddenNeuron : INeuron private double _analogSignal; private double _spikingSignal; + /// + /// Predictors + /// + private readonly HiddenNeuronPredictors _predictors; + //Constructor /// @@ -106,6 +108,7 @@ public class HiddenNeuron : INeuron /// Neuron's role (Excitatory/Inhibitory). /// Instantiated activation function. /// Output signaling restriction. Spiking activation causes output signal always restricted to SpikingOnly. + /// Enabled/Disabled predictors for the neuron. /// Constant bias to be applied. /// A number between 0 and 1 (LT1). Every time the new activation value is higher than the previous activation value by at least the threshold, it is evaluated as a firing event. Ignored in case of spiking activation. /// Strength of the analog neuron's retainment property. Ignored in case of spiking activation. @@ -113,7 +116,7 @@ public HiddenNeuron(NeuronPlacement placement, CommonEnums.NeuronRole role, IActivationFunction activation, CommonEnums.NeuronSignalingRestrictionType signalingRestriction, - PredictorsSettings predictorsCfg, + HiddenNeuronPredictorsSettings predictorsCfg, double bias = 0, double analogFiringThreshold = PoolSettings.NeuronGroupSettings.DefaultAnalogFiringThreshold, double retainmentStrength = 0 @@ -121,13 +124,13 @@ public HiddenNeuron(NeuronPlacement placement, { Placement = placement; Statistics = new NeuronStatistics(); - if(role == CommonEnums.NeuronRole.Input) + if (role == CommonEnums.NeuronRole.Input) { throw new ArgumentException("Role of the hidden neuron can not be Input.", "role"); } Role = role; - PredictorsCfg = predictorsCfg; Bias = bias; + PredictorsCfg = predictorsCfg; //Activation specific _activation = activation; if (activation.ActivationType == CommonEnums.ActivationType.Spiking) @@ -139,12 +142,12 @@ public HiddenNeuron(NeuronPlacement placement, } else { - //Anaolg + //Analog SignalingRestriction = signalingRestriction; _analogFiringThreshold = analogFiringThreshold; _retainmentStrength = retainmentStrength; } - _firingRate = new FiringRate(); + _predictors = new HiddenNeuronPredictors(predictorsCfg); Reset(false); return; } @@ -163,7 +166,7 @@ public HiddenNeuron(NeuronPlacement placement, public void Reset(bool statistics) { _activation.Reset(); - _firingRate.Reset(); + _predictors.Reset(); _iStimuli = 0; _rStimuli = 0; _tStimuli = 0; @@ -212,7 +215,6 @@ public void ComputeSignal(bool collectStatistics) //Spiking activation _spikingSignal = _activation.Compute(_tStimuli); _activationState = _activation.InternalState; - _firingRate.Update(_spikingSignal > 0); _analogSignal = _spikingSignal; } else @@ -223,9 +225,10 @@ public void ComputeSignal(bool collectStatistics) _activationState = (_retainmentStrength * _activationState) + (1d - _retainmentStrength) * newState; _analogSignal = _outputRange.Rescale(_activationState, _activation.OutputRange); bool firingEvent = (_activationState - prevActivationState) > _analogFiringThreshold; - _firingRate.Update(firingEvent); _spikingSignal = firingEvent ? 1d : 0d; } + //Update predictors + _predictors.Update(_activationState, (_spikingSignal > 0)); //Update statistics if (collectStatistics) { @@ -262,56 +265,7 @@ public double GetSignal(CommonEnums.ActivationType targetActivationType) /// public int CopyPredictorsTo(double[] predictors, int idx) { - if (PredictorsCfg == null || PredictorsCfg.NumOfEnabledPredictors == 0) - { - return 0; - } - if (PredictorsCfg.Activation) - { - predictors[idx] = _activationState; - ++idx; - } - if (PredictorsCfg.SquaredActivation) - { - predictors[idx] = _activationState * _activationState; - ++idx; - } - if (PredictorsCfg.ExpWAvgFiringRate64) - { - predictors[idx] = _firingRate.GetRecentExpWRate(); - ++idx; - } - if (PredictorsCfg.FadingNumOfFirings) - { - predictors[idx] = _firingRate.FadingNumOfFirings; - ++idx; - } - if (PredictorsCfg.NumOfFirings64) - { - predictors[idx] = _firingRate.NumOfRecentFirings; - ++idx; - } - if (PredictorsCfg.LastBin32FiringHist) - { - predictors[idx] = _firingRate.GetLastSpikes(32); - ++idx; - } - if (PredictorsCfg.LastBin16FiringHist) - { - predictors[idx] = _firingRate.GetLastSpikes(16); - ++idx; - } - if (PredictorsCfg.LastBin8FiringHist) - { - predictors[idx] = _firingRate.GetLastSpikes(8); - ++idx; - } - if (PredictorsCfg.LastBin1FiringHist) - { - predictors[idx] = _firingRate.GetLastSpikes(1); - ++idx; - } - return PredictorsCfg.NumOfEnabledPredictors; + return _predictors.CopyPredictorsTo(predictors, idx); } /// @@ -320,13 +274,7 @@ public int CopyPredictorsTo(double[] predictors, int idx) /// public double[] GetPredictors() { - if (PredictorsCfg == null || PredictorsCfg.NumOfEnabledPredictors == 0) - { - return null; - } - double[] predictors = new double[PredictorsCfg.NumOfEnabledPredictors]; - CopyPredictorsTo(predictors, 0); - return predictors; + return _predictors.GetPredictors(); } diff --git a/RCNet/Neural/Network/SM/Neuron/HiddenNeuronPredictors.cs b/RCNet/Neural/Network/SM/Neuron/HiddenNeuronPredictors.cs new file mode 100644 index 0000000..51348d7 --- /dev/null +++ b/RCNet/Neural/Network/SM/Neuron/HiddenNeuronPredictors.cs @@ -0,0 +1,214 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using RCNet.Extensions; + +namespace RCNet.Neural.Network.SM.Neuron +{ + /// + /// Class converts recent spikes to an exponentially weighted firing rate (0-1) + /// + [Serializable] + class HiddenNeuronPredictors + { + //Constants + private const int HistBuffLength = 64; + private const double FadingDecay = 0.005; + + //Static members + private static readonly double[] _spikeExpValueCache; + private static readonly double _sumOfSpikeExpValues; + + //Instance members + //Attribute properties + /// + /// Enabling/Disabling switches of predictors + /// + public HiddenNeuronPredictorsSettings Settings { get; } + + //Attributes + private readonly double[] _spikes; + private readonly double[] _activations; + private int _numOfBufferedData; + //Predictors + private int _numOfRecentFirings; + private double _fadingSumOfFirings; + private double _expWAvgOfFirings; + + /// + /// Static constructor + /// + static HiddenNeuronPredictors() + { + _spikeExpValueCache = new double[HistBuffLength]; + _sumOfSpikeExpValues = 0; + for (int i = 0; i < HistBuffLength; i++) + { + double val = Math.Exp(-i); + _spikeExpValueCache[i] = val; + _sumOfSpikeExpValues += val; + } + return; + } + + /// + /// Creates new initialized instance + /// + public HiddenNeuronPredictors(HiddenNeuronPredictorsSettings settings) + { + Settings = settings; + _spikes = new double[HistBuffLength]; + _activations = new double[HistBuffLength]; + Reset(); + return; + } + + //Methods + /// + /// Resets internal state to initial state + /// + public void Reset() + { + _spikes.Populate(0); + _activations.Populate(0); + _numOfBufferedData = 0; + _numOfRecentFirings = 0; + _fadingSumOfFirings = 0; + _expWAvgOfFirings = 0; + return; + } + + /// + /// Updates internal state + /// + /// Current value of the activation + /// Is firing spike? + public void Update(double activation, bool spike) + { + //Rotate buffers and compute predictors + _numOfRecentFirings -= (_spikes[_spikes.Length - 1] > 0 ? 1 : 0); + _fadingSumOfFirings *= (1d - FadingDecay); + _expWAvgOfFirings = 0; + for (int i = HistBuffLength - 1; i >= 0; i--) + { + _spikes[i] = (i > 0 ? _spikes[i - 1] : (spike ? 1d : 0d)); + _expWAvgOfFirings += _spikes[i] * _spikeExpValueCache[i]; + _activations[i] = (i > 0 ? _activations[i - 1] : activation); + } + //Finalize predictors + _numOfRecentFirings += (_spikes[0] > 0 ? 1 : 0); + _fadingSumOfFirings += _spikes[0]; + _expWAvgOfFirings /= _sumOfSpikeExpValues; + //Update buffers usage + if (_numOfBufferedData < HistBuffLength) + { + ++_numOfBufferedData; + } + return; + } + + /// + /// Returns last spikes as an integer number where bits are in order that the higher bit represents the more recent spike + /// + /// Last spikes history as an integer + private ulong GetLastSpikes(int histLength) + { + //Checks and corrections + if (_numOfBufferedData == 0 || histLength < 1) + { + return 0; + } + if(histLength > _numOfBufferedData) + { + histLength = _numOfBufferedData; + } + //Pick up requested history + ulong result = 0; + for (int i = 0; i < histLength; i++) + { + result <<= 1; + result += (_spikes[i] > 0 ? (ulong)1 : (ulong)0); + } + return result; + } + + /// + /// Copies values of enabled predictors to a given buffer starting from specified position (idx) + /// + /// Buffer where to be copied enabled predictors + /// Starting position index + /// + public int CopyPredictorsTo(double[] predictors, int idx) + { + if (Settings == null || Settings.NumOfEnabledPredictors == 0) + { + return 0; + } + if (Settings.Activation) + { + predictors[idx] = _activations[0]; + ++idx; + } + if (Settings.SquaredActivation) + { + predictors[idx] = _activations[0] * _activations[0]; + ++idx; + } + if (Settings.ExpWAvgFiringRate64) + { + predictors[idx] = _expWAvgOfFirings; + ++idx; + } + if (Settings.FadingNumOfFirings) + { + predictors[idx] = _fadingSumOfFirings; + ++idx; + } + if (Settings.NumOfFirings64) + { + predictors[idx] = _numOfRecentFirings; + ++idx; + } + if (Settings.LastBin32FiringHist) + { + predictors[idx] = GetLastSpikes(32); + ++idx; + } + if (Settings.LastBin16FiringHist) + { + predictors[idx] = GetLastSpikes(16); + ++idx; + } + if (Settings.LastBin8FiringHist) + { + predictors[idx] = GetLastSpikes(8); + ++idx; + } + if (Settings.LastBin1FiringHist) + { + predictors[idx] = GetLastSpikes(1); + ++idx; + } + return Settings.NumOfEnabledPredictors; + } + + /// + /// Returns array containing values of enabled predictors + /// + /// + public double[] GetPredictors() + { + if (Settings == null || Settings.NumOfEnabledPredictors == 0) + { + return null; + } + double[] predictors = new double[Settings.NumOfEnabledPredictors]; + CopyPredictorsTo(predictors, 0); + return predictors; + } + + }//HiddenNeuronPredictors + +}//Namespace diff --git a/RCNet/Neural/Network/SM/Preprocessing/PredictorsSettings.cs b/RCNet/Neural/Network/SM/Neuron/HiddenNeuronPredictorsSettings.cs similarity index 91% rename from RCNet/Neural/Network/SM/Preprocessing/PredictorsSettings.cs rename to RCNet/Neural/Network/SM/Neuron/HiddenNeuronPredictorsSettings.cs index 6230336..80228f4 100644 --- a/RCNet/Neural/Network/SM/Preprocessing/PredictorsSettings.cs +++ b/RCNet/Neural/Network/SM/Neuron/HiddenNeuronPredictorsSettings.cs @@ -11,13 +11,13 @@ using RCNet.Neural.Activation; using RCNet.Neural.Network.SM.Synapse; -namespace RCNet.Neural.Network.SM.Preprocessing +namespace RCNet.Neural.Network.SM.Neuron { /// /// Configuration of the availble predictors /// [Serializable] - public class PredictorsSettings + public class HiddenNeuronPredictorsSettings { //Attribute properties /// @@ -67,7 +67,7 @@ public class PredictorsSettings /// /// Neuron group predictors settings /// Reservoir predictors settings - public PredictorsSettings(PredictorsSettings groupPredictorsSettings, PredictorsSettings reservoirPredictorsSettings) + public HiddenNeuronPredictorsSettings(HiddenNeuronPredictorsSettings groupPredictorsSettings, HiddenNeuronPredictorsSettings reservoirPredictorsSettings) { Activation = (groupPredictorsSettings.Activation && reservoirPredictorsSettings.Activation); SquaredActivation = (groupPredictorsSettings.SquaredActivation && reservoirPredictorsSettings.SquaredActivation); @@ -86,7 +86,7 @@ public PredictorsSettings(PredictorsSettings groupPredictorsSettings, Predictors /// The deep copy constructor /// /// Source instance - public PredictorsSettings(PredictorsSettings source) + public HiddenNeuronPredictorsSettings(HiddenNeuronPredictorsSettings source) { Activation = source.Activation; SquaredActivation = source.SquaredActivation; @@ -108,12 +108,12 @@ public PredictorsSettings(PredictorsSettings source) /// Xml element containing the settings. /// Content of xml element is always validated against the xml schema. /// - public PredictorsSettings(XElement elem) + public HiddenNeuronPredictorsSettings(XElement elem) { //Validation ElemValidator validator = new ElemValidator(); Assembly assemblyRCNet = Assembly.GetExecutingAssembly(); - validator.AddXsdFromResources(assemblyRCNet, "RCNet.Neural.Network.SM.Preprocessing.PredictorsSettings.xsd"); + validator.AddXsdFromResources(assemblyRCNet, "RCNet.Neural.Network.SM.Neuron.HiddenNeuronPredictorsSettings.xsd"); validator.AddXsdFromResources(assemblyRCNet, "RCNet.RCNetTypes.xsd"); XElement PredictorsSettingsElem = validator.Validate(elem, "rootElem"); //Parsing @@ -155,7 +155,7 @@ private int GetNumOfEnabledPredictors() public override bool Equals(object obj) { if (obj == null) return false; - PredictorsSettings cmpSettings = obj as PredictorsSettings; + HiddenNeuronPredictorsSettings cmpSettings = obj as HiddenNeuronPredictorsSettings; if (Activation != cmpSettings.Activation || SquaredActivation != cmpSettings.SquaredActivation || ExpWAvgFiringRate64 != cmpSettings.ExpWAvgFiringRate64 || @@ -184,13 +184,13 @@ public override int GetHashCode() /// /// Creates the deep copy instance of this instance /// - public PredictorsSettings DeepClone() + public HiddenNeuronPredictorsSettings DeepClone() { - PredictorsSettings clone = new PredictorsSettings(this); + HiddenNeuronPredictorsSettings clone = new HiddenNeuronPredictorsSettings(this); return clone; } - }//PredictorsSettings + }//HiddenNeuronPredictorsSettings }//Namespace diff --git a/RCNet/Neural/Network/SM/Preprocessing/PredictorsSettings.xsd b/RCNet/Neural/Network/SM/Neuron/HiddenNeuronPredictorsSettings.xsd similarity index 100% rename from RCNet/Neural/Network/SM/Preprocessing/PredictorsSettings.xsd rename to RCNet/Neural/Network/SM/Neuron/HiddenNeuronPredictorsSettings.xsd diff --git a/RCNet/Neural/Network/SM/Neuron/InputNeuron.cs b/RCNet/Neural/Network/SM/Neuron/InputNeuron.cs index f5b1e6a..e75c306 100644 --- a/RCNet/Neural/Network/SM/Neuron/InputNeuron.cs +++ b/RCNet/Neural/Network/SM/Neuron/InputNeuron.cs @@ -17,7 +17,7 @@ namespace RCNet.Neural.Network.SM.Neuron public class InputNeuron : INeuron { //Static attributes - private static Interval _spikingTargetRange = new Interval(0, 1); + private static readonly Interval _spikingTargetRange = new Interval(0, 1); //Attribute properties /// diff --git a/RCNet/Neural/Network/SM/Preprocessing/NeuralPreprocessorSettings.cs b/RCNet/Neural/Network/SM/Preprocessing/NeuralPreprocessorSettings.cs index d4a9ac2..feef946 100644 --- a/RCNet/Neural/Network/SM/Preprocessing/NeuralPreprocessorSettings.cs +++ b/RCNet/Neural/Network/SM/Preprocessing/NeuralPreprocessorSettings.cs @@ -8,6 +8,7 @@ using RCNet.XmlTools; using RCNet.RandomValue; using RCNet.Neural.Data.Generators; +using RCNet.Neural.Network.SM.Neuron; using RCNet.Neural.Network.SM.Synapse; namespace RCNet.Neural.Network.SM.Preprocessing @@ -92,7 +93,7 @@ public NeuralPreprocessorSettings(XElement elem) Settings = (from settings in availableResSettings where settings.SettingsName == reservoirInstanceElem.Attribute("cfg").Value select settings).FirstOrDefault(), - PredictorsCfg = new PredictorsSettings(reservoirInstanceElem.Descendants("predictors").First()) + PredictorsCfg = new HiddenNeuronPredictorsSettings(reservoirInstanceElem.Descendants("predictors").First()) }; if (reservoirInstanceDefinition.Settings == null) { @@ -663,7 +664,7 @@ public class ReservoirInstanceDefinition /// /// Configuration of the predictors (enabling/disabling) /// - public PredictorsSettings PredictorsCfg { get; set; } + public HiddenNeuronPredictorsSettings PredictorsCfg { get; set; } //Constructors /// diff --git a/RCNet/Neural/Network/SM/Preprocessing/PoolSettings.cs b/RCNet/Neural/Network/SM/Preprocessing/PoolSettings.cs index 73ffa41..abe3128 100644 --- a/RCNet/Neural/Network/SM/Preprocessing/PoolSettings.cs +++ b/RCNet/Neural/Network/SM/Preprocessing/PoolSettings.cs @@ -10,6 +10,7 @@ using RCNet.XmlTools; using RCNet.RandomValue; using RCNet.Neural.Activation; +using RCNet.Neural.Network.SM.Neuron; using RCNet.Neural.Network.SM.Synapse; namespace RCNet.Neural.Network.SM.Preprocessing @@ -251,7 +252,7 @@ public class NeuronGroupSettings /// /// Configuration of the predictors (enabling/disabling) /// - public PredictorsSettings PredictorsCfg { get; set; } + public HiddenNeuronPredictorsSettings PredictorsCfg { get; set; } //Constructors /// @@ -346,7 +347,7 @@ public NeuronGroupSettings(XElement elem, CommonEnums.ActivationType activationT } } //Predictors - PredictorsCfg = new PredictorsSettings(settingsElem.Descendants("predictors").First()); + PredictorsCfg = new HiddenNeuronPredictorsSettings(settingsElem.Descendants("predictors").First()); return; } diff --git a/RCNet/Neural/Network/SM/Preprocessing/Reservoir.cs b/RCNet/Neural/Network/SM/Preprocessing/Reservoir.cs index f19e1e0..1b7b482 100644 --- a/RCNet/Neural/Network/SM/Preprocessing/Reservoir.cs +++ b/RCNet/Neural/Network/SM/Preprocessing/Reservoir.cs @@ -122,7 +122,7 @@ public Reservoir(NeuralPreprocessorSettings.ReservoirInstanceDefinition instance foreach (PoolSettings.NeuronGroupSettings ngs in poolSettings.NeuronGroups) { NeuronCreationParams[] grpNCP = new NeuronCreationParams[ngs.Count]; - PredictorsSettings predictorsCfg = new PredictorsSettings(ngs.PredictorsCfg, instanceDefinition.PredictorsCfg); + HiddenNeuronPredictorsSettings predictorsCfg = new HiddenNeuronPredictorsSettings(ngs.PredictorsCfg, instanceDefinition.PredictorsCfg); //Group neuron params for (int i = 0; i < ngs.Count; i++) { @@ -746,19 +746,28 @@ public ReservoirStat CollectStatistics() poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].MinTStimuliStat.AddSampleValue(neuron.Statistics.TotalStimuliStat.Min); poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].TStimuliSpansStat.AddSampleValue(neuron.Statistics.TotalStimuliStat.Span); } - poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].AvgAnalogSignalStat.AddSampleValue(neuron.Statistics.AnalogSignalStat.ArithAvg); - poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].MaxAnalogSignalStat.AddSampleValue(neuron.Statistics.AnalogSignalStat.Max); - poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].MinAnalogSignalStat.AddSampleValue(neuron.Statistics.AnalogSignalStat.Min); - poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].AvgFiringStat.AddSampleValue(neuron.Statistics.FiringStat.ArithAvg); - poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].MaxFiringStat.AddSampleValue(neuron.Statistics.FiringStat.Max); - poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].MinFiringStat.AddSampleValue(neuron.Statistics.FiringStat.Min); + if (neuron.Statistics.AnalogSignalStat.NumOfNonzeroSamples > 0) + { + poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].AvgAnalogSignalStat.AddSampleValue(neuron.Statistics.AnalogSignalStat.ArithAvg); + poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].MaxAnalogSignalStat.AddSampleValue(neuron.Statistics.AnalogSignalStat.Max); + poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].MinAnalogSignalStat.AddSampleValue(neuron.Statistics.AnalogSignalStat.Min); + } + if (neuron.Statistics.FiringStat.NumOfNonzeroSamples > 0) + { + poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].AvgFiringStat.AddSampleValue(neuron.Statistics.FiringStat.ArithAvg); + poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].MaxFiringStat.AddSampleValue(neuron.Statistics.FiringStat.Max); + poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].MinFiringStat.AddSampleValue(neuron.Statistics.FiringStat.Min); + } //Synapses efficacy statistics foreach (ISynapse rSynapse in _neuronNeuronConnectionsCollection[neuron.Placement.ReservoirFlatIdx].Values) { - poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].AvgSynEfficacyStat.AddSampleValue(rSynapse.EfficacyStat.ArithAvg); - poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].MaxSynEfficacyStat.AddSampleValue(rSynapse.EfficacyStat.Max); - poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].MinSynEfficacyStat.AddSampleValue(rSynapse.EfficacyStat.Min); - poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].SynEfficacySpansStat.AddSampleValue(rSynapse.EfficacyStat.Span); + if (rSynapse.EfficacyStat.NumOfSamples > 0) + { + poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].AvgSynEfficacyStat.AddSampleValue(rSynapse.EfficacyStat.ArithAvg); + poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].MaxSynEfficacyStat.AddSampleValue(rSynapse.EfficacyStat.Max); + poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].MinSynEfficacyStat.AddSampleValue(rSynapse.EfficacyStat.Min); + poolStat.NeuronGroupStatCollection[neuron.Placement.PoolGroupID].SynEfficacySpansStat.AddSampleValue(rSynapse.EfficacyStat.Span); + } } if (neuron.Statistics.ReservoirStimuliStat.NumOfNonzeroSamples == 0) { @@ -931,7 +940,7 @@ private class NeuronCreationParams public int GroupID { get; set; } public double AnalogFiringThreshold { get; set; } public double RetainmentStrength { get; set; } - public PredictorsSettings PredictorsCfg { get; set; } + public HiddenNeuronPredictorsSettings PredictorsCfg { get; set; } } private class RelatedNeuron diff --git a/RCNet/Neural/Network/SM/StateMachine.cs b/RCNet/Neural/Network/SM/StateMachine.cs index 859b2f3..2cb7add 100644 --- a/RCNet/Neural/Network/SM/StateMachine.cs +++ b/RCNet/Neural/Network/SM/StateMachine.cs @@ -365,7 +365,7 @@ public string CreateReport(int margin = 0) sb.Append(leftMargin + $" Const firing: {resStat.NumOfConstFiringNeurons} neurons" + Environment.NewLine); foreach (ReservoirStat.PoolStat poolStat in resStat.PoolStatCollection) { - sb.Append(leftMargin + $" Pool: {poolStat.PoolName} ({poolStat.NumOfNeurons} neurons, {Math.Round(poolStat.ExcitatoryNeuronsRatio * 100, 1).ToString(CultureInfo.InvariantCulture)}% excitatory neurons, {poolStat.InternalAnalogWeightsStat.NumOfSamples} internal synapses)" + Environment.NewLine); + sb.Append(leftMargin + $" Pool: {poolStat.PoolName} ({poolStat.NumOfNeurons} neurons, {Math.Round(poolStat.ExcitatoryNeuronsRatio * 100, 1).ToString(CultureInfo.InvariantCulture)}% excitatory neurons, {poolStat.InternalAnalogWeightsStat.NumOfSamples + poolStat.InternalSpikingWeightsStat.NumOfSamples} internal synapses)" + Environment.NewLine); sb.Append(leftMargin + $" Zero incoming res. stimuli : {poolStat.NumOfNoRStimuliNeurons} neurons" + Environment.NewLine); sb.Append(leftMargin + $" Zero emitted analog signal : {poolStat.NumOfNoAnalogOutputSignalNeurons} neurons" + Environment.NewLine); sb.Append(leftMargin + $" Const emitted analog signal: {poolStat.NumOfConstAnalogOutputSignalNeurons} neurons" + Environment.NewLine); diff --git a/RCNet/Neural/Network/SM/Synapse/InternalSynapse.cs b/RCNet/Neural/Network/SM/Synapse/InternalSynapse.cs index 3513d06..43060b2 100644 --- a/RCNet/Neural/Network/SM/Synapse/InternalSynapse.cs +++ b/RCNet/Neural/Network/SM/Synapse/InternalSynapse.cs @@ -65,7 +65,7 @@ public InternalSynapse(INeuron sourceNeuron, public void Reset(bool statistics) { _facilitation = _restingEfficacy; - _depression = 1; + _depression = 1d; if (statistics) { EfficacyStat.Reset(); diff --git a/RCNet/RCNet.csproj b/RCNet/RCNet.csproj index ca5af42..48683f2 100644 --- a/RCNet/RCNet.csproj +++ b/RCNet/RCNet.csproj @@ -128,15 +128,15 @@ + - + - @@ -402,7 +402,7 @@ - + Designer