Skip to content
This repository has been archived by the owner on May 11, 2023. It is now read-only.

Commit

Permalink
More skeptical approach to evaluation of the predictive capabilities …
Browse files Browse the repository at this point in the history 
…of the StateMachine.
  • Loading branch information
@okozelsk
okozelsk committed Nov 16, 2018
1 parent 624b4aa commit 7416736
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 21 deletions.
6 changes: 3 additions & 3 deletions Demo/DemoConsoleApp/DemoSettings.xml
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<demo dataFolder=".\\Data">
<case name="Prediction task Mackey Glass chaotic time series: Analog reservoir, Linear regression">
<case name="Prediction task Mackey Glass chaotic time series: Analog reservoir (ESN), Linear regression">
<samples fileName="MackeyGlass.csv" bootSamples="500" singleNormalizer="false" normalizerReserve="0.1"/>
<stateMachineCfg taskType="Prediction" randomizerSeek="0">
<inputFields routeToReadout="false">
<field name="Value"/>
</inputFields>
<reservoirCfgContainer>
<reservoirCfg name="Analog" inputCoding="Analog" inputDuration="1" spectralRadius="0.99">
<reservoirCfg name="Analog (ESN)" inputCoding="Analog" inputDuration="1" spectralRadius="0.99">
<pools>
<pool name="Analog pool" dimX="8" dimY="8" dimZ="8" routeToReadout="true">
<neuronGroups>
Expand All @@ -31,7 +31,7 @@
</reservoirCfg>
</reservoirCfgContainer>
<reservoirInstanceContainer>
<reservoirInstance name="Experimental" cfg="Analog" augmentedStates="true">
<reservoirInstance name="Fully analog reservoir " cfg="Analog (ESN)" augmentedStates="true">
<inputFieldAssignments>
<inputFieldAssignment inputFieldName="Value" poolName="Analog pool" density="1">
<weight min="0" max="0.6" randomSign="false" distribution="Uniform"/>
Expand Down
4 changes: 2 additions & 2 deletions Demo/DemoConsoleApp/SMDemo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ private static void ReportReservoirsStatistics(List<ReservoirStat> statisticsCol
+ poolStat.NeuronsStimuliSpansStat.Max.ToString("N4", CultureInfo.InvariantCulture) + ", "
+ poolStat.NeuronsStimuliSpansStat.Min.ToString("N4", CultureInfo.InvariantCulture) + ", "
+ poolStat.NeuronsStimuliSpansStat.StdDev.ToString("N4", CultureInfo.InvariantCulture), false);
log.Write($" Neurons transmission signals", false);
log.Write($" Neurons output signal", false);
log.Write(" AVG Avg, Max, Min, SDdev: " + poolStat.NeuronsAvgOutputSignalStat.ArithAvg.ToString("N4", CultureInfo.InvariantCulture) + ", "
+ poolStat.NeuronsAvgOutputSignalStat.Max.ToString("N4", CultureInfo.InvariantCulture) + ", "
+ poolStat.NeuronsAvgOutputSignalStat.Min.ToString("N4", CultureInfo.InvariantCulture) + ", "
Expand All @@ -94,7 +94,7 @@ private static void ReportReservoirsStatistics(List<ReservoirStat> statisticsCol
+ poolStat.NeuronsMinOutputSignalStat.Max.ToString("N4", CultureInfo.InvariantCulture) + ", "
+ poolStat.NeuronsMinOutputSignalStat.Min.ToString("N4", CultureInfo.InvariantCulture) + ", "
+ poolStat.NeuronsMinOutputSignalStat.StdDev.ToString("N4", CultureInfo.InvariantCulture), false);
log.Write($" Neurons transmission frequencies", false);
log.Write($" Neurons output signal frequency", false);
log.Write(" AVG Avg, Max, Min, SDdev: " + poolStat.NeuronsAvgOutputFreqStat.ArithAvg.ToString("N4", CultureInfo.InvariantCulture) + ", "
+ poolStat.NeuronsAvgOutputFreqStat.Max.ToString("N4", CultureInfo.InvariantCulture) + ", "
+ poolStat.NeuronsAvgOutputFreqStat.Min.ToString("N4", CultureInfo.InvariantCulture) + ", "
Expand Down
44 changes: 28 additions & 16 deletions RCNet/Neural/Network/SM/ReadoutLayer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,17 +93,23 @@ Random rand
/// <param name="idealOutputsCollection">Collection of all available desired outputs related to predictors</param>
/// <param name="regressionController">Regression controller delegate</param>
/// <param name="regressionControllerData">An user object</param>
/// <returns>Returned ValidationBundle is something like a protocol.
/// There is recorded fold by fold (unit by unit) predicted and corresponding ideal values.
/// This is the pesimistic approach. Real results on unseen data could be better due to the clustering synergy.
/// </returns>
public ValidationBundle Build(List<double[]> predictorsCollection,
List<double[]> idealOutputsCollection,
ReadoutUnit.RegressionCallbackDelegate regressionController,
Object regressionControllerData
)
{
//Allocation of computed vectors for validation bundle
List<double[]> computedVectorCollection = new List<double[]>(idealOutputsCollection.Count);
for(int i = 0; i < idealOutputsCollection.Count; i++)
//Allocation of computed and ideal vectors for validation bundle
List<double[]> validationComputedVectorCollection = new List<double[]>(idealOutputsCollection.Count);
List<double[]> validationIdealVectorCollection = new List<double[]>(idealOutputsCollection.Count);
for (int i = 0; i < idealOutputsCollection.Count; i++)
{
computedVectorCollection.Add(new double[idealOutputsCollection[0].Length]);
validationComputedVectorCollection.Add(new double[idealOutputsCollection[0].Length]);
validationIdealVectorCollection.Add(new double[idealOutputsCollection[0].Length]);
}
//Test dataset size
if (_settings.TestDataRatio > MaxRatioOfTestData)
Expand Down Expand Up @@ -172,7 +178,9 @@ Object regressionControllerData
testDataSetLength
);
}
//Readout units in the cluster
//Best predicting unit per each fold in the cluster.
ClusterErrStatistics ces = new ClusterErrStatistics(_taskType, numOfFolds, refBinDistr);
int arrayPos = 0;
for (int foldIdx = 0; foldIdx < numOfFolds; foldIdx++)
{
//Build training samples
Expand All @@ -186,7 +194,8 @@ Object regressionControllerData
trainingIdealValueCollection.AddRange(subBundleCollection[bundleIdx].OutputVectorCollection);
}
}
//Call training regression for the single readout unit
//Call training regression to get the best fold's readout unit.
//The best unit becomes to be the predicting cluster member.
_clusterCollection[clusterIdx][foldIdx] = ReadoutUnit.CreateTrained(_taskType,
clusterIdx,
_settings.OutputFieldNameCollection[clusterIdx],
Expand All @@ -202,20 +211,23 @@ Object regressionControllerData
regressionController,
regressionControllerData
);
//Cluster error statistics & data for validation bundle (pesimistic approach)
for (int sampleIdx = 0; sampleIdx < subBundleCollection[foldIdx].OutputVectorCollection.Count; sampleIdx++)
{

double value = _clusterCollection[clusterIdx][foldIdx].Network.Compute(subBundleCollection[foldIdx].InputVectorCollection[sampleIdx])[0];
ces.Update(value, subBundleCollection[foldIdx].OutputVectorCollection[sampleIdx][0]);
validationIdealVectorCollection[arrayPos][clusterIdx] = subBundleCollection[foldIdx].OutputVectorCollection[sampleIdx][0];
validationComputedVectorCollection[arrayPos][clusterIdx] = value;
++arrayPos;
}

}//foldIdx
//Cluster error statistics & data for validation bundle
ClusterErrStatistics ces = new ClusterErrStatistics(_taskType, numOfFolds, refBinDistr);
for (int sampleIdx = 0; sampleIdx < idealOutputsCollection.Count; sampleIdx++)
{
double value = Compute(predictorsCollection[sampleIdx], clusterIdx);
ces.Update(value, idealOutputsCollection[sampleIdx][clusterIdx]);
computedVectorCollection[sampleIdx][clusterIdx] = value;
}
_clusterErrStatisticsCollection.Add(ces);

}//clusterIdx

return new ValidationBundle(computedVectorCollection, idealOutputsCollection);
//Validation bundle is returned.
return new ValidationBundle(validationComputedVectorCollection, validationIdealVectorCollection);
}

//Properties
Expand Down

0 comments on commit 7416736

Please sign in to comment.