Issue #24 - Enhancement of the networks cluster computation (ReadoutU…

…nit)

Demo/DemoConsoleApp/DemoConsoleApp.csproj

@@ -15,13 +15,17 @@
   </ItemGroup>
 
   <ItemGroup>
-    <EmbeddedResource Include="SM\SMDemoSettings.xsd">
+    <EmbeddedResource Include="SMDemoSettings.xsd">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </EmbeddedResource>
-    <EmbeddedResource Include="SM\SMDemoSettings.xml">
+    <EmbeddedResource Include="SMDemoSettings.xml">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </EmbeddedResource>
   </ItemGroup>
+
+  <ItemGroup>
+    <Resource Include="..\..\RCNet\RCNetTypes.xsd" Link="RCNetTypes.xsd" />
+  </ItemGroup>
   <ItemGroup>
     <None Update="Data\**\*.*">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>

Demo/DemoConsoleApp/Program.cs

@@ -15,15 +15,15 @@ static void Main()
                 //Main menu
                 Console.Clear();
                 Console.WriteLine("Main menu:");
-                Console.WriteLine("  1. State Machine performance demo. It sequentially performs the tasks defined in SMDemoSettings.xml.");
-                Console.WriteLine("  2. Feed Forward network trained to solve boolean algebra. Shows use of FF network as an alone component with no relationship to State Machine.");
+                Console.WriteLine("  1. State Machine performance demo. Sequentially performs the tasks defined in SMDemoSettings.xml.");
+                Console.WriteLine("  2. Feed Forward network trained to solve boolean algebra. Shows use of the FF network stndalone component (no relation to State Machine).");
                 Console.WriteLine("  3. TTOO share prices forecast (ESN design from scratch).");
                 Console.WriteLine("  4. TTOO share prices forecast (ESN design using StateMachineDesigner).");
                 Console.WriteLine("  5. Libras Movement classification (ESN design using StateMachineDesigner).");
                 Console.WriteLine("  6. Libras Movement classification (LSM design using StateMachineDesigner, horizontal spiking input encoding).");
                 Console.WriteLine("  7. Libras Movement classification (LSM design using StateMachineDesigner, vertical spiking input encoding).");
                 Console.WriteLine("  8. Libras Movement classification (LSM design using StateMachineDesigner, analog input direct routing).");
-                Console.WriteLine("  9. Libras Movement classification (No preprocessing - alone Readout Layer design using StateMachineDesigner, indicative benchmark for ESN and LSM).");
+                Console.WriteLine("  9. Libras Movement classification (No preprocessing - alone Readout Layer design using StateMachineDesigner, an indicative benchmark for ESN and LSM).");
                 Console.WriteLine("  A. Playground");
                 Console.WriteLine("  X. Exit");
                 Console.WriteLine();
@@ -35,7 +35,7 @@ static void Main()
                         try
                         {
                             //Run the demo
-                            (new SMDemo(new ConsoleLog())).RunDemo(@"./SM/SMDemoSettings.xml");
+                            (new SMDemo(new ConsoleLog())).RunDemo(@"./SMDemoSettings.xml");
                         }
                         catch (Exception e)
                         {

Demo/DemoConsoleApp/SM/SMDemoSettings.cs

@@ -40,7 +40,7 @@ public SMDemoSettings(string fileName)
             validator.AddSchema(RCNetBaseSettings.LoadRCNetTypesSchema());
             //Add SMDemoSettings.xsd
             Assembly assembly = Assembly.GetExecutingAssembly();
-            using (Stream schemaStream = assembly.GetManifestResourceStream("DemoConsoleApp.SM.SMDemoSettings.xsd"))
+            using (Stream schemaStream = assembly.GetManifestResourceStream("DemoConsoleApp.SMDemoSettings.xsd"))
             {
                 validator.AddSchema(schemaStream);
             }

RCNet/Extensions/DoubleArrayExtensions.cs

@@ -38,9 +38,12 @@ public static void Rescale(this double[] array, Interval newRange = null)
                 newRange = new Interval(0d, 1d);
             }
             Interval orgMinMax = new Interval(array);
-            for (int i = 0; i < array.Length; i++)
+            if (orgMinMax.Min != orgMinMax.Max && newRange.Min != newRange.Max)
             {
-                array[i] = newRange.Rescale(array[i], orgMinMax);
+                for (int i = 0; i < array.Length; i++)
+                {
+                    array[i] = newRange.Rescale(array[i], orgMinMax);
+                }
             }
             return;
         }
@@ -93,6 +96,52 @@ public static double Sum(this double[] array)
             return sum;
         }
 
+
+        /// <summary>
+        /// Scales members in the way the sum equals to new desired value.
+        /// </summary>
+        /// <param name="array">Array of doubles.</param>
+        /// <param name="newSum">New sum to be ensured.</param>
+        public static void ScaleToNewSum(this double[] array, double newSum = 1d)
+        {
+            double sum = Sum(array);
+            if (sum != newSum)
+            {
+                if (sum != 0d)
+                {
+                    array.Scale(newSum / sum);
+                }
+                else
+                {
+                    for(int i = 0; i < array.Length; i++)
+                    {
+                        array[i] += newSum / array.Length;
+                    }
+                }
+            }
+            return;
+        }
+
+        /// <summary>
+        /// Makes min is max and others proportionally
+        /// </summary>
+        public static void RevertMeaning(this double[] array)
+        {
+            double max = array.Max();
+            double min = array.Min();
+            if(min != max)
+            {
+                array.Rescale();
+                for (int i = 0; i < array.Length; i++)
+                {
+                    array[i] = min + (1d - array[i]) * (max - min);
+                }
+            }
+            return;
+        }
+
+
+
     }//DoubleArrayExtensions
 
 }//Namespace

RCNet/Neural/Network/NonRecurrent/NetworkClusterSecondLevelCompSettings.cs

@@ -0,0 +1,201 @@
+using System;
+using System.Globalization;
+using System.Linq;
+using System.Xml.Linq;
+using RCNet.Neural.Network.NonRecurrent.FF;
+
+namespace RCNet.Neural.Network.NonRecurrent
+{
+    /// <summary>
+    /// Configuration of the 2nd level computation of the network cluster
+    /// </summary>
+    [Serializable]
+    public class NetworkClusterSecondLevelCompSettings : RCNetBaseSettings
+    {
+        //Constants
+        /// <summary>
+        /// Name of the associated xsd type
+        /// </summary>
+        public const string XsdTypeName = "NetworkClusterSecondLevelCompType";
+        /// <summary>
+        /// Maximum allowed test data ratio
+        /// </summary>
+        public const double MaxTestDataRatio = 0.5d;
+        /// <summary>
+        /// Automatic number of folds (code)
+        /// </summary>
+        public const string AutoFoldsCode = "Auto";
+        /// <summary>
+        /// Automatic number of folds (num)
+        /// </summary>
+        public const int AutoFolds = 0;
+        //Default values
+        /// <summary>
+        /// Default value of the parameter specifying computation mode of the cluster
+        /// </summary>
+        public const TrainedNetworkCluster.SecondLevelCompMode DefaultCompMode = TrainedNetworkCluster.SecondLevelCompMode.AveragedOutputs;
+        /// <summary>
+        /// Default value of the parameter specifying required test data ratio constituting one fold
+        /// </summary>
+        public const double DefaultTestDataRatio = 0.333333333d;
+        /// <summary>
+        /// Default number of folds - string code
+        /// </summary>
+        public const string DefaultFoldsString = AutoFoldsCode;
+        /// <summary>
+        /// Default number of folds - numeric code
+        /// </summary>
+        public const int DefaultFoldsNum = AutoFolds;
+
+        //Attribute properties
+        /// <summary>
+        /// 2nd level network configuration
+        /// </summary>
+        public FeedForwardNetworkSettings NetCfg { get; }
+
+        /// <summary>
+        /// Computation mode of the cluster
+        /// </summary>
+        public TrainedNetworkCluster.SecondLevelCompMode CompMode { get; }
+
+        /// <summary>
+        /// Required test data ratio constituting one fold
+        /// </summary>
+        public double TestDataRatio { get; }
+
+        /// <summary>
+        /// Number of folds of 2nd level x-fold cross-validation computation
+        /// </summary>
+        public int Folds { get; }
+
+
+        //Constructors
+        /// <summary>
+        /// Creates an unitialized instance
+        /// </summary>
+        /// <param name="netCfg">2nd level network configuration</param>
+        /// <param name="compMode">Computation mode</param>
+        /// <param name="testDataRatio">Reqired test data ratio constituing one fold</param>
+        /// <param name="folds">Number of folds of 2nd level x-fold cross-validation computation</param>
+        public NetworkClusterSecondLevelCompSettings(FeedForwardNetworkSettings netCfg,
+                                                  TrainedNetworkCluster.SecondLevelCompMode compMode = DefaultCompMode,
+                                                  double testDataRatio = DefaultTestDataRatio,
+                                                  int folds = DefaultFoldsNum
+                                                  )
+        {
+            NetCfg = (FeedForwardNetworkSettings)netCfg.DeepClone();
+            CompMode = compMode;
+            TestDataRatio = testDataRatio;
+            Folds = folds;
+            Check();
+            return;
+        }
+
+        /// <summary>
+        /// Copy constructor
+        /// </summary>
+        /// <param name="source">Source instance</param>
+        public NetworkClusterSecondLevelCompSettings(NetworkClusterSecondLevelCompSettings source)
+            : this(source.NetCfg, source.CompMode, source.TestDataRatio, source.Folds)
+        {
+            return;
+        }
+
+        /// <summary>
+        /// Creates an initialized instance.
+        /// </summary>
+        /// <param name="elem">Xml data containing the settings.</param>
+        public NetworkClusterSecondLevelCompSettings(XElement elem)
+        {
+            //Validation
+            XElement settingsElem = Validate(elem, XsdTypeName);
+            //Parsing
+            NetCfg = new FeedForwardNetworkSettings(settingsElem.Element("ff"));
+            CompMode = (TrainedNetworkCluster.SecondLevelCompMode)Enum.Parse(typeof(TrainedNetworkCluster.SecondLevelCompMode), settingsElem.Attribute("mode").Value, true);
+            TestDataRatio = double.Parse(settingsElem.Attribute("testDataRatio").Value, CultureInfo.InvariantCulture);
+            Folds = settingsElem.Attribute("folds").Value == DefaultFoldsString ? DefaultFoldsNum : int.Parse(settingsElem.Attribute("folds").Value, CultureInfo.InvariantCulture);
+            Check();
+            return;
+        }
+
+        //Properties
+        /// <summary>
+        /// Checks if settings are default
+        /// </summary>
+        public bool IsDefaultCompMode { get { return (CompMode == DefaultCompMode); } }
+        /// <summary>
+        /// Checks if settings are default
+        /// </summary>
+        public bool IsDefaultTestDataRatio { get { return (TestDataRatio == DefaultTestDataRatio); } }
+        /// <summary>
+        /// Checks if settings are default
+        /// </summary>
+        public bool IsDefaultFolds { get { return (Folds == DefaultFoldsNum); } }
+        /// <summary>
+        /// Identifies settings containing only default values
+        /// </summary>
+        public override bool ContainsOnlyDefaults { get { return false; } }
+
+        //Methods
+        /// <summary>
+        /// Checks consistency
+        /// </summary>
+        protected override void Check()
+        {
+            if (TestDataRatio <= 0 || TestDataRatio > MaxTestDataRatio)
+            {
+                throw new ArgumentException($"Invalid TestDataRatio {TestDataRatio.ToString(CultureInfo.InvariantCulture)}. TestDataRatio must be GT 0 and GE {MaxTestDataRatio.ToString(CultureInfo.InvariantCulture)}.", "TestDataRatio");
+            }
+            if (Folds < 0)
+            {
+                throw new ArgumentException($"Invalid Folds {Folds.ToString(CultureInfo.InvariantCulture)}. Folds must be GE to 0 (0 means Auto folds).", "Folds");
+            }
+            return;
+        }
+
+        /// <summary>
+        /// Creates the deep copy instance of this instance.
+        /// </summary>
+        public override RCNetBaseSettings DeepClone()
+        {
+            return new NetworkClusterSecondLevelCompSettings(this);
+        }
+
+        /// <summary>
+        /// Generates xml element containing the settings.
+        /// </summary>
+        /// <param name="rootElemName">Name to be used as a name of the root element.</param>
+        /// <param name="suppressDefaults">Specifies whether to ommit optional nodes having set default values</param>
+        /// <returns>XElement containing the settings</returns>
+        public override XElement GetXml(string rootElemName, bool suppressDefaults)
+        {
+            XElement rootElem = new XElement(rootElemName, NetCfg.GetXml(suppressDefaults));
+            if (!suppressDefaults || !IsDefaultCompMode)
+            {
+                rootElem.Add(new XAttribute("mode", CompMode.ToString()));
+            }
+            if (!suppressDefaults || !IsDefaultTestDataRatio)
+            {
+                rootElem.Add(new XAttribute("testDataRatio", TestDataRatio.ToString(CultureInfo.InvariantCulture)));
+            }
+            if (!suppressDefaults || !IsDefaultFolds)
+            {
+                rootElem.Add(new XAttribute("folds", Folds == DefaultFoldsNum ? DefaultFoldsString : Folds.ToString(CultureInfo.InvariantCulture)));
+            }
+            Validate(rootElem, XsdTypeName);
+            return rootElem;
+        }
+
+        /// <summary>
+        /// Generates default named xml element containing the settings.
+        /// </summary>
+        /// <param name="suppressDefaults">Specifies whether to ommit optional nodes having set default values</param>
+        /// <returns>XElement containing the settings</returns>
+        public override XElement GetXml(bool suppressDefaults)
+        {
+            return GetXml("clusterSecondLevelComputation", suppressDefaults);
+        }
+
+    }//NetworkClusterSecondLevelCompSettings
+
+}//Namespace

RCNet/Neural/Network/NonRecurrent/TrainedNetworkBuilder.cs

@@ -267,7 +267,7 @@ out INonRecurrentNetworkTrainer trainer
                     lastImprovementCombinedBinaryError = currNetwork.CombinedBinaryError;
                 }
                 //Raise notification event
-                RegressionEpochDone(regrState, instructions.CurrentIsBetter);
+                RegressionEpochDone?.Invoke(regrState, instructions.CurrentIsBetter);
                 //Process instructions
                 if (instructions.StopProcess)
                 {