diff --git a/README.md b/README.md index 6930af07..e9c9db3f 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,30 @@ DeepScaleAndShiftOperation FlattenOperation +### Vector Neural Network (VNN) Operations +These types of operations typically operate on instances of the Matrix class where the left half are magnitudes and the right half are angles in radians. +Learn more about Vector Neural Networks [here](https://www.amazon.com/Vector-Neural-Networks-Geometric-Tensors-ebook/dp/B0CXBV3DY5/ref=sr_1_1). + +ElementwiseSquareOperation + +ElementwiseVectorAddOperation + +ElementwiseVectorCartesianSummationOperation + +ElementwiseVectorConstituentMultiplyOperation + +ElementwiseVectorDecompositionOperation + +ElementwiseVectorMiniDecompositionOperation + +PairwiseSineSoftmaxOperation + +VectorAttentionBinaryOperation + +VectorAttentionOperation + +VectorizeOperation + ### Neural Network Parameters Each neural network base class has a set of parameters that can be used to configure the neural network. They are as follows: diff --git a/examples/gravnet/ParallelReverseAutoDiff.GravNetExample/VectorNetwork/RMAD/VectorAttentionOperation.cs b/examples/gravnet/ParallelReverseAutoDiff.GravNetExample/VectorNetwork/RMAD/VectorAttentionOperation.cs index a3c2301b..a1da82e8 100644 --- a/examples/gravnet/ParallelReverseAutoDiff.GravNetExample/VectorNetwork/RMAD/VectorAttentionOperation.cs +++ b/examples/gravnet/ParallelReverseAutoDiff.GravNetExample/VectorNetwork/RMAD/VectorAttentionOperation.cs @@ -95,6 +95,5 @@ public override BackwardResult Backward(Matrix dOutput) .AddInputGradient(dProbabilities) .Build(); } - } } diff --git a/src/ParallelReverseAutoDiff.1.1.65.nupkg b/src/ParallelReverseAutoDiff.1.2.0.nupkg similarity index 59% rename from src/ParallelReverseAutoDiff.1.1.65.nupkg rename to src/ParallelReverseAutoDiff.1.2.0.nupkg index e1dba2e8..e3439894 100644 Binary files a/src/ParallelReverseAutoDiff.1.1.65.nupkg and b/src/ParallelReverseAutoDiff.1.2.0.nupkg differ diff --git a/src/ParallelReverseAutoDiff.nuspec b/src/ParallelReverseAutoDiff.nuspec index 1be4d04f..08448961 100644 --- a/src/ParallelReverseAutoDiff.nuspec +++ b/src/ParallelReverseAutoDiff.nuspec @@ -2,7 +2,7 @@ ParallelReverseAutoDiff - 1.1.65 + 1.2.0 ameritusweb ameritusweb LGPL-2.1-only @@ -11,7 +11,7 @@ false A library for parallelized reverse mode automatic differentiation in C# for custom neural network development. - Fix GPU Matrix Multiply. + Add VNN Operations. ameritusweb, 2024 autodiff automatic-differentiation parallel reverse-mode differentiation C# neural network diff --git a/src/RMAD/ElementwiseSquareOperation.cs b/src/RMAD/ElementwiseSquareOperation.cs new file mode 100644 index 00000000..21587629 --- /dev/null +++ b/src/RMAD/ElementwiseSquareOperation.cs @@ -0,0 +1,85 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) 2023 ameritusweb All rights reserved. +// +//------------------------------------------------------------------------------ +namespace ParallelReverseAutoDiff.RMAD +{ + using System; + + /// + /// Performs the forward and backward operations for the element-wise square function. + /// + public class ElementwiseSquareOperation : Operation + { + private Matrix input; + + /// + /// A common factory method for instantiating this operation. + /// + /// The neural network. + /// The instantiated operation. + public static IOperation Instantiate(NeuralNetwork net) + { + return new ElementwiseSquareOperation(); + } + + /// + public override void Store(Guid id) + { + this.IntermediateMatrices.AddOrUpdate(id, this.input, (x, y) => this.input); + } + + /// + public override void Restore(Guid id) + { + this.input = this.IntermediateMatrices[id]; + } + + /// + /// Performs the forward operation for the element-wise square function. + /// + /// The input to the element-wise square operation. + /// The output of the element-wise square operation. + public Matrix Forward(Matrix input) + { + this.input = input; + int rows = input.Length; + int cols = input[0].Length; + this.Output = new Matrix(rows, cols); + + for (int i = 0; i < rows; i++) + { + for (int j = 0; j < cols; j++) + { + double x = input[i][j]; + this.Output[i][j] = x * x; + } + } + + return this.Output; + } + + /// + public override BackwardResult Backward(Matrix dLdOutput) + { + int rows = dLdOutput.Length; + int cols = dLdOutput[0].Length; + Matrix dLdInput = new Matrix(rows, cols); + + for (int i = 0; i < rows; i++) + { + for (int j = 0; j < cols; j++) + { + double x = this.input[i][j]; + double gradient = 2 * x; + dLdInput[i][j] = dLdOutput[i][j] * gradient; + } + } + + return new BackwardResultBuilder() + .AddInputGradient(dLdInput) + .Build(); + } + } +} diff --git a/src/RMAD/ElementwiseVectorAddOperation.cs b/src/RMAD/ElementwiseVectorAddOperation.cs new file mode 100644 index 00000000..d5adbd75 --- /dev/null +++ b/src/RMAD/ElementwiseVectorAddOperation.cs @@ -0,0 +1,122 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) 2023 ameritusweb All rights reserved. +// +//------------------------------------------------------------------------------ +namespace ParallelReverseAutoDiff.RMAD +{ + using System; + using System.Threading.Tasks; + + /// + /// Element-wise add operation. + /// + public class ElementwiseVectorAddOperation : Operation + { + private Matrix input1; + private Matrix input2; + + /// + /// A common method for instantiating an operation. + /// + /// The neural network. + /// The instantiated operation. + public static IOperation Instantiate(NeuralNetwork net) + { + return new ElementwiseVectorAddOperation(); + } + + /// + /// Performs the forward operation for the element-wise vector summation function. + /// + /// The first input to the element-wise vector summation operation. + /// The second input to the element-wise vector summation operation. + /// The output of the element-wise vector summation operation. + public Matrix Forward(Matrix input1, Matrix input2) + { + this.input1 = input1; + this.input2 = input2; + + this.Output = new Matrix(this.input1.Rows, this.input1.Cols); + Parallel.For(0, input1.Rows, i => + { + for (int j = 0; j < input1.Cols / 2; j++) + { + // Accessing the magnitudes and angles from the concatenated matrices + double magnitude = input1[i, j]; + double angle = input1[i, j + (input1.Cols / 2)]; + + double wMagnitude = input2[i, j]; + double wAngle = input2[i, j + (input2.Cols / 2)]; + + // Compute vector components + double x1 = magnitude * Math.Cos(angle); + double y1 = magnitude * Math.Sin(angle); + double x2 = wMagnitude * Math.Cos(wAngle); + double y2 = wMagnitude * Math.Sin(wAngle); + + double sumx = x1 + x2; + double sumy = y1 + y2; + + // Compute resultant vector magnitude and angle + double resultMagnitude = Math.Sqrt((sumx * sumx) + (sumy * sumy)); + double resultAngle = Math.Atan2(sumy, sumx); + + this.Output[i, j] = resultMagnitude; + this.Output[i, j + (this.input1.Cols / 2)] = resultAngle; + } + }); + + return this.Output; + } + + /// + public override BackwardResult Backward(Matrix dOutput) + { + Matrix dInput1 = new Matrix(this.input1.Rows, this.input1.Cols); + Matrix dInput2 = new Matrix(this.input2.Rows, this.input2.Cols); + + Parallel.For(0, this.input1.Rows, i => + { + for (int j = 0; j < this.input1.Cols / 2; j++) + { + var magnitude = this.input1[i, j]; + var angle = this.input1[i, j + (this.input1.Cols / 2)]; + var wMagnitude = this.input2[i, j]; + var wAngle = this.input2[i, j + (this.input2.Cols / 2)]; + + var x1 = magnitude * Math.Cos(angle); + var y1 = magnitude * Math.Sin(angle); + var x2 = wMagnitude * Math.Cos(wAngle); + var y2 = wMagnitude * Math.Sin(wAngle); + + var combinedX = x1 + x2; + var combinedY = y1 + y2; + + // Compute gradients for magnitude and angle + double dResultMagnitude_dX = combinedX / this.Output[i, j]; + double dResultMagnitude_dY = combinedY / this.Output[i, j]; + + double dResultAngle_dX = -combinedY / ((combinedX * combinedX) + (combinedY * combinedY)); + double dResultAngle_dY = combinedX / ((combinedX * combinedX) + (combinedY * combinedY)); + + // Chain rule to compute gradients for input vectors + dInput1[i, j] = (dOutput[i, j] * dResultMagnitude_dX * Math.Cos(angle)) + + (dOutput[i, j + (this.input1.Cols / 2)] * dResultAngle_dX * -Math.Sin(angle)); + dInput1[i, j + (this.input1.Cols / 2)] = (dOutput[i, j] * dResultMagnitude_dY * Math.Sin(angle)) + + (dOutput[i, j + (this.input1.Cols / 2)] * dResultAngle_dY * Math.Cos(angle)); + + dInput2[i, j] = (dOutput[i, j] * dResultMagnitude_dX * Math.Cos(wAngle)) + + (dOutput[i, j + (this.input2.Cols / 2)] * dResultAngle_dX * -Math.Sin(wAngle)); + dInput2[i, j + (this.input2.Cols / 2)] = (dOutput[i, j] * dResultMagnitude_dY * Math.Sin(wAngle)) + + (dOutput[i, j + (this.input2.Cols / 2)] * dResultAngle_dY * Math.Cos(wAngle)); + } + }); + + return new BackwardResultBuilder() + .AddInputGradient(dInput1) + .AddInputGradient(dInput2) + .Build(); + } + } +} diff --git a/src/RMAD/ElementwiseVectorCartesianSummationOperation.cs b/src/RMAD/ElementwiseVectorCartesianSummationOperation.cs new file mode 100644 index 00000000..e15dd749 --- /dev/null +++ b/src/RMAD/ElementwiseVectorCartesianSummationOperation.cs @@ -0,0 +1,222 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) 2023 ameritusweb All rights reserved. +// +//------------------------------------------------------------------------------ +namespace ParallelReverseAutoDiff.RMAD +{ + using System; + using System.Linq; + using System.Threading.Tasks; + + /// + /// Element-wise cartesian summation operation. + /// + public class ElementwiseVectorCartesianSummationOperation : Operation + { + private Matrix input1; + private Matrix input2; + private Matrix weights; + private double[] summationX; + private double[] summationY; + private CalculatedValues[,] calculatedValues; + + /// + /// A common method for instantiating an operation. + /// + /// The neural network. + /// The instantiated operation. + public static IOperation Instantiate(NeuralNetwork net) + { + return new ElementwiseVectorCartesianSummationOperation(); + } + + /// + /// Performs the forward operation for the element-wise vector summation function. + /// + /// The first input to the element-wise vector summation operation. + /// The second input to the element-wise vector summation operation. + /// The weights. + /// The output of the element-wise vector summation operation. + public Matrix Forward(Matrix input1, Matrix input2, Matrix weights) + { + this.input1 = input1; + this.input2 = input2; + this.weights = weights; + + this.Output = new Matrix(1, 2); + + this.calculatedValues = new CalculatedValues[this.input1.Rows, this.input1.Cols / 2]; + + double[] summationX = new double[input1.Rows]; + double[] summationY = new double[input1.Rows]; + double[,] resultVectors = new double[input1.Rows * (input1.Cols / 2), 2]; + Parallel.For(0, input1.Rows, i => + { + double sumX = 0.0d; + double sumY = 0.0d; + (double, double)[] resultMagnitudes = new (double, double)[input1.Cols / 2]; + for (int j = 0; j < (input1.Cols / 2); j++) + { + // Accessing the magnitudes and angles from the concatenated matrices + double magnitude = input1[i, j]; + double angle = input1[i, j + (input1.Cols / 2)]; + + double wMagnitude = input2[i, j]; + double wAngle = input2[i, j + (input2.Cols / 2)]; + + // Compute vector components + double x1 = magnitude * Math.Cos(angle); + double y1 = magnitude * Math.Sin(angle); + double x2 = wMagnitude * Math.Cos(wAngle); + double y2 = wMagnitude * Math.Sin(wAngle); + + double sumx = x1 + x2; + double sumy = y1 + y2; + + double dsumx_dAngle = -magnitude * Math.Sin(angle); + double dsumx_dWAngle = -wMagnitude * Math.Sin(wAngle); + double dsumy_dAngle = magnitude * Math.Cos(angle); + double dsumy_dWAngle = wMagnitude * Math.Cos(wAngle); + double dsumx_dMagnitude = Math.Cos(angle); + double dsumx_dWMagnitude = Math.Cos(wAngle); + double dsumy_dMagnitude = Math.Sin(angle); + double dsumy_dWMagnitude = Math.Sin(wAngle); + + // Compute resultant vector magnitude and angle + double resultMagnitude = Math.Sqrt((sumx * sumx) + (sumy * sumy)) * weights[i, j]; + double resultAngle = Math.Atan2(sumy, sumx); + + double dResultMagnitude_dsumx = (sumx * weights[i, j]) / Math.Sqrt((sumx * sumx) + (sumy * sumy)); + double dResultMagnitude_dsumy = (sumy * weights[i, j]) / Math.Sqrt((sumx * sumx) + (sumy * sumy)); + double dResultAngle_dsumx = -sumy / ((sumx * sumx) + (sumy * sumy)); + double dResultAngle_dsumy = sumx / ((sumx * sumx) + (sumy * sumy)); + + double dResultMagnitude_dAngle = (dResultMagnitude_dsumx * dsumx_dAngle) + (dResultMagnitude_dsumy * dsumy_dAngle); + double dResultMagnitude_dWAngle = (dResultMagnitude_dsumx * dsumx_dWAngle) + (dResultMagnitude_dsumy * dsumy_dWAngle); + double dResultAngle_dAngle = (dResultAngle_dsumx * dsumx_dAngle) + (dResultAngle_dsumy * dsumy_dAngle); + double dResultAngle_dWAngle = (dResultAngle_dsumx * dsumx_dWAngle) + (dResultAngle_dsumy * dsumy_dWAngle); + + double dResultMagnitude_dMagnitude = (dResultMagnitude_dsumx * dsumx_dMagnitude) + (dResultMagnitude_dsumy * dsumy_dMagnitude); + double dResultMagnitude_dWMagnitude = (dResultMagnitude_dsumx * dsumx_dWMagnitude) + (dResultMagnitude_dsumy * dsumy_dWMagnitude); + double dResultAngle_dMagnitude = (dResultAngle_dsumx * dsumx_dMagnitude) + (dResultAngle_dsumy * dsumy_dMagnitude); + double dResultAngle_dWMagnitude = (dResultAngle_dsumx * dsumx_dWMagnitude) + (dResultAngle_dsumy * dsumy_dWMagnitude); + + resultVectors[(i * (input1.Cols / 2)) + j, 0] = resultMagnitude; + resultVectors[(i * (input1.Cols / 2)) + j, 1] = resultAngle; + + double localSumX = resultMagnitude * Math.Cos(resultAngle); + double localSumY = resultMagnitude * Math.Sin(resultAngle); + + double localSumXFull = Math.Sqrt((sumx * sumx) + (sumy * sumy)) * weights[i, j] * Math.Cos(resultAngle); + double localSumYFull = Math.Sqrt((sumx * sumx) + (sumy * sumy)) * weights[i, j] * Math.Sin(resultAngle); + + double dLocalSumX_dWeight = Math.Sqrt((sumx * sumx) + (sumy * sumy)) * Math.Cos(resultAngle); + double dLocalSumY_dWeight = Math.Sqrt((sumx * sumx) + (sumy * sumy)) * Math.Sin(resultAngle); + + this.calculatedValues[i, j].DLocalSumX_DWeight = dLocalSumX_dWeight; + this.calculatedValues[i, j].DLocalSumY_DWeight = dLocalSumY_dWeight; + + double dLocalSumX_dResultMagnitude = Math.Cos(resultAngle); + double dLocalSumX_dResultAngle = -resultMagnitude * Math.Sin(resultAngle); + + double dLocalSumX_dAngle = (dLocalSumX_dResultMagnitude * dResultMagnitude_dAngle) + (dLocalSumX_dResultAngle * dResultAngle_dAngle); + double dLocalSumX_dWAngle = (dLocalSumX_dResultMagnitude * dResultMagnitude_dWAngle) + (dLocalSumX_dResultAngle * dResultAngle_dWAngle); + double dLocalSumX_dMagnitude = (dLocalSumX_dResultMagnitude * dResultMagnitude_dMagnitude) + (dLocalSumX_dResultAngle * dResultAngle_dMagnitude); + double dLocalSumX_dWMagnitude = (dLocalSumX_dResultMagnitude * dResultMagnitude_dWMagnitude) + (dLocalSumX_dResultAngle * dResultAngle_dWMagnitude); + + this.calculatedValues[i, j].DLocalSumX_DAngle = dLocalSumX_dAngle; + this.calculatedValues[i, j].DLocalSumX_DWAngle = dLocalSumX_dWAngle; + this.calculatedValues[i, j].DLocalSumX_DMagnitude = dLocalSumX_dMagnitude; + this.calculatedValues[i, j].DLocalSumX_DWMagnitude = dLocalSumX_dWMagnitude; + + double dLocalSumY_dResultMagnitude = Math.Sin(resultAngle); + double dLocalSumY_dResultAngle = resultMagnitude * Math.Cos(resultAngle); + + double dLocalSumY_dAngle = (dLocalSumY_dResultMagnitude * dResultMagnitude_dAngle) + (dLocalSumY_dResultAngle * dResultAngle_dAngle); + double dLocalSumY_dWAngle = (dLocalSumY_dResultMagnitude * dResultMagnitude_dWAngle) + (dLocalSumY_dResultAngle * dResultAngle_dWAngle); + double dLocalSumY_dMagnitude = (dLocalSumY_dResultMagnitude * dResultMagnitude_dMagnitude) + (dLocalSumY_dResultAngle * dResultAngle_dMagnitude); + double dLocalSumY_dWMagnitude = (dLocalSumY_dResultMagnitude * dResultMagnitude_dWMagnitude) + (dLocalSumY_dResultAngle * dResultAngle_dWMagnitude); + + this.calculatedValues[i, j].DLocalSumY_DAngle = dLocalSumY_dAngle; + this.calculatedValues[i, j].DLocalSumY_DWAngle = dLocalSumY_dWAngle; + this.calculatedValues[i, j].DLocalSumY_DMagnitude = dLocalSumY_dMagnitude; + this.calculatedValues[i, j].DLocalSumY_DWMagnitude = dLocalSumY_dWMagnitude; + + sumX += localSumX; + sumY += localSumY; + } + + summationX[i] = sumX; + summationY[i] = sumY; + }); + + this.summationX = summationX; + this.summationY = summationY; + + this.Output[0, 0] = this.summationX.Sum(); + this.Output[0, 1] = this.summationY.Sum(); + + return this.Output; + } + + /// + public override BackwardResult Backward(Matrix dOutput) + { + Matrix dInput1 = new Matrix(this.input1.Rows, this.input1.Cols); + Matrix dInput2 = new Matrix(this.input2.Rows, this.input2.Cols); + Matrix dWeights = new Matrix(this.weights.Rows, this.weights.Cols); + + double dSummationXOutput = dOutput[0, 0]; // Gradient of the loss function with respect to the output X + double dSummationYOutput = dOutput[0, 1]; // Gradient of the loss function with respect to the output Y + + // Updating gradients with respect to resultMagnitude and resultAngle + Parallel.For(0, this.input1.Rows, i => + { + for (int j = 0; j < this.input1.Cols / 2; j++) + { + var values = this.calculatedValues[i, j]; + + // Update dWeights with direct contributions from summationX and summationY + dWeights[i, j] = (dSummationXOutput * values.DLocalSumX_DWeight) + (dSummationYOutput * values.DLocalSumY_DWeight); + + // Apply chain rule to propagate back to dInput1 and dInput2 + dInput1[i, j] = (dSummationXOutput * values.DLocalSumX_DMagnitude) + (dSummationYOutput * values.DLocalSumY_DMagnitude); + dInput1[i, j + (this.input1.Cols / 2)] = (dSummationXOutput * values.DLocalSumX_DAngle) + (dSummationYOutput * values.DLocalSumY_DAngle); + + dInput2[i, j] = (dSummationXOutput * values.DLocalSumX_DWMagnitude) + (dSummationYOutput * values.DLocalSumY_DWMagnitude); + dInput2[i, j + (this.input2.Cols / 2)] = (dSummationXOutput * values.DLocalSumX_DWAngle) + (dSummationYOutput * values.DLocalSumY_DWAngle); + } + }); + + return new BackwardResultBuilder() + .AddInputGradient(dInput1) + .AddInputGradient(dInput2) + .AddInputGradient(dWeights) + .Build(); + } + + private struct CalculatedValues + { + public double DLocalSumX_DAngle { get; internal set; } + + public double DLocalSumX_DWAngle { get; internal set; } + + public double DLocalSumX_DMagnitude { get; internal set; } + + public double DLocalSumX_DWMagnitude { get; internal set; } + + public double DLocalSumY_DAngle { get; internal set; } + + public double DLocalSumY_DWAngle { get; internal set; } + + public double DLocalSumY_DMagnitude { get; internal set; } + + public double DLocalSumY_DWMagnitude { get; internal set; } + + public double DLocalSumX_DWeight { get; internal set; } + + public double DLocalSumY_DWeight { get; internal set; } + } + } +} diff --git a/src/RMAD/ElementwiseVectorConstituentMultiplyOperation.cs b/src/RMAD/ElementwiseVectorConstituentMultiplyOperation.cs new file mode 100644 index 00000000..6b30a09b --- /dev/null +++ b/src/RMAD/ElementwiseVectorConstituentMultiplyOperation.cs @@ -0,0 +1,343 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) 2023 ameritusweb All rights reserved. +// +//------------------------------------------------------------------------------ +namespace ParallelReverseAutoDiff.RMAD +{ + using System; + using System.Threading.Tasks; + + /// + /// Element-wise vector constituent multiplication operation. + /// + public class ElementwiseVectorConstituentMultiplyOperation : Operation + { + private Matrix input1; + private Matrix input2; + private Matrix weights; + private Matrix sumX; + private Matrix sumY; + private CalculatedValues calculatedValues; + + /// + /// A common method for instantiating an operation. + /// + /// The neural network. + /// The instantiated operation. + public static IOperation Instantiate(NeuralNetwork net) + { + return new ElementwiseVectorConstituentMultiplyOperation(); + } + + /// + /// Performs the forward operation for the vector constituent multiply function. + /// + /// The first input to the vector constituent multiply operation. + /// The second input to the vector constituent multiply operation. + /// The weights to the vector constituent multiply operation. + /// The output of the vector constituent multiply operation. + public Matrix Forward(Matrix input1, Matrix input2, Matrix weights) + { + this.input1 = input1; + this.input2 = input2; + this.weights = weights; + + this.Output = new Matrix(input1.Rows, input2.Cols); + this.sumX = new Matrix(input1.Rows, input2.Cols / 2); + this.sumY = new Matrix(input1.Rows, input2.Cols / 2); + + double[,] dInputMag_dOutputMag = new double[input1.Rows, input2.Rows / 2]; + double[,] dInputMag_dOutputAngle = new double[input1.Rows, input2.Rows / 2]; + double[,] dInputAngle_dOutputMag = new double[input1.Rows, input2.Rows / 2]; + double[,] dInputAngle_dOutputAngle = new double[input1.Rows, input2.Rows / 2]; + double[,] dInput2Mag_dOutputMag = new double[input2.Rows / 2, input2.Cols / 2]; + double[,] dInput2Mag_dOutputAngle = new double[input2.Rows / 2, input2.Cols / 2]; + double[,] dInput2Angle_dOutputMag = new double[input2.Rows / 2, input2.Cols / 2]; + double[,] dInput2Angle_dOutputAngle = new double[input2.Rows / 2, input2.Cols / 2]; + double[,] dWeight_dOutputMag = new double[input2.Rows / 2, input2.Cols / 2]; + double[,] dWeight_dOutputAngle = new double[input2.Rows / 2, input2.Cols / 2]; + + Parallel.For(0, input1.Rows, i => + { + for (int j = 0; j < input2.Cols / 2; j++) + { + double sumX = 0.0d; + double sumY = 0.0d; + + double[] dDeltaX_dX1 = new double[input2.Rows / 2]; + double[] dDeltaY_dY1 = new double[input2.Rows / 2]; + double[] dDeltaX_dX2 = new double[input2.Rows / 2]; + double[] dDeltaY_dY2 = new double[input2.Rows / 2]; + double[] dSumX_dDeltaX = new double[input2.Rows / 2]; + double[] dSumX_dDeltaY = new double[input2.Rows / 2]; + double[] dSumY_dDeltaX = new double[input2.Rows / 2]; + double[] dSumY_dDeltaY = new double[input2.Rows / 2]; + double[] dDeltaX_dWeight = new double[input2.Rows / 2]; + double[] dDeltaY_dWeight = new double[input2.Rows / 2]; + double[] dX1_dMagnitude = new double[input2.Rows / 2]; + double[] dY1_dMagnitude = new double[input2.Rows / 2]; + double[] dX1_dAngle = new double[input2.Rows / 2]; + double[] dY1_dAngle = new double[input2.Rows / 2]; + double[] dX2_dWMagnitude = new double[input2.Rows / 2]; + double[] dY2_dWMagnitude = new double[input2.Rows / 2]; + double[] dX2_dWAngle = new double[input2.Rows / 2]; + double[] dY2_dWAngle = new double[input2.Rows / 2]; + double[] dSumX_dResultMagnitude = new double[input2.Rows / 2]; + double[] dSumY_dResultMagnitude = new double[input2.Rows / 2]; + double[] dResultMagnitude_dWeight = new double[input2.Rows / 2]; + + for (int k = 0; k < input2.Rows / 2; k++) + { + // Accessing the magnitudes and angles from the concatenated matrices + double magnitude = input1[i, k]; + double angle = input1[i, k + (input1.Cols / 2)]; + + double wMagnitude = input2[k, j]; + double wAngle = input2[k, j + (input2.Cols / 2)]; + + // Compute vector components + double x1 = magnitude * Math.Cos(angle); + double y1 = magnitude * Math.Sin(angle); + double x2 = wMagnitude * Math.Cos(wAngle); + double y2 = wMagnitude * Math.Sin(wAngle); + + // Select vector direction based on weight + double deltax = weights[k, j] > 0 ? x2 - x1 : x1 - x2; + double deltay = weights[k, j] > 0 ? y2 - y1 : y1 - y2; + + double deltaXYSquared = (deltax * deltax) + (deltay * deltay); + + // Compute resultant vector magnitude and angle + double resultMagnitude = Math.Sqrt(deltaXYSquared) * weights[k, j]; + double resultAngle = Math.Atan2(deltay, deltax); + + double dResultMagnitude_dDeltaX = (deltax * weights[k, j]) / Math.Sqrt(deltaXYSquared); + double dResultMagnitude_dDeltaY = (deltay * weights[k, j]) / Math.Sqrt(deltaXYSquared); + double dResultAngle_dDeltaX = -deltay / deltaXYSquared; + double dResultAngle_dDeltaY = deltax / deltaXYSquared; + + double localSumX = resultMagnitude * Math.Cos(resultAngle); + double localSumY = resultMagnitude * Math.Sin(resultAngle); + + double dLocalSumX_dResultMagnitude = Math.Cos(resultAngle); + double dLocalSumY_dResultMagnitude = Math.Sin(resultAngle); + + double dLocalSumX_dResultAngle = -resultMagnitude * Math.Sin(resultAngle); + double dLocalSumY_dResultAngle = resultMagnitude * Math.Cos(resultAngle); + + double dLocalSumX_dDeltaX = (dLocalSumX_dResultMagnitude * dResultMagnitude_dDeltaX) + + (dLocalSumX_dResultAngle * dResultAngle_dDeltaX); + double dLocalSumX_dDeltaY = (dLocalSumX_dResultMagnitude * dResultMagnitude_dDeltaY) + + (dLocalSumX_dResultAngle * dResultAngle_dDeltaY); + double dLocalSumY_dDeltaX = (dLocalSumY_dResultMagnitude * dResultMagnitude_dDeltaX) + + (dLocalSumY_dResultAngle * dResultAngle_dDeltaX); + double dLocalSumY_dDeltaY = (dLocalSumY_dResultMagnitude * dResultMagnitude_dDeltaY) + + (dLocalSumY_dResultAngle * dResultAngle_dDeltaY); + + sumX += localSumX; + sumY += localSumY; + + dSumX_dDeltaX[k] = dLocalSumX_dDeltaX; + dSumX_dDeltaY[k] = dLocalSumX_dDeltaY; + dSumY_dDeltaX[k] = dLocalSumY_dDeltaX; + dSumY_dDeltaY[k] = dLocalSumY_dDeltaY; + + // Derivatives of delta components with respect to inputs + dDeltaX_dX1[k] = this.weights[k, j] > 0 ? -1 : 1; // Depending on weight sign + dDeltaY_dY1[k] = this.weights[k, j] > 0 ? -1 : 1; // Depending on weight sign + dDeltaX_dX2[k] = this.weights[k, j] > 0 ? 1 : -1; // Depending on weight sign + dDeltaY_dY2[k] = this.weights[k, j] > 0 ? 1 : -1; // Depending on weight sign + + dX1_dMagnitude[k] = Math.Cos(angle); + dY1_dMagnitude[k] = Math.Sin(angle); + + dX1_dAngle[k] = -magnitude * Math.Sin(angle); + dY1_dAngle[k] = magnitude * Math.Cos(angle); + + dX2_dWMagnitude[k] = Math.Cos(wAngle); + dY2_dWMagnitude[k] = Math.Sin(wAngle); + + dX2_dWAngle[k] = -wMagnitude * Math.Sin(wAngle); + dY2_dWAngle[k] = wMagnitude * Math.Cos(wAngle); + + // Derivatives of delta components with respect to weight + dDeltaX_dWeight[k] = (weights[k, j] > 0) ? (x2 - x1) : (x1 - x2); + dDeltaY_dWeight[k] = (weights[k, j] > 0) ? (y2 - y1) : (y1 - y2); + + dResultMagnitude_dWeight[k] = Math.Sqrt(deltaXYSquared); + dSumX_dResultMagnitude[k] = Math.Cos(resultAngle); + dSumY_dResultMagnitude[k] = Math.Sin(resultAngle); + } + + this.sumX[i, j] = sumX; + this.sumY[i, j] = sumY; + + // Analytically determined gradients for combined magnitude + double magSumXY = (this.sumX[i, j] * this.sumX[i, j]) + (this.sumY[i, j] * this.sumY[i, j]); + double dCombinedMagnitude_dSumX = this.sumX[i, j] / Math.Sqrt(magSumXY); + double dCombinedMagnitude_dSumY = this.sumY[i, j] / Math.Sqrt(magSumXY); + + double dCombinedAngle_dSumX = -this.sumY[i, j] / magSumXY; + double dCombinedAngle_dSumY = this.sumX[i, j] / magSumXY; + + for (int k = 0; k < input2.Rows / 2; k++) + { + dInputMag_dOutputMag[i, k] += + (dCombinedMagnitude_dSumX * dSumX_dDeltaX[k] * dDeltaX_dX1[k] * dX1_dMagnitude[k]) + + (dCombinedMagnitude_dSumY * dSumY_dDeltaY[k] * dDeltaY_dY1[k] * dY1_dMagnitude[k]) + + (dCombinedMagnitude_dSumX * dSumX_dDeltaY[k] * dDeltaY_dY1[k] * dY1_dMagnitude[k]) + + (dCombinedMagnitude_dSumY * dSumY_dDeltaX[k] * dDeltaX_dX1[k] * dX1_dMagnitude[k]); + + dInput2Mag_dOutputMag[k, j] += + (dCombinedMagnitude_dSumX * dSumX_dDeltaX[k] * dDeltaX_dX1[k] * dX2_dWMagnitude[k]) + + (dCombinedMagnitude_dSumY * dSumY_dDeltaY[k] * dDeltaY_dY1[k] * dY2_dWMagnitude[k]) + + (dCombinedMagnitude_dSumX * dSumX_dDeltaY[k] * dDeltaY_dY1[k] * dY2_dWMagnitude[k]) + + (dCombinedMagnitude_dSumY * dSumY_dDeltaX[k] * dDeltaX_dX1[k] * dX2_dWMagnitude[k]); + + dInputMag_dOutputAngle[i, k] += + (dCombinedAngle_dSumX * dSumX_dDeltaX[k] * dDeltaX_dX1[k] * dX1_dMagnitude[k]) + + (dCombinedAngle_dSumY * dSumY_dDeltaY[k] * dDeltaY_dY1[k] * dY1_dMagnitude[k]) + + (dCombinedAngle_dSumX * dSumX_dDeltaY[k] * dDeltaY_dY1[k] * dY1_dMagnitude[k]) + + (dCombinedAngle_dSumY * dSumY_dDeltaX[k] * dDeltaX_dX1[k] * dX1_dMagnitude[k]); + + dInput2Mag_dOutputAngle[k, j] += + (dCombinedAngle_dSumX * dSumX_dDeltaX[k] * dDeltaX_dX2[k] * dX2_dWMagnitude[k]) + + (dCombinedAngle_dSumY * dSumY_dDeltaY[k] * dDeltaY_dY2[k] * dY2_dWMagnitude[k]) + + (dCombinedAngle_dSumX * dSumX_dDeltaY[k] * dDeltaY_dY2[k] * dY2_dWMagnitude[k]) + + (dCombinedAngle_dSumY * dSumY_dDeltaX[k] * dDeltaX_dX2[k] * dX2_dWMagnitude[k]); + + dInputAngle_dOutputMag[i, k] += + (dCombinedMagnitude_dSumX * dSumX_dDeltaX[k] * dDeltaX_dX1[k] * dX1_dAngle[k]) + + (dCombinedMagnitude_dSumY * dSumY_dDeltaY[k] * dDeltaY_dY1[k] * dY1_dAngle[k]) + + (dCombinedMagnitude_dSumX * dSumX_dDeltaY[k] * dDeltaY_dY1[k] * dY1_dAngle[k]) + + (dCombinedMagnitude_dSumY * dSumY_dDeltaX[k] * dDeltaX_dX1[k] * dX1_dAngle[k]); + + dInput2Angle_dOutputMag[k, j] += + (dCombinedMagnitude_dSumX * dSumX_dDeltaX[k] * dDeltaX_dX2[k] * dX2_dWAngle[k]) + + (dCombinedMagnitude_dSumY * dSumY_dDeltaY[k] * dDeltaY_dY2[k] * dY2_dWAngle[k]) + + (dCombinedMagnitude_dSumX * dSumX_dDeltaY[k] * dDeltaY_dY2[k] * dY2_dWAngle[k]) + + (dCombinedMagnitude_dSumY * dSumY_dDeltaX[k] * dDeltaX_dX2[k] * dX2_dWAngle[k]); + + dInputAngle_dOutputAngle[i, k] += + (dCombinedAngle_dSumX * dSumX_dDeltaX[k] * dDeltaX_dX1[k] * dX1_dAngle[k]) + + (dCombinedAngle_dSumY * dSumY_dDeltaY[k] * dDeltaY_dY1[k] * dY1_dAngle[k]) + + (dCombinedAngle_dSumX * dSumX_dDeltaY[k] * dDeltaY_dY1[k] * dY1_dAngle[k]) + + (dCombinedAngle_dSumY * dSumY_dDeltaX[k] * dDeltaX_dX1[k] * dX1_dAngle[k]); + + dInput2Angle_dOutputAngle[k, j] += + (dCombinedAngle_dSumX * dSumX_dDeltaX[k] * dDeltaX_dX2[k] * dX2_dWAngle[k]) + + (dCombinedAngle_dSumY * dSumY_dDeltaY[k] * dDeltaY_dY2[k] * dY2_dWAngle[k]) + + (dCombinedAngle_dSumX * dSumX_dDeltaY[k] * dDeltaY_dY2[k] * dY2_dWAngle[k]) + + (dCombinedAngle_dSumY * dSumY_dDeltaX[k] * dDeltaX_dX2[k] * dX2_dWAngle[k]); + + dWeight_dOutputMag[k, j] += + (dCombinedMagnitude_dSumX * dSumX_dDeltaX[k] * dDeltaX_dWeight[k]) + + (dCombinedMagnitude_dSumY * dSumY_dDeltaY[k] * dDeltaY_dWeight[k]) + + (dCombinedMagnitude_dSumX * dSumX_dDeltaY[k] * dDeltaY_dWeight[k]) + + (dCombinedMagnitude_dSumY * dSumY_dDeltaX[k] * dDeltaX_dWeight[k]) + + (dCombinedMagnitude_dSumX * dSumX_dResultMagnitude[k] * dResultMagnitude_dWeight[k]) + + (dCombinedMagnitude_dSumY * dSumY_dResultMagnitude[k] * dResultMagnitude_dWeight[k]); + + dWeight_dOutputAngle[k, j] += + (dCombinedAngle_dSumX * dSumX_dDeltaX[k] * dDeltaX_dWeight[k]) + + (dCombinedAngle_dSumY * dSumY_dDeltaY[k] * dDeltaY_dWeight[k]) + + (dCombinedAngle_dSumX * dSumX_dDeltaY[k] * dDeltaY_dWeight[k]) + + (dCombinedAngle_dSumY * dSumY_dDeltaX[k] * dDeltaX_dWeight[k]) + + (dCombinedAngle_dSumX * dSumX_dResultMagnitude[k] * dResultMagnitude_dWeight[k]) + + (dCombinedAngle_dSumY * dSumY_dResultMagnitude[k] * dResultMagnitude_dWeight[k]); + } + + this.Output[i, j] = Math.Sqrt((sumX * sumX) + (sumY * sumY)); // Magnitude + this.Output[i, j + (input2.Cols / 2)] = Math.Atan2(sumY, sumX); // Angle in radians + } + }); + + this.calculatedValues = new CalculatedValues + { + DInputMag_dOutputMag = dInputMag_dOutputMag, + DInputMag_dOutputAngle = dInputMag_dOutputAngle, + DInputAngle_dOutputMag = dInputAngle_dOutputMag, + DInputAngle_dOutputAngle = dInputAngle_dOutputAngle, + DInput2Mag_dOutputMag = dInput2Mag_dOutputMag, + DInput2Mag_dOutputAngle = dInput2Mag_dOutputAngle, + DInput2Angle_dOutputMag = dInput2Angle_dOutputMag, + DInput2Angle_dOutputAngle = dInput2Angle_dOutputAngle, + DWeight_dOutputMag = dWeight_dOutputMag, + DWeight_dOutputAngle = dWeight_dOutputAngle, + }; + + return this.Output; + } + + /// + public override BackwardResult Backward(Matrix dOutput) + { + // Initialize gradient matrices + Matrix dInput1 = new Matrix(this.input1.Rows, this.input1.Cols); + Matrix dInput2 = new Matrix(this.input2.Rows, this.input2.Cols); + Matrix dWeights = new Matrix(this.weights.Rows, this.weights.Cols); + + // Loop through each element in input1 + Parallel.For(0, this.input1.Rows, i => + { + for (int k = 0; k < this.input2.Rows / 2; k++) + { + for (int j = 0; j < this.input2.Cols / 2; j++) + { + dInput1[i, k] += dOutput[i, j] * this.calculatedValues.DInputMag_dOutputMag[i, k]; + dInput1[i, k] += dOutput[i, j + (this.input2.Cols / 2)] * this.calculatedValues.DInputMag_dOutputAngle[i, k]; + dInput1[i, k + (this.input1.Cols / 2)] += dOutput[i, j] * this.calculatedValues.DInputAngle_dOutputMag[i, k]; + dInput1[i, k + (this.input1.Cols / 2)] += dOutput[i, j + (this.input2.Cols / 2)] * this.calculatedValues.DInputAngle_dOutputAngle[i, k]; + } + } + }); + + Parallel.For(0, this.input2.Rows / 2, k => + { + for (int j = 0; j < this.input2.Cols / 2; j++) + { + for (int i = 0; i < this.input1.Rows; ++i) + { + dInput2[k, j] += dOutput[i, j] * this.calculatedValues.DInput2Mag_dOutputMag[k, j]; + dInput2[k, j] += dOutput[i, j + (this.input2.Cols / 2)] * this.calculatedValues.DInput2Mag_dOutputAngle[k, j]; + dInput2[k, j + (this.input2.Cols / 2)] += dOutput[i, j] * this.calculatedValues.DInput2Angle_dOutputMag[k, j]; + dInput2[k, j + (this.input2.Cols / 2)] += dOutput[i, j + (this.input2.Cols / 2)] * this.calculatedValues.DInput2Angle_dOutputAngle[k, j]; + + dWeights[k, j] += dOutput[i, j] * this.calculatedValues.DWeight_dOutputMag[k, j]; + dWeights[k, j] += dOutput[i, j + (this.input2.Cols / 2)] * this.calculatedValues.DWeight_dOutputAngle[k, j]; + } + } + }); + + return new BackwardResultBuilder() + .AddInputGradient(dInput1) + .AddInputGradient(dInput2) + .AddInputGradient(dWeights) + .Build(); + } + + private struct CalculatedValues + { + public double[,] DInputMag_dOutputMag { get; internal set; } + + public double[,] DInputMag_dOutputAngle { get; internal set; } + + public double[,] DInputAngle_dOutputMag { get; internal set; } + + public double[,] DInputAngle_dOutputAngle { get; internal set; } + + public double[,] DInput2Mag_dOutputMag { get; internal set; } + + public double[,] DInput2Mag_dOutputAngle { get; internal set; } + + public double[,] DInput2Angle_dOutputMag { get; internal set; } + + public double[,] DInput2Angle_dOutputAngle { get; internal set; } + + public double[,] DWeight_dOutputMag { get; internal set; } + + public double[,] DWeight_dOutputAngle { get; internal set; } + } + } +} diff --git a/src/RMAD/ElementwiseVectorDecompositionOperation.cs b/src/RMAD/ElementwiseVectorDecompositionOperation.cs new file mode 100644 index 00000000..330ff491 --- /dev/null +++ b/src/RMAD/ElementwiseVectorDecompositionOperation.cs @@ -0,0 +1,614 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) 2023 ameritusweb All rights reserved. +// +//------------------------------------------------------------------------------ +namespace ParallelReverseAutoDiff.RMAD +{ + using System; + using System.Threading.Tasks; + + /// + /// Element-wise vector projection operation. + /// + public class ElementwiseVectorDecompositionOperation : Operation + { + private Matrix input1; + private Matrix input2; + private Matrix weights; + private CalculatedValues[,] calculatedValues; + + /// + /// A common method for instantiating an operation. + /// + /// The neural network. + /// The instantiated operation. + public static IOperation Instantiate(NeuralNetwork net) + { + return new ElementwiseVectorDecompositionOperation(); + } + + /// + /// Performs the forward operation for the element-wise vector projection function. + /// + /// The first input to the element-wise vector projection operation. + /// The second input to the element-wise vector projection operation. + /// The weights input to the element-wise vector projection operation. + /// The output of the element-wise vector projection operation. + public Matrix Forward(Matrix input1, Matrix input2, Matrix weights) + { + this.input1 = input1; + this.input2 = input2; + this.weights = weights; + + this.Output = new Matrix(this.input1.Rows, this.input1.Cols * 10); + + this.calculatedValues = new CalculatedValues[this.input1.Rows, this.input1.Cols / 2]; + + Parallel.For(0, input1.Rows, i => + { + for (int j = 0; j < input1.Cols / 2; j++) + { + // Accessing the magnitudes and angles from the concatenated matrices + double magnitude = input1[i, j]; + double angle = input1[i, j + (input1.Cols / 2)]; + + double wMagnitudePivot = input2[i, j * 5]; + double wAnglePivot = input2[i, (j * 5) + (input2.Cols / 2)]; + + double wMagnitude1 = input2[i, (j * 5) + 1]; + double wAngle1 = input2[i, (j * 5) + 1 + (input2.Cols / 2)]; + + double wMagnitude2 = input2[i, (j * 5) + 2]; + double wAngle2 = input2[i, (j * 5) + 2 + (input2.Cols / 2)]; + + double wMagnitude3 = input2[i, (j * 5) + 3]; + double wAngle3 = input2[i, (j * 5) + 3 + (input2.Cols / 2)]; + + double wMagnitude4 = input2[i, (j * 5) + 4]; + double wAngle4 = input2[i, (j * 5) + 4 + (input2.Cols / 2)]; + + // Compute vector components + double x = magnitude * Math.Cos(angle); + double y = magnitude * Math.Sin(angle); + double xPivot = wMagnitudePivot * Math.Cos(wAnglePivot); + double yPivot = wMagnitudePivot * Math.Sin(wAnglePivot); + + double dx_dMagnitude = Math.Cos(angle); + double dx_dAngle = -magnitude * Math.Sin(angle); + double dy_dMagnitude = Math.Sin(angle); + double dy_dAngle = magnitude * Math.Cos(angle); + double dXPivot_dWMagnitudePivot = Math.Cos(wAnglePivot); + double dXPivot_dWAnglePivot = -wMagnitudePivot * Math.Sin(wAnglePivot); + double dYPivot_dWMagnitudePivot = Math.Sin(wAnglePivot); + double dYPivot_dWAnglePivot = wMagnitudePivot * Math.Cos(wAnglePivot); + + this.calculatedValues[i, j] = new CalculatedValues() + { + CV_dx_dMagnitude = dx_dMagnitude, + CV_dx_dAngle = dx_dAngle, + CV_dy_dMagnitude = dy_dMagnitude, + CV_dy_dAngle = dy_dAngle, + CV_dXPivot_dWMagnitudePivot = dXPivot_dWMagnitudePivot, + CV_dXPivot_dWAnglePivot = dXPivot_dWAnglePivot, + CV_dYPivot_dWMagnitudePivot = dYPivot_dWMagnitudePivot, + CV_dYPivot_dWAnglePivot = dYPivot_dWAnglePivot, + }; + + double x1 = wMagnitude1 * Math.Cos(wAngle1); + double y1 = wMagnitude1 * Math.Sin(wAngle1); + + double dX1_wMagnitude1 = Math.Cos(wAngle1); + double dX1_wAngle1 = -wMagnitude1 * Math.Sin(wAngle1); + double dY1_wMagnitude1 = Math.Sin(wAngle1); + double dY1_wAngle1 = wMagnitude1 * Math.Cos(wAngle1); + + this.calculatedValues[i, j].CV_dX1_wMagnitude1 = dX1_wMagnitude1; + this.calculatedValues[i, j].CV_dX1_wAngle1 = dX1_wAngle1; + this.calculatedValues[i, j].CV_dY1_wMagnitude1 = dY1_wMagnitude1; + this.calculatedValues[i, j].CV_dY1_wAngle1 = dY1_wAngle1; + + double x2 = wMagnitude2 * Math.Cos(wAngle2); + double y2 = wMagnitude2 * Math.Sin(wAngle2); + + double dX2_wMagnitude2 = Math.Cos(wAngle2); + double dX2_wAngle2 = -wMagnitude2 * Math.Sin(wAngle2); + double dY2_wMagnitude2 = Math.Sin(wAngle2); + double dY2_wAngle2 = wMagnitude2 * Math.Cos(wAngle2); + + this.calculatedValues[i, j].CV_dX2_wMagnitude2 = dX2_wMagnitude2; + this.calculatedValues[i, j].CV_dX2_wAngle2 = dX2_wAngle2; + this.calculatedValues[i, j].CV_dY2_wMagnitude2 = dY2_wMagnitude2; + this.calculatedValues[i, j].CV_dY2_wAngle2 = dY2_wAngle2; + + double x3 = wMagnitude3 * Math.Cos(wAngle3); + double y3 = wMagnitude3 * Math.Sin(wAngle3); + + double dX3_wMagnitude3 = Math.Cos(wAngle3); + double dX3_wAngle3 = -wMagnitude3 * Math.Sin(wAngle3); + double dY3_wMagnitude3 = Math.Sin(wAngle3); + double dY3_wAngle3 = wMagnitude3 * Math.Cos(wAngle3); + + this.calculatedValues[i, j].CV_dX3_wMagnitude3 = dX3_wMagnitude3; + this.calculatedValues[i, j].CV_dX3_wAngle3 = dX3_wAngle3; + this.calculatedValues[i, j].CV_dY3_wMagnitude3 = dY3_wMagnitude3; + this.calculatedValues[i, j].CV_dY3_wAngle3 = dY3_wAngle3; + + double x4 = wMagnitude4 * Math.Cos(wAngle4); + double y4 = wMagnitude4 * Math.Sin(wAngle4); + + double dX4_wMagnitude4 = Math.Cos(wAngle4); + double dX4_wAngle4 = -wMagnitude4 * Math.Sin(wAngle4); + double dY4_wMagnitude4 = Math.Sin(wAngle4); + double dY4_wAngle4 = wMagnitude4 * Math.Cos(wAngle4); + + this.calculatedValues[i, j].CV_dX4_wMagnitude4 = dX4_wMagnitude4; + this.calculatedValues[i, j].CV_dX4_wAngle4 = dX4_wAngle4; + this.calculatedValues[i, j].CV_dY4_wMagnitude4 = dY4_wMagnitude4; + this.calculatedValues[i, j].CV_dY4_wAngle4 = dY4_wAngle4; + + double sumx = (x + xPivot) / (this.weights[i, j] + 1E-9); + double sumy = (y + yPivot) / (this.weights[i, j] + 1E-9); + + double dsumx_dX = 1d / (this.weights[i, j] + 1E-9); + double dsumx_dXPivot = 1d / (this.weights[i, j] + 1E-9); + double dsumx_dWeight = -(x + xPivot) / ((this.weights[i, j] + 1E-9) * (this.weights[i, j] + 1E-9)); + double dsumy_dY = 1d / (this.weights[i, j] + 1E-9); + double dsumy_dYPivot = 1d / (this.weights[i, j] + 1E-9); + double dsumy_dWeight = -(y + yPivot) / ((this.weights[i, j] + 1E-9) * (this.weights[i, j] + 1E-9)); + + this.calculatedValues[i, j].CV_dsumx_dX = dsumx_dX; + this.calculatedValues[i, j].CV_dsumx_dXPivot = dsumx_dXPivot; + this.calculatedValues[i, j].CV_dsumx_dWeight = dsumx_dWeight; + this.calculatedValues[i, j].CV_dsumy_dY = dsumy_dY; + this.calculatedValues[i, j].CV_dsumy_dYPivot = dsumy_dYPivot; + this.calculatedValues[i, j].CV_dsumy_dWeight = dsumy_dWeight; + + double diffx1 = sumx - x1; + double diffy1 = sumy - y1; + + double dDiffX1_dSumX = 1d; + double dDiffX1_dX1 = -1d; + double dDiffY1_dSumY = 1d; + double dDiffY1_dY1 = -1d; + + this.calculatedValues[i, j].CV_dDiffX1_dSumX = dDiffX1_dSumX; + this.calculatedValues[i, j].CV_dDiffX1_dX1 = dDiffX1_dX1; + this.calculatedValues[i, j].CV_dDiffY1_dSumY = dDiffY1_dSumY; + this.calculatedValues[i, j].CV_dDiffY1_dY1 = dDiffY1_dY1; + + double diffx2 = -sumx - x2; + double diffy2 = -sumy - y2; + + double dDiffX2_dSumX = -1d; + double dDiffX2_dX2 = -1d; + double dDiffY2_dSumY = -1d; + double dDiffY2_dY2 = -1d; + + this.calculatedValues[i, j].CV_dDiffX2_dSumX = dDiffX2_dSumX; + this.calculatedValues[i, j].CV_dDiffX2_dX2 = dDiffX2_dX2; + this.calculatedValues[i, j].CV_dDiffY2_dSumY = dDiffY2_dSumY; + this.calculatedValues[i, j].CV_dDiffY2_dY2 = dDiffY2_dY2; + + double diffx3 = sumx - x3; + double diffy3 = sumy - y3; + + double dDiffX3_dSumX = 1d; + double dDiffX3_dX3 = -1d; + double dDiffY3_dSumY = 1d; + double dDiffY3_dY3 = -1d; + + this.calculatedValues[i, j].CV_dDiffX3_dSumX = dDiffX3_dSumX; + this.calculatedValues[i, j].CV_dDiffX3_dX3 = dDiffX3_dX3; + this.calculatedValues[i, j].CV_dDiffY3_dSumY = dDiffY3_dSumY; + this.calculatedValues[i, j].CV_dDiffY3_dY3 = dDiffY3_dY3; + + double diffx4 = -sumx - x4; + double diffy4 = -sumy - y4; + + double dDiffX4_dSumX = -1d; + double dDiffX4_dX4 = -1d; + double dDiffY4_dSumY = -1d; + double dDiffY4_dY4 = -1d; + + this.calculatedValues[i, j].CV_dDiffX4_dSumX = dDiffX4_dSumX; + this.calculatedValues[i, j].CV_dDiffX4_dX4 = dDiffX4_dX4; + this.calculatedValues[i, j].CV_dDiffY4_dSumY = dDiffY4_dSumY; + this.calculatedValues[i, j].CV_dDiffY4_dY4 = dDiffY4_dY4; + + // Compute resultant vector magnitude and angle + double resultMagnitude1 = Math.Sqrt((diffx1 * diffx1) + (diffy1 * diffy1)); + double resultAngle1 = Math.Atan2(diffy1, diffx1); + + double dResultMagnitude1_dDiffX1 = diffx1 / resultMagnitude1; + double dResultMagnitude1_dDiffY1 = diffy1 / resultMagnitude1; + double dResultAngle1_dDiffX1 = -diffy1 / ((diffx1 * diffx1) + (diffy1 * diffy1)); + double dResultAngle1_dDiffY1 = diffx1 / ((diffx1 * diffx1) + (diffy1 * diffy1)); + + this.calculatedValues[i, j].CV_dResultMagnitude1_dDiffX1 = dResultMagnitude1_dDiffX1; + this.calculatedValues[i, j].CV_dResultMagnitude1_dDiffY1 = dResultMagnitude1_dDiffY1; + this.calculatedValues[i, j].CV_dResultAngle1_dDiffX1 = dResultAngle1_dDiffX1; + this.calculatedValues[i, j].CV_dResultAngle1_dDiffY1 = dResultAngle1_dDiffY1; + + double resultMagnitude2 = Math.Sqrt((diffx2 * diffx2) + (diffy2 * diffy2)); + double resultAngle2 = Math.Atan2(diffy2, diffx2); + + double dResultMagnitude2_dDiffX2 = diffx2 / resultMagnitude2; + double dResultMagnitude2_dDiffY2 = diffy2 / resultMagnitude2; + double dResultAngle2_dDiffX2 = -diffy2 / ((diffx2 * diffx2) + (diffy2 * diffy2)); + double dResultAngle2_dDiffY2 = diffx2 / ((diffx2 * diffx2) + (diffy2 * diffy2)); + + this.calculatedValues[i, j].CV_dResultMagnitude2_dDiffX2 = dResultMagnitude2_dDiffX2; + this.calculatedValues[i, j].CV_dResultMagnitude2_dDiffY2 = dResultMagnitude2_dDiffY2; + this.calculatedValues[i, j].CV_dResultAngle2_dDiffX2 = dResultAngle2_dDiffX2; + this.calculatedValues[i, j].CV_dResultAngle2_dDiffY2 = dResultAngle2_dDiffY2; + + double resultMagnitude3 = Math.Sqrt((diffx3 * diffx3) + (diffy3 * diffy3)); + double resultAngle3 = Math.Atan2(diffy3, diffx3); + + double dResultMagnitude3_dDiffX3 = diffx3 / resultMagnitude3; + double dResultMagnitude3_dDiffY3 = diffy3 / resultMagnitude3; + double dResultAngle3_dDiffX3 = -diffy3 / ((diffx3 * diffx3) + (diffy3 * diffy3)); + double dResultAngle3_dDiffY3 = diffx3 / ((diffx3 * diffx3) + (diffy3 * diffy3)); + + this.calculatedValues[i, j].CV_dResultMagnitude3_dDiffX3 = dResultMagnitude3_dDiffX3; + this.calculatedValues[i, j].CV_dResultMagnitude3_dDiffY3 = dResultMagnitude3_dDiffY3; + this.calculatedValues[i, j].CV_dResultAngle3_dDiffX3 = dResultAngle3_dDiffX3; + this.calculatedValues[i, j].CV_dResultAngle3_dDiffY3 = dResultAngle3_dDiffY3; + + double resultMagnitude4 = Math.Sqrt((diffx4 * diffx4) + (diffy4 * diffy4)); + double resultAngle4 = Math.Atan2(diffy4, diffx4); + + double dResultMagnitude4_dDiffX4 = diffx4 / resultMagnitude4; + double dResultMagnitude4_dDiffY4 = diffy4 / resultMagnitude4; + double dResultAngle4_dDiffX4 = -diffy4 / ((diffx4 * diffx4) + (diffy4 * diffy4)); + double dResultAngle4_dDiffY4 = diffx4 / ((diffx4 * diffx4) + (diffy4 * diffy4)); + + this.calculatedValues[i, j].CV_dResultMagnitude4_dDiffX4 = dResultMagnitude4_dDiffX4; + this.calculatedValues[i, j].CV_dResultMagnitude4_dDiffY4 = dResultMagnitude4_dDiffY4; + this.calculatedValues[i, j].CV_dResultAngle4_dDiffX4 = dResultAngle4_dDiffX4; + this.calculatedValues[i, j].CV_dResultAngle4_dDiffY4 = dResultAngle4_dDiffY4; + + this.Output[i, j * 10] = magnitude; + this.Output[i, (j * 10) + (this.input1.Cols * 10 / 2)] = angle; + + this.Output[i, (j * 10) + 1] = wMagnitudePivot; + this.Output[i, (j * 10) + 1 + (this.input1.Cols * 10 / 2)] = wAnglePivot; + + this.Output[i, (j * 10) + 2] = wMagnitude1; + this.Output[i, (j * 10) + 2 + (this.input1.Cols * 10 / 2)] = wAngle1; + + this.Output[i, (j * 10) + 3] = wMagnitude2; + this.Output[i, (j * 10) + 3 + (this.input1.Cols * 10 / 2)] = wAngle2; + + this.Output[i, (j * 10) + 4] = wMagnitude3; + this.Output[i, (j * 10) + 4 + (this.input1.Cols * 10 / 2)] = wAngle3; + + this.Output[i, (j * 10) + 5] = wMagnitude4; + this.Output[i, (j * 10) + 5 + (this.input1.Cols * 10 / 2)] = wAngle4; + + this.Output[i, (j * 10) + 6] = resultMagnitude1; + this.Output[i, (j * 10) + 6 + (this.input1.Cols * 10 / 2)] = resultAngle1; + + this.Output[i, (j * 10) + 7] = resultMagnitude2; + this.Output[i, (j * 10) + 7 + (this.input1.Cols * 10 / 2)] = resultAngle2; + + this.Output[i, (j * 10) + 8] = resultMagnitude3; + this.Output[i, (j * 10) + 8 + (this.input1.Cols * 10 / 2)] = resultAngle3; + + this.Output[i, (j * 10) + 9] = resultMagnitude4; + this.Output[i, (j * 10) + 9 + (this.input1.Cols * 10 / 2)] = resultAngle4; + } + }); + + return this.Output; + } + + /// + public override BackwardResult Backward(Matrix dOutput) + { + Matrix dInput1 = new Matrix(this.input1.Rows, this.input1.Cols); + Matrix dInput2 = new Matrix(this.input2.Rows, this.input2.Cols); + Matrix dWeights = new Matrix(this.weights.Rows, this.weights.Cols); + + Parallel.For(0, this.input1.Rows, i => + { + for (int j = 0; j < this.input1.Cols / 2; j++) + { + var grad = this.calculatedValues[i, j]; + + dInput1[i, j] += dOutput[i, j * 10]; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 10) + (this.input1.Cols * 10 / 2)]; + + dInput1[i, j] += dOutput[i, (j * 10) + 6] * grad.CV_dResultMagnitude1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 10) + 6] * grad.CV_dResultMagnitude1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 10) + 6 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 10) + 6 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dMagnitude; + + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 10) + 6] * grad.CV_dResultAngle1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 10) + 6 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 10) + 6] * grad.CV_dResultMagnitude1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 10) + 6 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultMagnitude1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dAngle; + + dInput1[i, j] += dOutput[i, (j * 10) + 7] * grad.CV_dResultMagnitude2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 10) + 7] * grad.CV_dResultMagnitude2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 10) + 7 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 10) + 7 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dMagnitude; + + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 10) + 7] * grad.CV_dResultAngle2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 10) + 7 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 10) + 7] * grad.CV_dResultMagnitude2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 10) + 7 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultMagnitude2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dAngle; + + dInput1[i, j] += dOutput[i, (j * 10) + 8] * grad.CV_dResultMagnitude3_dDiffX3 * grad.CV_dDiffX3_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 10) + 8] * grad.CV_dResultMagnitude3_dDiffY3 * grad.CV_dDiffY3_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 10) + 8 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle3_dDiffX3 * grad.CV_dDiffX3_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 10) + 8 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle3_dDiffY3 * grad.CV_dDiffY3_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dMagnitude; + + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 10) + 8] * grad.CV_dResultAngle3_dDiffX3 * grad.CV_dDiffX3_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 10) + 8 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle3_dDiffY3 * grad.CV_dDiffY3_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 10) + 8] * grad.CV_dResultMagnitude3_dDiffX3 * grad.CV_dDiffX3_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 10) + 8 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultMagnitude3_dDiffY3 * grad.CV_dDiffY3_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dAngle; + + dInput1[i, j] += dOutput[i, (j * 10) + 9] * grad.CV_dResultMagnitude4_dDiffX4 * grad.CV_dDiffX4_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 10) + 9] * grad.CV_dResultMagnitude4_dDiffY4 * grad.CV_dDiffY4_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 10) + 9 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle4_dDiffX4 * grad.CV_dDiffX4_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 10) + 9 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle4_dDiffY4 * grad.CV_dDiffY4_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dMagnitude; + + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 10) + 9] * grad.CV_dResultAngle4_dDiffX4 * grad.CV_dDiffX4_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 10) + 9 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle4_dDiffY4 * grad.CV_dDiffY4_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 10) + 9] * grad.CV_dResultMagnitude4_dDiffX4 * grad.CV_dDiffX4_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 10) + 9 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultMagnitude4_dDiffY4 * grad.CV_dDiffY4_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dAngle; + + dInput2[i, j * 5] += dOutput[i, (j * 10) + 1]; + dInput2[i, (j * 5) + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 1 + (this.input1.Cols * 10 / 2)]; + + dInput2[i, (j * 5) + 1] += dOutput[i, (j * 10) + 2]; + dInput2[i, (j * 5) + 1 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 2 + (this.input1.Cols * 10 / 2)]; + + dInput2[i, (j * 5) + 2] += dOutput[i, (j * 10) + 3]; + dInput2[i, (j * 5) + 2 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 3 + (this.input1.Cols * 10 / 2)]; + + dInput2[i, (j * 5) + 3] += dOutput[i, (j * 10) + 4]; + dInput2[i, (j * 5) + 3 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 4 + (this.input1.Cols * 10 / 2)]; + + dInput2[i, (j * 5) + 4] += dOutput[i, (j * 10) + 5]; + dInput2[i, (j * 5) + 4 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 5 + (this.input1.Cols * 10 / 2)]; + + dInput2[i, j * 5] += dOutput[i, (j * 10) + 6] * grad.CV_dResultMagnitude1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWMagnitudePivot; + dInput2[i, j * 5] += dOutput[i, (j * 10) + 6] * grad.CV_dResultMagnitude1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWMagnitudePivot; + dInput2[i, j * 5] += dOutput[i, (j * 10) + 6 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWMagnitudePivot; + dInput2[i, j * 5] += dOutput[i, (j * 10) + 6 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWMagnitudePivot; + + dInput2[i, (j * 5) + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 6] * grad.CV_dResultMagnitude1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWAnglePivot; + dInput2[i, (j * 5) + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 6 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWAnglePivot; + dInput2[i, (j * 5) + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 6] * grad.CV_dResultMagnitude1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWAnglePivot; + dInput2[i, (j * 5) + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 6 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWAnglePivot; + + dInput2[i, j * 5] += dOutput[i, (j * 10) + 7] * grad.CV_dResultMagnitude2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWMagnitudePivot; + dInput2[i, j * 5] += dOutput[i, (j * 10) + 7] * grad.CV_dResultMagnitude2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWMagnitudePivot; + dInput2[i, j * 5] += dOutput[i, (j * 10) + 7 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWMagnitudePivot; + dInput2[i, j * 5] += dOutput[i, (j * 10) + 7 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWMagnitudePivot; + + dInput2[i, (j * 5) + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 7] * grad.CV_dResultMagnitude2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWAnglePivot; + dInput2[i, (j * 5) + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 7 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWAnglePivot; + dInput2[i, (j * 5) + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 7] * grad.CV_dResultMagnitude2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWAnglePivot; + dInput2[i, (j * 5) + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 7 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWAnglePivot; + + dInput2[i, j * 5] += dOutput[i, (j * 10) + 8] * grad.CV_dResultMagnitude3_dDiffX3 * grad.CV_dDiffX3_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWMagnitudePivot; + dInput2[i, j * 5] += dOutput[i, (j * 10) + 8] * grad.CV_dResultMagnitude3_dDiffY3 * grad.CV_dDiffY3_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWMagnitudePivot; + dInput2[i, j * 5] += dOutput[i, (j * 10) + 8 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle3_dDiffX3 * grad.CV_dDiffX3_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWMagnitudePivot; + dInput2[i, j * 5] += dOutput[i, (j * 10) + 8 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle3_dDiffY3 * grad.CV_dDiffY3_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWMagnitudePivot; + + dInput2[i, (j * 5) + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 8] * grad.CV_dResultMagnitude3_dDiffX3 * grad.CV_dDiffX3_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWAnglePivot; + dInput2[i, (j * 5) + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 8 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle3_dDiffX3 * grad.CV_dDiffX3_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWAnglePivot; + dInput2[i, (j * 5) + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 8] * grad.CV_dResultMagnitude3_dDiffY3 * grad.CV_dDiffY3_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWAnglePivot; + dInput2[i, (j * 5) + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 8 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle3_dDiffY3 * grad.CV_dDiffY3_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWAnglePivot; + + dInput2[i, j * 5] += dOutput[i, (j * 10) + 9] * grad.CV_dResultMagnitude4_dDiffX4 * grad.CV_dDiffX4_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWMagnitudePivot; + dInput2[i, j * 5] += dOutput[i, (j * 10) + 9] * grad.CV_dResultMagnitude4_dDiffY4 * grad.CV_dDiffY4_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWMagnitudePivot; + dInput2[i, j * 5] += dOutput[i, (j * 10) + 9 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle4_dDiffX4 * grad.CV_dDiffX4_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWMagnitudePivot; + dInput2[i, j * 5] += dOutput[i, (j * 10) + 9 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle4_dDiffY4 * grad.CV_dDiffY4_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWMagnitudePivot; + + dInput2[i, (j * 5) + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 9] * grad.CV_dResultMagnitude4_dDiffX4 * grad.CV_dDiffX4_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWAnglePivot; + dInput2[i, (j * 5) + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 9 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle4_dDiffX4 * grad.CV_dDiffX4_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWAnglePivot; + dInput2[i, (j * 5) + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 9] * grad.CV_dResultMagnitude4_dDiffY4 * grad.CV_dDiffY4_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWAnglePivot; + dInput2[i, (j * 5) + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 9 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle4_dDiffY4 * grad.CV_dDiffY4_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWAnglePivot; + + dInput2[i, (j * 5) + 1] += dOutput[i, (j * 10) + 6] * grad.CV_dResultMagnitude1_dDiffX1 * grad.CV_dDiffX1_dX1 * grad.CV_dX1_wMagnitude1; + dInput2[i, (j * 5) + 1] += dOutput[i, (j * 10) + 6] * grad.CV_dResultMagnitude1_dDiffY1 * grad.CV_dDiffY1_dY1 * grad.CV_dY1_wMagnitude1; + dInput2[i, (j * 5) + 1] += dOutput[i, (j * 10) + 6 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle1_dDiffX1 * grad.CV_dDiffX1_dX1 * grad.CV_dX1_wMagnitude1; + dInput2[i, (j * 5) + 1] += dOutput[i, (j * 10) + 6 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle1_dDiffY1 * grad.CV_dDiffY1_dY1 * grad.CV_dY1_wMagnitude1; + + dInput2[i, (j * 5) + 1 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 6] * grad.CV_dResultMagnitude1_dDiffX1 * grad.CV_dDiffX1_dX1 * grad.CV_dX1_wAngle1; + dInput2[i, (j * 5) + 1 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 6 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle1_dDiffX1 * grad.CV_dDiffX1_dX1 * grad.CV_dX1_wAngle1; + dInput2[i, (j * 5) + 1 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 6] * grad.CV_dResultMagnitude1_dDiffY1 * grad.CV_dDiffY1_dY1 * grad.CV_dY1_wAngle1; + dInput2[i, (j * 5) + 1 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 6 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle1_dDiffY1 * grad.CV_dDiffY1_dY1 * grad.CV_dY1_wAngle1; + + dInput2[i, (j * 5) + 2] += dOutput[i, (j * 10) + 7] * grad.CV_dResultMagnitude2_dDiffX2 * grad.CV_dDiffX2_dX2 * grad.CV_dX2_wMagnitude2; + dInput2[i, (j * 5) + 2] += dOutput[i, (j * 10) + 7] * grad.CV_dResultMagnitude2_dDiffY2 * grad.CV_dDiffY2_dY2 * grad.CV_dY2_wMagnitude2; + dInput2[i, (j * 5) + 2] += dOutput[i, (j * 10) + 7 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle2_dDiffX2 * grad.CV_dDiffX2_dX2 * grad.CV_dX2_wMagnitude2; + dInput2[i, (j * 5) + 2] += dOutput[i, (j * 10) + 7 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle2_dDiffY2 * grad.CV_dDiffY2_dY2 * grad.CV_dY2_wMagnitude2; + + dInput2[i, (j * 5) + 2 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 7] * grad.CV_dResultMagnitude2_dDiffX2 * grad.CV_dDiffX2_dX2 * grad.CV_dX2_wAngle2; + dInput2[i, (j * 5) + 2 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 7 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle2_dDiffX2 * grad.CV_dDiffX2_dX2 * grad.CV_dX2_wAngle2; + dInput2[i, (j * 5) + 2 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 7] * grad.CV_dResultMagnitude2_dDiffY2 * grad.CV_dDiffY2_dY2 * grad.CV_dY2_wAngle2; + dInput2[i, (j * 5) + 2 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 7 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle2_dDiffY2 * grad.CV_dDiffY2_dY2 * grad.CV_dY2_wAngle2; + + dInput2[i, (j * 5) + 3] += dOutput[i, (j * 10) + 8] * grad.CV_dResultMagnitude3_dDiffX3 * grad.CV_dDiffX3_dX3 * grad.CV_dX3_wMagnitude3; + dInput2[i, (j * 5) + 3] += dOutput[i, (j * 10) + 8] * grad.CV_dResultMagnitude3_dDiffY3 * grad.CV_dDiffY3_dY3 * grad.CV_dY3_wMagnitude3; + dInput2[i, (j * 5) + 3] += dOutput[i, (j * 10) + 8 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle3_dDiffX3 * grad.CV_dDiffX3_dX3 * grad.CV_dX3_wMagnitude3; + dInput2[i, (j * 5) + 3] += dOutput[i, (j * 10) + 8 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle3_dDiffY3 * grad.CV_dDiffY3_dY3 * grad.CV_dY3_wMagnitude3; + + dInput2[i, (j * 5) + 3 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 8] * grad.CV_dResultMagnitude3_dDiffX3 * grad.CV_dDiffX3_dX3 * grad.CV_dX3_wAngle3; + dInput2[i, (j * 5) + 3 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 8 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle3_dDiffX3 * grad.CV_dDiffX3_dX3 * grad.CV_dX3_wAngle3; + dInput2[i, (j * 5) + 3 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 8] * grad.CV_dResultMagnitude3_dDiffY3 * grad.CV_dDiffY3_dY3 * grad.CV_dY3_wAngle3; + dInput2[i, (j * 5) + 3 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 8 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle3_dDiffY3 * grad.CV_dDiffY3_dY3 * grad.CV_dY3_wAngle3; + + dInput2[i, (j * 5) + 4] += dOutput[i, (j * 10) + 9] * grad.CV_dResultMagnitude4_dDiffX4 * grad.CV_dDiffX4_dX4 * grad.CV_dX4_wMagnitude4; + dInput2[i, (j * 5) + 4] += dOutput[i, (j * 10) + 9] * grad.CV_dResultMagnitude4_dDiffY4 * grad.CV_dDiffY4_dY4 * grad.CV_dY4_wMagnitude4; + dInput2[i, (j * 5) + 4] += dOutput[i, (j * 10) + 9 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle4_dDiffX4 * grad.CV_dDiffX4_dX4 * grad.CV_dX4_wMagnitude4; + dInput2[i, (j * 5) + 4] += dOutput[i, (j * 10) + 9 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle4_dDiffY4 * grad.CV_dDiffY4_dY4 * grad.CV_dY4_wMagnitude4; + + dInput2[i, (j * 5) + 4 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 9] * grad.CV_dResultMagnitude4_dDiffX4 * grad.CV_dDiffX4_dX4 * grad.CV_dX4_wAngle4; + dInput2[i, (j * 5) + 4 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 9 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle4_dDiffX4 * grad.CV_dDiffX4_dX4 * grad.CV_dX4_wAngle4; + dInput2[i, (j * 5) + 4 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 9] * grad.CV_dResultMagnitude4_dDiffY4 * grad.CV_dDiffY4_dY4 * grad.CV_dY4_wAngle4; + dInput2[i, (j * 5) + 4 + (this.input2.Cols / 2)] += dOutput[i, (j * 10) + 9 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle4_dDiffY4 * grad.CV_dDiffY4_dY4 * grad.CV_dY4_wAngle4; + + dWeights[i, j] += dOutput[i, (j * 10) + 6] * grad.CV_dResultMagnitude1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dWeight; + dWeights[i, j] += dOutput[i, (j * 10) + 6] * grad.CV_dResultMagnitude1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dWeight; + dWeights[i, j] += dOutput[i, (j * 10) + 6 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dWeight; + dWeights[i, j] += dOutput[i, (j * 10) + 6 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dWeight; + + dWeights[i, j] += dOutput[i, (j * 10) + 7] * grad.CV_dResultMagnitude2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dWeight; + dWeights[i, j] += dOutput[i, (j * 10) + 7] * grad.CV_dResultMagnitude2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dWeight; + dWeights[i, j] += dOutput[i, (j * 10) + 7 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dWeight; + dWeights[i, j] += dOutput[i, (j * 10) + 7 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dWeight; + + dWeights[i, j] += dOutput[i, (j * 10) + 8] * grad.CV_dResultMagnitude3_dDiffX3 * grad.CV_dDiffX3_dSumX * grad.CV_dsumx_dWeight; + dWeights[i, j] += dOutput[i, (j * 10) + 8] * grad.CV_dResultMagnitude3_dDiffY3 * grad.CV_dDiffY3_dSumY * grad.CV_dsumy_dWeight; + dWeights[i, j] += dOutput[i, (j * 10) + 8 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle3_dDiffX3 * grad.CV_dDiffX3_dSumX * grad.CV_dsumx_dWeight; + dWeights[i, j] += dOutput[i, (j * 10) + 8 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle3_dDiffY3 * grad.CV_dDiffY3_dSumY * grad.CV_dsumy_dWeight; + + dWeights[i, j] += dOutput[i, (j * 10) + 9] * grad.CV_dResultMagnitude4_dDiffX4 * grad.CV_dDiffX4_dSumX * grad.CV_dsumx_dWeight; + dWeights[i, j] += dOutput[i, (j * 10) + 9] * grad.CV_dResultMagnitude4_dDiffY4 * grad.CV_dDiffY4_dSumY * grad.CV_dsumy_dWeight; + dWeights[i, j] += dOutput[i, (j * 10) + 9 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle4_dDiffX4 * grad.CV_dDiffX4_dSumX * grad.CV_dsumx_dWeight; + dWeights[i, j] += dOutput[i, (j * 10) + 9 + (this.input1.Cols * 10 / 2)] * grad.CV_dResultAngle4_dDiffY4 * grad.CV_dDiffY4_dSumY * grad.CV_dsumy_dWeight; + } + }); + + return new BackwardResultBuilder() + .AddInputGradient(dInput1) + .AddInputGradient(dInput2) + .AddInputGradient(dWeights) + .Build(); + } + + private struct CalculatedValues + { + public double CV_dx_dMagnitude { get; internal set; } + + public double CV_dx_dAngle { get; internal set; } + + public double CV_dy_dMagnitude { get; internal set; } + + public double CV_dy_dAngle { get; internal set; } + + public double CV_dXPivot_dWMagnitudePivot { get; internal set; } + + public double CV_dXPivot_dWAnglePivot { get; internal set; } + + public double CV_dYPivot_dWMagnitudePivot { get; internal set; } + + public double CV_dYPivot_dWAnglePivot { get; internal set; } + + public double CV_dX1_wMagnitude1 { get; internal set; } + + public double CV_dX1_wAngle1 { get; internal set; } + + public double CV_dY1_wMagnitude1 { get; internal set; } + + public double CV_dY1_wAngle1 { get; internal set; } + + public double CV_dX2_wMagnitude2 { get; internal set; } + + public double CV_dX2_wAngle2 { get; internal set; } + + public double CV_dY2_wMagnitude2 { get; internal set; } + + public double CV_dY2_wAngle2 { get; internal set; } + + public double CV_dX3_wMagnitude3 { get; internal set; } + + public double CV_dX3_wAngle3 { get; internal set; } + + public double CV_dY3_wMagnitude3 { get; internal set; } + + public double CV_dY3_wAngle3 { get; internal set; } + + public double CV_dX4_wMagnitude4 { get; internal set; } + + public double CV_dX4_wAngle4 { get; internal set; } + + public double CV_dY4_wMagnitude4 { get; internal set; } + + public double CV_dY4_wAngle4 { get; internal set; } + + public double CV_dsumx_dX { get; internal set; } + + public double CV_dsumx_dXPivot { get; internal set; } + + public double CV_dsumx_dWeight { get; internal set; } + + public double CV_dsumy_dY { get; internal set; } + + public double CV_dsumy_dYPivot { get; internal set; } + + public double CV_dsumy_dWeight { get; internal set; } + + public double CV_dDiffX1_dSumX { get; internal set; } + + public double CV_dDiffX1_dX1 { get; internal set; } + + public double CV_dDiffY1_dSumY { get; internal set; } + + public double CV_dDiffY1_dY1 { get; internal set; } + + public double CV_dDiffX2_dSumX { get; internal set; } + + public double CV_dDiffX2_dX2 { get; internal set; } + + public double CV_dDiffY2_dSumY { get; internal set; } + + public double CV_dDiffY2_dY2 { get; internal set; } + + public double CV_dDiffX3_dSumX { get; internal set; } + + public double CV_dDiffX3_dX3 { get; internal set; } + + public double CV_dDiffY3_dSumY { get; internal set; } + + public double CV_dDiffY3_dY3 { get; internal set; } + + public double CV_dDiffX4_dSumX { get; internal set; } + + public double CV_dDiffX4_dX4 { get; internal set; } + + public double CV_dDiffY4_dSumY { get; internal set; } + + public double CV_dDiffY4_dY4 { get; internal set; } + + public double CV_dResultMagnitude1_dDiffX1 { get; internal set; } + + public double CV_dResultMagnitude1_dDiffY1 { get; internal set; } + + public double CV_dResultAngle1_dDiffX1 { get; internal set; } + + public double CV_dResultAngle1_dDiffY1 { get; internal set; } + + public double CV_dResultMagnitude2_dDiffX2 { get; internal set; } + + public double CV_dResultMagnitude2_dDiffY2 { get; internal set; } + + public double CV_dResultAngle2_dDiffX2 { get; internal set; } + + public double CV_dResultAngle2_dDiffY2 { get; internal set; } + + public double CV_dResultMagnitude3_dDiffX3 { get; internal set; } + + public double CV_dResultMagnitude3_dDiffY3 { get; internal set; } + + public double CV_dResultAngle3_dDiffX3 { get; internal set; } + + public double CV_dResultAngle3_dDiffY3 { get; internal set; } + + public double CV_dResultMagnitude4_dDiffX4 { get; internal set; } + + public double CV_dResultMagnitude4_dDiffY4 { get; internal set; } + + public double CV_dResultAngle4_dDiffX4 { get; internal set; } + + public double CV_dResultAngle4_dDiffY4 { get; internal set; } + } + } +} diff --git a/src/RMAD/ElementwiseVectorMiniDecompositionOperation.cs b/src/RMAD/ElementwiseVectorMiniDecompositionOperation.cs new file mode 100644 index 00000000..24cd402a --- /dev/null +++ b/src/RMAD/ElementwiseVectorMiniDecompositionOperation.cs @@ -0,0 +1,394 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) 2023 ameritusweb All rights reserved. +// +//------------------------------------------------------------------------------ +namespace ParallelReverseAutoDiff.RMAD +{ + using System; + using System.Threading.Tasks; + + /// + /// Element-wise vector mini decomposition operation. + /// + public class ElementwiseVectorMiniDecompositionOperation : Operation + { + private Matrix input1; + private Matrix input2; + private Matrix weights; + private CalculatedValues[,] calculatedValues; + + /// + /// A common method for instantiating an operation. + /// + /// The neural network. + /// The instantiated operation. + public static IOperation Instantiate(NeuralNetwork net) + { + return new ElementwiseVectorMiniDecompositionOperation(); + } + + /// + /// Performs the forward operation for the element-wise vector mini decomposition function. + /// + /// The first input to the element-wise vector mini decomposition operation. + /// The second input to the element-wise vector mini decomposition operation. + /// The weights input to the element-wise vector mini decomposition operation. + /// The output of the element-wise vector mini decomposition operation. + public Matrix Forward(Matrix input1, Matrix input2, Matrix weights) + { + this.input1 = input1; + this.input2 = input2; + this.weights = weights; + + this.Output = new Matrix(this.input1.Rows, this.input1.Cols * 6); + + this.calculatedValues = new CalculatedValues[this.input1.Rows, this.input1.Cols / 2]; + + Parallel.For(0, input1.Rows, i => + { + for (int j = 0; j < input1.Cols / 2; j++) + { + // Accessing the magnitudes and angles from the concatenated matrices + double magnitude = input1[i, j]; + double angle = input1[i, j + (input1.Cols / 2)]; + + double wMagnitudePivot = input2[i, j * 3]; + double wAnglePivot = input2[i, (j * 3) + (input2.Cols / 2)]; + + double wMagnitude1 = input2[i, (j * 3) + 1]; + double wAngle1 = input2[i, (j * 3) + 1 + (input2.Cols / 2)]; + + double wMagnitude2 = input2[i, (j * 3) + 2]; + double wAngle2 = input2[i, (j * 3) + 2 + (input2.Cols / 2)]; + + // Compute vector components + double x = magnitude * Math.Cos(angle); + double y = magnitude * Math.Sin(angle); + double xPivot = wMagnitudePivot * Math.Cos(wAnglePivot); + double yPivot = wMagnitudePivot * Math.Sin(wAnglePivot); + + double dx_dMagnitude = Math.Cos(angle); + double dx_dAngle = -magnitude * Math.Sin(angle); + double dy_dMagnitude = Math.Sin(angle); + double dy_dAngle = magnitude * Math.Cos(angle); + double dXPivot_dWMagnitudePivot = Math.Cos(wAnglePivot); + double dXPivot_dWAnglePivot = -wMagnitudePivot * Math.Sin(wAnglePivot); + double dYPivot_dWMagnitudePivot = Math.Sin(wAnglePivot); + double dYPivot_dWAnglePivot = wMagnitudePivot * Math.Cos(wAnglePivot); + + this.calculatedValues[i, j] = new CalculatedValues() + { + CV_dx_dMagnitude = dx_dMagnitude, + CV_dx_dAngle = dx_dAngle, + CV_dy_dMagnitude = dy_dMagnitude, + CV_dy_dAngle = dy_dAngle, + CV_dXPivot_dWMagnitudePivot = dXPivot_dWMagnitudePivot, + CV_dXPivot_dWAnglePivot = dXPivot_dWAnglePivot, + CV_dYPivot_dWMagnitudePivot = dYPivot_dWMagnitudePivot, + CV_dYPivot_dWAnglePivot = dYPivot_dWAnglePivot, + }; + + double x1 = wMagnitude1 * Math.Cos(wAngle1); + double y1 = wMagnitude1 * Math.Sin(wAngle1); + + double dX1_wMagnitude1 = Math.Cos(wAngle1); + double dX1_wAngle1 = -wMagnitude1 * Math.Sin(wAngle1); + double dY1_wMagnitude1 = Math.Sin(wAngle1); + double dY1_wAngle1 = wMagnitude1 * Math.Cos(wAngle1); + + this.calculatedValues[i, j].CV_dX1_wMagnitude1 = dX1_wMagnitude1; + this.calculatedValues[i, j].CV_dX1_wAngle1 = dX1_wAngle1; + this.calculatedValues[i, j].CV_dY1_wMagnitude1 = dY1_wMagnitude1; + this.calculatedValues[i, j].CV_dY1_wAngle1 = dY1_wAngle1; + + double x2 = wMagnitude2 * Math.Cos(wAngle2); + double y2 = wMagnitude2 * Math.Sin(wAngle2); + + double dX2_wMagnitude2 = Math.Cos(wAngle2); + double dX2_wAngle2 = -wMagnitude2 * Math.Sin(wAngle2); + double dY2_wMagnitude2 = Math.Sin(wAngle2); + double dY2_wAngle2 = wMagnitude2 * Math.Cos(wAngle2); + + this.calculatedValues[i, j].CV_dX2_wMagnitude2 = dX2_wMagnitude2; + this.calculatedValues[i, j].CV_dX2_wAngle2 = dX2_wAngle2; + this.calculatedValues[i, j].CV_dY2_wMagnitude2 = dY2_wMagnitude2; + this.calculatedValues[i, j].CV_dY2_wAngle2 = dY2_wAngle2; + + double sumx = (x + xPivot) / (this.weights[i, j] + 1E-9); + double sumy = (y + yPivot) / (this.weights[i, j] + 1E-9); + + double dsumx_dX = 1d / (this.weights[i, j] + 1E-9); + double dsumx_dXPivot = 1d / (this.weights[i, j] + 1E-9); + double dsumx_dWeight = -(x + xPivot) / ((this.weights[i, j] + 1E-9) * (this.weights[i, j] + 1E-9)); + double dsumy_dY = 1d / (this.weights[i, j] + 1E-9); + double dsumy_dYPivot = 1d / (this.weights[i, j] + 1E-9); + double dsumy_dWeight = -(y + yPivot) / ((this.weights[i, j] + 1E-9) * (this.weights[i, j] + 1E-9)); + + this.calculatedValues[i, j].CV_dsumx_dX = dsumx_dX; + this.calculatedValues[i, j].CV_dsumx_dXPivot = dsumx_dXPivot; + this.calculatedValues[i, j].CV_dsumx_dWeight = dsumx_dWeight; + this.calculatedValues[i, j].CV_dsumy_dY = dsumy_dY; + this.calculatedValues[i, j].CV_dsumy_dYPivot = dsumy_dYPivot; + this.calculatedValues[i, j].CV_dsumy_dWeight = dsumy_dWeight; + + double diffx1 = sumx - x1; + double diffy1 = sumy - y1; + + double dDiffX1_dSumX = 1d; + double dDiffX1_dX1 = -1d; + double dDiffY1_dSumY = 1d; + double dDiffY1_dY1 = -1d; + + this.calculatedValues[i, j].CV_dDiffX1_dSumX = dDiffX1_dSumX; + this.calculatedValues[i, j].CV_dDiffX1_dX1 = dDiffX1_dX1; + this.calculatedValues[i, j].CV_dDiffY1_dSumY = dDiffY1_dSumY; + this.calculatedValues[i, j].CV_dDiffY1_dY1 = dDiffY1_dY1; + + double diffx2 = -sumx - x2; + double diffy2 = -sumy - y2; + + double dDiffX2_dSumX = -1d; + double dDiffX2_dX2 = -1d; + double dDiffY2_dSumY = -1d; + double dDiffY2_dY2 = -1d; + + this.calculatedValues[i, j].CV_dDiffX2_dSumX = dDiffX2_dSumX; + this.calculatedValues[i, j].CV_dDiffX2_dX2 = dDiffX2_dX2; + this.calculatedValues[i, j].CV_dDiffY2_dSumY = dDiffY2_dSumY; + this.calculatedValues[i, j].CV_dDiffY2_dY2 = dDiffY2_dY2; + + // Compute resultant vector magnitude and angle + double resultMagnitude1 = Math.Sqrt((diffx1 * diffx1) + (diffy1 * diffy1)); + double resultAngle1 = Math.Atan2(diffy1, diffx1); + + double dResultMagnitude1_dDiffX1 = diffx1 / resultMagnitude1; + double dResultMagnitude1_dDiffY1 = diffy1 / resultMagnitude1; + double dResultAngle1_dDiffX1 = -diffy1 / ((diffx1 * diffx1) + (diffy1 * diffy1)); + double dResultAngle1_dDiffY1 = diffx1 / ((diffx1 * diffx1) + (diffy1 * diffy1)); + + this.calculatedValues[i, j].CV_dResultMagnitude1_dDiffX1 = dResultMagnitude1_dDiffX1; + this.calculatedValues[i, j].CV_dResultMagnitude1_dDiffY1 = dResultMagnitude1_dDiffY1; + this.calculatedValues[i, j].CV_dResultAngle1_dDiffX1 = dResultAngle1_dDiffX1; + this.calculatedValues[i, j].CV_dResultAngle1_dDiffY1 = dResultAngle1_dDiffY1; + + double resultMagnitude2 = Math.Sqrt((diffx2 * diffx2) + (diffy2 * diffy2)); + double resultAngle2 = Math.Atan2(diffy2, diffx2); + + double dResultMagnitude2_dDiffX2 = diffx2 / resultMagnitude2; + double dResultMagnitude2_dDiffY2 = diffy2 / resultMagnitude2; + double dResultAngle2_dDiffX2 = -diffy2 / ((diffx2 * diffx2) + (diffy2 * diffy2)); + double dResultAngle2_dDiffY2 = diffx2 / ((diffx2 * diffx2) + (diffy2 * diffy2)); + + this.calculatedValues[i, j].CV_dResultMagnitude2_dDiffX2 = dResultMagnitude2_dDiffX2; + this.calculatedValues[i, j].CV_dResultMagnitude2_dDiffY2 = dResultMagnitude2_dDiffY2; + this.calculatedValues[i, j].CV_dResultAngle2_dDiffX2 = dResultAngle2_dDiffX2; + this.calculatedValues[i, j].CV_dResultAngle2_dDiffY2 = dResultAngle2_dDiffY2; + + this.Output[i, j * 6] = magnitude; + this.Output[i, (j * 6) + (this.input1.Cols * 6 / 2)] = angle; + + this.Output[i, (j * 6) + 1] = wMagnitudePivot; + this.Output[i, (j * 6) + 1 + (this.input1.Cols * 6 / 2)] = wAnglePivot; + + this.Output[i, (j * 6) + 2] = wMagnitude1; + this.Output[i, (j * 6) + 2 + (this.input1.Cols * 6 / 2)] = wAngle1; + + this.Output[i, (j * 6) + 3] = wMagnitude2; + this.Output[i, (j * 6) + 3 + (this.input1.Cols * 6 / 2)] = wAngle2; + + this.Output[i, (j * 6) + 4] = resultMagnitude1; + this.Output[i, (j * 6) + 4 + (this.input1.Cols * 6 / 2)] = resultAngle1; + + this.Output[i, (j * 6) + 5] = resultMagnitude2; + this.Output[i, (j * 6) + 5 + (this.input1.Cols * 6 / 2)] = resultAngle2; + } + }); + + return this.Output; + } + + /// + public override BackwardResult Backward(Matrix dOutput) + { + Matrix dInput1 = new Matrix(this.input1.Rows, this.input1.Cols); + Matrix dInput2 = new Matrix(this.input2.Rows, this.input2.Cols); + Matrix dWeights = new Matrix(this.weights.Rows, this.weights.Cols); + + Parallel.For(0, this.input1.Rows, i => + { + for (int j = 0; j < this.input1.Cols / 2; j++) + { + var grad = this.calculatedValues[i, j]; + + dInput1[i, j] += dOutput[i, j * 6]; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 6) + (this.input1.Cols * 6 / 2)]; + + dInput1[i, j] += dOutput[i, (j * 6) + 4] * grad.CV_dResultMagnitude1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 6) + 4] * grad.CV_dResultMagnitude1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 6) + 4 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 6) + 4 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dMagnitude; + + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 6) + 4] * grad.CV_dResultAngle1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 6) + 4 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 6) + 4] * grad.CV_dResultMagnitude1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 6) + 4 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultMagnitude1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dAngle; + + dInput1[i, j] += dOutput[i, (j * 6) + 5] * grad.CV_dResultMagnitude2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 6) + 5] * grad.CV_dResultMagnitude2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 6) + 5 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dMagnitude; + dInput1[i, j] += dOutput[i, (j * 6) + 5 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dMagnitude; + + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 6) + 5] * grad.CV_dResultAngle2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 6) + 5 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 6) + 5] * grad.CV_dResultMagnitude2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dX * grad.CV_dx_dAngle; + dInput1[i, j + (this.input1.Cols / 2)] += dOutput[i, (j * 6) + 5 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultMagnitude2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dY * grad.CV_dy_dAngle; + + dInput2[i, j * 3] += dOutput[i, (j * 6) + 1]; + dInput2[i, (j * 3) + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 1 + (this.input1.Cols * 6 / 2)]; + + dInput2[i, (j * 3) + 1] += dOutput[i, (j * 6) + 2]; + dInput2[i, (j * 3) + 1 + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 2 + (this.input1.Cols * 6 / 2)]; + + dInput2[i, (j * 3) + 2] += dOutput[i, (j * 6) + 3]; + dInput2[i, (j * 3) + 2 + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 3 + (this.input1.Cols * 6 / 2)]; + + dInput2[i, j * 3] += dOutput[i, (j * 6) + 4] * grad.CV_dResultMagnitude1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWMagnitudePivot; + dInput2[i, j * 3] += dOutput[i, (j * 6) + 4] * grad.CV_dResultMagnitude1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWMagnitudePivot; + dInput2[i, j * 3] += dOutput[i, (j * 6) + 4 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWMagnitudePivot; + dInput2[i, j * 3] += dOutput[i, (j * 6) + 4 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWMagnitudePivot; + + dInput2[i, (j * 3) + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 4] * grad.CV_dResultMagnitude1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWAnglePivot; + dInput2[i, (j * 3) + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 4 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWAnglePivot; + dInput2[i, (j * 3) + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 4] * grad.CV_dResultMagnitude1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWAnglePivot; + dInput2[i, (j * 3) + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 4 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWAnglePivot; + + dInput2[i, j * 3] += dOutput[i, (j * 6) + 5] * grad.CV_dResultMagnitude2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWMagnitudePivot; + dInput2[i, j * 3] += dOutput[i, (j * 6) + 5] * grad.CV_dResultMagnitude2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWMagnitudePivot; + dInput2[i, j * 3] += dOutput[i, (j * 6) + 5 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWMagnitudePivot; + dInput2[i, j * 3] += dOutput[i, (j * 6) + 5 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWMagnitudePivot; + + dInput2[i, (j * 3) + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 5] * grad.CV_dResultMagnitude2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWAnglePivot; + dInput2[i, (j * 3) + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 5 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dXPivot * grad.CV_dXPivot_dWAnglePivot; + dInput2[i, (j * 3) + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 5] * grad.CV_dResultMagnitude2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWAnglePivot; + dInput2[i, (j * 3) + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 5 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dYPivot * grad.CV_dYPivot_dWAnglePivot; + + dInput2[i, (j * 3) + 1] += dOutput[i, (j * 6) + 4] * grad.CV_dResultMagnitude1_dDiffX1 * grad.CV_dDiffX1_dX1 * grad.CV_dX1_wMagnitude1; + dInput2[i, (j * 3) + 1] += dOutput[i, (j * 6) + 4] * grad.CV_dResultMagnitude1_dDiffY1 * grad.CV_dDiffY1_dY1 * grad.CV_dY1_wMagnitude1; + dInput2[i, (j * 3) + 1] += dOutput[i, (j * 6) + 4 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle1_dDiffX1 * grad.CV_dDiffX1_dX1 * grad.CV_dX1_wMagnitude1; + dInput2[i, (j * 3) + 1] += dOutput[i, (j * 6) + 4 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle1_dDiffY1 * grad.CV_dDiffY1_dY1 * grad.CV_dY1_wMagnitude1; + + dInput2[i, (j * 3) + 1 + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 4] * grad.CV_dResultMagnitude1_dDiffX1 * grad.CV_dDiffX1_dX1 * grad.CV_dX1_wAngle1; + dInput2[i, (j * 3) + 1 + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 4 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle1_dDiffX1 * grad.CV_dDiffX1_dX1 * grad.CV_dX1_wAngle1; + dInput2[i, (j * 3) + 1 + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 4] * grad.CV_dResultMagnitude1_dDiffY1 * grad.CV_dDiffY1_dY1 * grad.CV_dY1_wAngle1; + dInput2[i, (j * 3) + 1 + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 4 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle1_dDiffY1 * grad.CV_dDiffY1_dY1 * grad.CV_dY1_wAngle1; + + dInput2[i, (j * 3) + 2] += dOutput[i, (j * 6) + 5] * grad.CV_dResultMagnitude2_dDiffX2 * grad.CV_dDiffX2_dX2 * grad.CV_dX2_wMagnitude2; + dInput2[i, (j * 3) + 2] += dOutput[i, (j * 6) + 5] * grad.CV_dResultMagnitude2_dDiffY2 * grad.CV_dDiffY2_dY2 * grad.CV_dY2_wMagnitude2; + dInput2[i, (j * 3) + 2] += dOutput[i, (j * 6) + 5 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle2_dDiffX2 * grad.CV_dDiffX2_dX2 * grad.CV_dX2_wMagnitude2; + dInput2[i, (j * 3) + 2] += dOutput[i, (j * 6) + 5 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle2_dDiffY2 * grad.CV_dDiffY2_dY2 * grad.CV_dY2_wMagnitude2; + + dInput2[i, (j * 3) + 2 + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 5] * grad.CV_dResultMagnitude2_dDiffX2 * grad.CV_dDiffX2_dX2 * grad.CV_dX2_wAngle2; + dInput2[i, (j * 3) + 2 + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 5 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle2_dDiffX2 * grad.CV_dDiffX2_dX2 * grad.CV_dX2_wAngle2; + dInput2[i, (j * 3) + 2 + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 5] * grad.CV_dResultMagnitude2_dDiffY2 * grad.CV_dDiffY2_dY2 * grad.CV_dY2_wAngle2; + dInput2[i, (j * 3) + 2 + (this.input2.Cols / 2)] += dOutput[i, (j * 6) + 5 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle2_dDiffY2 * grad.CV_dDiffY2_dY2 * grad.CV_dY2_wAngle2; + + dWeights[i, j] += dOutput[i, (j * 6) + 4] * grad.CV_dResultMagnitude1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dWeight; + dWeights[i, j] += dOutput[i, (j * 6) + 4] * grad.CV_dResultMagnitude1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dWeight; + dWeights[i, j] += dOutput[i, (j * 6) + 4 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle1_dDiffX1 * grad.CV_dDiffX1_dSumX * grad.CV_dsumx_dWeight; + dWeights[i, j] += dOutput[i, (j * 6) + 4 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle1_dDiffY1 * grad.CV_dDiffY1_dSumY * grad.CV_dsumy_dWeight; + + dWeights[i, j] += dOutput[i, (j * 6) + 5] * grad.CV_dResultMagnitude2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dWeight; + dWeights[i, j] += dOutput[i, (j * 6) + 5] * grad.CV_dResultMagnitude2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dWeight; + dWeights[i, j] += dOutput[i, (j * 6) + 5 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle2_dDiffX2 * grad.CV_dDiffX2_dSumX * grad.CV_dsumx_dWeight; + dWeights[i, j] += dOutput[i, (j * 6) + 5 + (this.input1.Cols * 6 / 2)] * grad.CV_dResultAngle2_dDiffY2 * grad.CV_dDiffY2_dSumY * grad.CV_dsumy_dWeight; + } + }); + + return new BackwardResultBuilder() + .AddInputGradient(dInput1) + .AddInputGradient(dInput2) + .AddInputGradient(dWeights) + .Build(); + } + + private struct CalculatedValues + { + public double CV_dx_dMagnitude { get; internal set; } + + public double CV_dx_dAngle { get; internal set; } + + public double CV_dy_dMagnitude { get; internal set; } + + public double CV_dy_dAngle { get; internal set; } + + public double CV_dXPivot_dWMagnitudePivot { get; internal set; } + + public double CV_dXPivot_dWAnglePivot { get; internal set; } + + public double CV_dYPivot_dWMagnitudePivot { get; internal set; } + + public double CV_dYPivot_dWAnglePivot { get; internal set; } + + public double CV_dX1_wMagnitude1 { get; internal set; } + + public double CV_dX1_wAngle1 { get; internal set; } + + public double CV_dY1_wMagnitude1 { get; internal set; } + + public double CV_dY1_wAngle1 { get; internal set; } + + public double CV_dX2_wMagnitude2 { get; internal set; } + + public double CV_dX2_wAngle2 { get; internal set; } + + public double CV_dY2_wMagnitude2 { get; internal set; } + + public double CV_dY2_wAngle2 { get; internal set; } + + public double CV_dsumx_dX { get; internal set; } + + public double CV_dsumx_dXPivot { get; internal set; } + + public double CV_dsumx_dWeight { get; internal set; } + + public double CV_dsumy_dY { get; internal set; } + + public double CV_dsumy_dYPivot { get; internal set; } + + public double CV_dsumy_dWeight { get; internal set; } + + public double CV_dDiffX1_dSumX { get; internal set; } + + public double CV_dDiffX1_dX1 { get; internal set; } + + public double CV_dDiffY1_dSumY { get; internal set; } + + public double CV_dDiffY1_dY1 { get; internal set; } + + public double CV_dDiffX2_dSumX { get; internal set; } + + public double CV_dDiffX2_dX2 { get; internal set; } + + public double CV_dDiffY2_dSumY { get; internal set; } + + public double CV_dDiffY2_dY2 { get; internal set; } + + public double CV_dResultMagnitude1_dDiffX1 { get; internal set; } + + public double CV_dResultMagnitude1_dDiffY1 { get; internal set; } + + public double CV_dResultAngle1_dDiffX1 { get; internal set; } + + public double CV_dResultAngle1_dDiffY1 { get; internal set; } + + public double CV_dResultMagnitude2_dDiffX2 { get; internal set; } + + public double CV_dResultMagnitude2_dDiffY2 { get; internal set; } + + public double CV_dResultAngle2_dDiffX2 { get; internal set; } + + public double CV_dResultAngle2_dDiffY2 { get; internal set; } + } + } +} diff --git a/src/RMAD/PairwiseSineSoftmaxOperation.cs b/src/RMAD/PairwiseSineSoftmaxOperation.cs new file mode 100644 index 00000000..c18bbabf --- /dev/null +++ b/src/RMAD/PairwiseSineSoftmaxOperation.cs @@ -0,0 +1,117 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) 2023 ameritusweb All rights reserved. +// +//------------------------------------------------------------------------------ +namespace ParallelReverseAutoDiff.RMAD +{ + using System; + + /// + /// Sine Softmax operation. + /// + public class PairwiseSineSoftmaxOperation : Operation + { + private Matrix input; + + /// + /// A common method for instantiating an operation. + /// + /// The neural network. + /// The instantiated operation. + public static IOperation Instantiate(NeuralNetwork net) + { + return new PairwiseSineSoftmaxOperation(); + } + + /// + public override void Store(Guid id) + { + this.IntermediateMatrixArrays.AddOrUpdate(id, new[] { this.input, this.Output }, (x, y) => new[] { this.input, this.Output }); + } + + /// + public override void Restore(Guid id) + { + var restored = this.IntermediateMatrixArrays[id]; + this.input = restored[0]; + this.Output = restored[1]; + } + + /// + /// Performs the forward operation for the softmax function. + /// + /// The input to the softmax operation. + /// The output of the softmax operation. + public Matrix Forward(Matrix input) + { + this.input = input; + this.Output = this.PairedSineSoftmax(input); + return this.Output; + } + + /// + public override BackwardResult Backward(Matrix dLdOutput) + { + int numRows = this.Output.Length; + int numCols = this.Output[0].Length; + int m = numCols / 2; + + Matrix dLdInput = new Matrix(numRows, numCols); + for (int i = 0; i < numRows; i++) + { + for (int j = 0; j < m; j++) + { + double a = this.input[i, j]; + double b = this.input[i, j + m]; + double expSinA = Math.Exp(Math.Sin(a)); + double expSinB = Math.Exp(Math.Sin(b)); + + double f1 = expSinA; + double fPrime1 = Math.Cos(a) * expSinA; + double g1 = expSinA + expSinB; + double gPrime1 = Math.Cos(a) * expSinA; + double dSinSoftmaxj1 = ((fPrime1 * g1) - (f1 * gPrime1)) / Math.Pow(g1, 2d); + + double f2 = expSinB; + double fPrime2 = Math.Cos(b) * expSinB; + double g2 = expSinA + expSinB; + double gPrime2 = Math.Cos(b) * expSinB; + double dSinSoftmaxj2 = ((fPrime2 * g2) - (f2 * gPrime2)) / Math.Pow(g2, 2d); + + dLdInput[i][j] = dLdOutput[i][j] * dSinSoftmaxj1; + dLdInput[i][j + m] = dLdOutput[i][j + m] * dSinSoftmaxj2; + } + } + + return new BackwardResultBuilder() + .AddInputGradient(dLdInput) + .Build(); + } + + private Matrix PairedSineSoftmax(Matrix input) + { + int numRows = input.Rows; + int numCols = input.Cols; + int m = numCols / 2; + + Matrix output = new Matrix(numRows, numCols); + + for (int i = 0; i < numRows; i++) + { + for (int j = 0; j < m; j++) + { + double a = input[i, j]; + double b = input[i, j + m]; + double sumExp = Math.Exp(Math.Sin(a)) + Math.Exp(Math.Sin(b)); + double numerator1 = Math.Exp(Math.Sin(a)); + output[i, j] = numerator1 / sumExp; + double numerator2 = Math.Exp(Math.Sin(b)); + output[i, j + m] = numerator2 / sumExp; + } + } + + return output; + } + } +} diff --git a/src/RMAD/VectorAttentionBinaryOperation.cs b/src/RMAD/VectorAttentionBinaryOperation.cs new file mode 100644 index 00000000..5061ee15 --- /dev/null +++ b/src/RMAD/VectorAttentionBinaryOperation.cs @@ -0,0 +1,110 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) 2023 ameritusweb All rights reserved. +// +//------------------------------------------------------------------------------ +namespace ParallelReverseAutoDiff.RMAD +{ + using System; + using System.Threading.Tasks; + + /// + /// Vector attention operation. + /// + public class VectorAttentionBinaryOperation : Operation + { + private Matrix vectors; + private Matrix probabilities; + + /// + /// A common method for instantiating an operation. + /// + /// The neural network. + /// The instantiated operation. + public static IOperation Instantiate(NeuralNetwork net) + { + return new VectorAttentionBinaryOperation(); + } + + /// + /// Performs the forward operation for the vector attention function. + /// + /// The first input to the vector attention operation. + /// The second input to the vector attention operation. + /// The output of the vector attention operation. + public Matrix Forward(Matrix vectors, Matrix probabilities) + { + this.vectors = vectors; + this.probabilities = probabilities; + + this.Output = new Matrix(vectors.Rows, vectors.Cols); + + int m = vectors.Cols / 2; + + Parallel.For(0, vectors.Rows, i => + { + for (int j = 0; j < m; j++) + { + // Calculate the scaling factor for the magnitude + double prob = probabilities[i, j]; + double magnitudeScale = 1.5 - prob; // Ranges from 1 (at prob = 0.5) to 2 (at prob = 0) to 0.5 (at prob = 1) + double magnitude = vectors[i, j] * magnitudeScale; + + // Adjust the angle based on the probability + double angle = vectors[i, j + m]; + double angleAdjustment = Math.PI * (1 - prob); // Ranges from 0 (at prob = 1) to π (at prob = 0) + angle = (angle + angleAdjustment) % (2 * Math.PI); + + this.Output[i, j] = magnitude; + this.Output[i, j + m] = angle; + } + }); + + return this.Output; + } + + /// + public override BackwardResult Backward(Matrix dOutput) + { + Matrix dVectors = new Matrix(this.vectors.Rows, this.vectors.Cols); + Matrix dProbabilities = new Matrix(this.probabilities.Rows, this.probabilities.Cols); + + int m = this.vectors.Cols / 2; + Parallel.For(0, this.vectors.Rows, i => + { + for (int j = 0; j < m; j++) + { + double prop1 = this.probabilities[i, j]; + + // Gradient for magnitude + dVectors[i, j] = dOutput[i, j] * (1.5d - prop1); // Direct gradient flow for magnitude + + // Gradient for angle + double dAngle = dOutput[i, j + m]; + dVectors[i, j + m] = dAngle; // Direct gradient flow for angle + + double dAngle_dProb1 = -Math.PI; + double dAngle_dProb2 = Math.PI; + + // Gradient for Prob1 (affects magnitude and angle) + dProbabilities[i, j] = dAngle * dAngle_dProb1; // From derivative dAngle/dProb1 = -π + + // Gradient for Prob2 (affects only angle) + dProbabilities[i, j + m] = dAngle * dAngle_dProb2; // From derivative dAngle/dProb2 = π + + double dMagnitude = dOutput[i, j]; + double originalMagnitude = this.vectors[i, j]; + + // Compute gradients for probabilities related to magnitude + dProbabilities[i, j] += -dMagnitude * originalMagnitude; // dMagnitude/dProb1 + dProbabilities[i, j + m] += dMagnitude * originalMagnitude; // dMagnitude/dProb2 + } + }); + + return new BackwardResultBuilder() + .AddInputGradient(dVectors) + .AddInputGradient(dProbabilities) + .Build(); + } + } +} diff --git a/src/RMAD/VectorAttentionOperation.cs b/src/RMAD/VectorAttentionOperation.cs new file mode 100644 index 00000000..b671e942 --- /dev/null +++ b/src/RMAD/VectorAttentionOperation.cs @@ -0,0 +1,99 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) 2023 ameritusweb All rights reserved. +// +//------------------------------------------------------------------------------ +namespace ParallelReverseAutoDiff.RMAD +{ + using System; + using System.Threading.Tasks; + + /// + /// Vector attention operation. + /// + public class VectorAttentionOperation : Operation + { + private Matrix vectors; + private Matrix probabilities; + + /// + /// A common method for instantiating an operation. + /// + /// The neural network. + /// The instantiated operation. + public static IOperation Instantiate(NeuralNetwork net) + { + return new VectorAttentionOperation(); + } + + /// + /// Performs the forward operation for the vector attention function. + /// + /// The first input to the vector attention operation. + /// The second input to the vector attention operation. + /// The output of the vector attention operation. + public Matrix Forward(Matrix vectors, Matrix probabilities) + { + this.vectors = vectors; + this.probabilities = probabilities; + + this.Output = new Matrix(vectors.Rows, vectors.Cols); + + int m = vectors.Cols / 2; + + Parallel.For(0, vectors.Rows, i => + { + for (int j = 0; j < m; j++) + { + // Calculate the scaling factor for the magnitude + double prob = probabilities[i, j]; + double magnitudeScale = 1.5 - prob; // Ranges from 1 (at prob = 0.5) to 2 (at prob = 0) to 0.5 (at prob = 1) + double magnitude = vectors[i, j] * magnitudeScale; + + // Adjust the angle based on the probability + double angle = vectors[i, j + m]; + double angleAdjustment = 1.5d * Math.PI * (1 - prob); // Ranges from 0 (at prob = 1) to 2π (at prob = 0) + angle = (angle + angleAdjustment) % (2 * Math.PI); + + this.Output[i, j] = magnitude; + this.Output[i, j + m] = angle; + } + }); + + return this.Output; + } + + /// + public override BackwardResult Backward(Matrix dOutput) + { + Matrix dVectors = new Matrix(this.vectors.Rows, this.vectors.Cols); + Matrix dProbabilities = new Matrix(this.probabilities.Rows, this.probabilities.Cols); + + int m = this.vectors.Cols / 2; + Parallel.For(0, this.vectors.Rows, i => + { + for (int j = 0; j < m; j++) + { + double prob = this.probabilities[i, j]; + double dMagnitude_dProb = -dOutput[i, j] * this.vectors[i, j]; // Derivative of magnitude w.r.t probability + double dMagnitude_dProb2 = dOutput[i, j] * this.vectors[i, j]; // Derivative of magnitude w.r.t probability 2 + double dAngle_dProb = -1.5d * Math.PI * dOutput[i, j + m]; // Derivative of angle w.r.t probability + double dAngle_dProb2 = 1.5d * Math.PI * dOutput[i, j + m]; // Derivative of angle w.r.t probability 2 + + // Update gradients for vectors + dVectors[i, j] = dOutput[i, j] * (1.5 - prob); // Corrected gradient flow for magnitude + dVectors[i, j + m] = dOutput[i, j + m]; // Direct gradient flow for angle is correct + + // Aggregate gradients for probabilities + dProbabilities[i, j] = dMagnitude_dProb + dAngle_dProb; // Correct aggregation of gradients + dProbabilities[i, j + m] = dMagnitude_dProb2 + dAngle_dProb2; // Correct aggregation of gradients 2 + } + }); + + return new BackwardResultBuilder() + .AddInputGradient(dVectors) + .AddInputGradient(dProbabilities) + .Build(); + } + } +} diff --git a/src/RMAD/VectorizeOperation.cs b/src/RMAD/VectorizeOperation.cs new file mode 100644 index 00000000..2c73210c --- /dev/null +++ b/src/RMAD/VectorizeOperation.cs @@ -0,0 +1,139 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) 2023 ameritusweb All rights reserved. +// +//------------------------------------------------------------------------------ +namespace ParallelReverseAutoDiff.RMAD +{ + using System; + using System.Collections.Generic; + + /// + /// Performs the forward and backward operations for the vectorize function. + /// + public class VectorizeOperation : Operation + { + private Matrix input; + private Matrix angles; + + /// + /// A common factory method for instantiating this operation. + /// + /// The neural network. + /// The instantiated operation. + public static IOperation Instantiate(NeuralNetwork net) + { + return new VectorizeOperation(); + } + + /// + public override void Store(Guid id) + { + this.IntermediateMatrices.AddOrUpdate(id, this.input, (x, y) => this.input); + } + + /// + public override void Restore(Guid id) + { + this.input = this.IntermediateMatrices[id]; + } + + /// + /// Performs the forward operation for the vectorize function. + /// + /// The input to the vectorize operation. + /// The angles to the vectorize operation. + /// The output of the vectorize operation. + public Matrix Forward(Matrix input, Matrix angles) + { + this.input = input; + this.angles = angles; + + int rows = input.Length; + int cols = input[0].Length; + + if (cols != angles[0].Length) + { + throw new ArgumentException("Input and angles matrices must have the same number of columns."); + } + + int m = cols * 2; + this.Output = new Matrix(rows, m); + + var angleMappings = new Dictionary<(double, double, double), double>(); + + for (int i = 0; i < rows; i++) + { + for (int j = 0; j < cols; j++) + { + // Left half: magnitudes from input + this.Output[i][j] = input[i][j]; + + // Right half: angles with context + double prev = j > 0 ? input[i][j - 1] : double.MinValue; + double next = j < cols - 1 ? input[i][j + 1] : double.MaxValue; + var context = (prev, input[i][j], next); + + if (!angleMappings.ContainsKey(context)) + { + angleMappings[context] = angles[i][j]; // Store new angle for the context + } + + this.Output[i][cols + j] = angleMappings[context]; // Use the angle for the output matrix + } + } + + return this.Output; + } + + /// + public override BackwardResult Backward(Matrix dLdOutput) + { + int rows = dLdOutput.Length; + int cols = this.input[0].Length; + Matrix dLdInput = new Matrix(rows, cols); // Gradient with respect to input magnitudes + Matrix dLdAngles = new Matrix(rows, cols); // Gradient with respect to input angles + + var angleUsage = new Dictionary>(); // Tracks angle usage + + // Track the usage of each angle in the forward pass + for (int i = 0; i < rows; i++) + { + for (int j = 0; j < cols; j++) + { + double angle = this.angles[i][j]; + if (!angleUsage.ContainsKey(angle)) + { + angleUsage[angle] = new List<(int, int)>(); + } + + angleUsage[angle].Add((i, j)); + } + } + + // Calculate the gradient for each angle based on its usage + foreach (var angle in angleUsage.Keys) + { + foreach (var (row, col) in angleUsage[angle]) + { + // Accumulate gradients from all outputs affected by this angle + dLdAngles[row][col] += dLdOutput[row][cols + col]; + } + } + + // Gradient for magnitudes is straightforward + for (int i = 0; i < rows; i++) + { + for (int j = 0; j < cols; j++) + { + dLdInput[i][j] = dLdOutput[i][j]; + } + } + + return new BackwardResultBuilder() + .AddInputGradient(dLdInput) + .AddInputGradient(dLdAngles) + .Build(); + } + } +} diff --git a/src/docs/README.md b/src/docs/README.md index 2ea869a1..e9c9db3f 100644 --- a/src/docs/README.md +++ b/src/docs/README.md @@ -127,6 +127,8 @@ MatrixConcatenateOperation MatrixDiagonalFilterOperation +MatrixHorizontalConcatenateOperation + MatrixMultiplyOperation MatrixMultiplyAndSumOperation @@ -197,6 +199,30 @@ DeepScaleAndShiftOperation FlattenOperation +### Vector Neural Network (VNN) Operations +These types of operations typically operate on instances of the Matrix class where the left half are magnitudes and the right half are angles in radians. +Learn more about Vector Neural Networks [here](https://www.amazon.com/Vector-Neural-Networks-Geometric-Tensors-ebook/dp/B0CXBV3DY5/ref=sr_1_1). + +ElementwiseSquareOperation + +ElementwiseVectorAddOperation + +ElementwiseVectorCartesianSummationOperation + +ElementwiseVectorConstituentMultiplyOperation + +ElementwiseVectorDecompositionOperation + +ElementwiseVectorMiniDecompositionOperation + +PairwiseSineSoftmaxOperation + +VectorAttentionBinaryOperation + +VectorAttentionOperation + +VectorizeOperation + ### Neural Network Parameters Each neural network base class has a set of parameters that can be used to configure the neural network. They are as follows: