Updated simple_neural_network.py

neural_network/simple_neural_network.py

@@ -1,63 +1,122 @@
-"""
-Forward propagation explanation:
-https://towardsdatascience.com/forward-propagation-in-neural-networks-simplified-math-and-code-version-bbcfef6f9250
-"""
-
-import math
-import random
-
-
-# Sigmoid
-def sigmoid_function(value: float, deriv: bool = False) -> float:
-    """Return the sigmoid function of a float.
-
-    >>> sigmoid_function(3.5)
-    0.9706877692486436
-    >>> sigmoid_function(3.5, True)
-    -8.75
-    """
-    if deriv:
-        return value * (1 - value)
-    return 1 / (1 + math.exp(-value))
-
-
-# Initial Value
-INITIAL_VALUE = 0.02
-
 
-def forward_propagation(expected: int, number_propagations: int) -> float:
-    """Return the value found after the forward propagation training.
-
-    >>> res = forward_propagation(32, 10000000)
-    >>> res > 31 and res < 33
-    True
-
-    >>> res = forward_propagation(32, 1000)
-    >>> res > 31 and res < 33
-    False
-    """
-
-    # Random weight
-    weight = float(2 * (random.randint(1, 100)) - 1)
-
-    for _ in range(number_propagations):
-        # Forward propagation
-        layer_1 = sigmoid_function(INITIAL_VALUE * weight)
-        # How much did we miss?
-        layer_1_error = (expected / 100) - layer_1
-        # Error delta
-        layer_1_delta = layer_1_error * sigmoid_function(layer_1, True)
-        # Update weight
-        weight += INITIAL_VALUE * layer_1_delta
-
-    return layer_1 * 100
-
-
-if __name__ == "__main__":
-    import doctest
+"""
+Simple Neural Network
 
-    doctest.testmod()
+https://machinelearningmastery.com/implement-backpropagation-algorithm-scratch-python/
 
-    expected = int(input("Expected value: "))
-    number_propagations = int(input("Number of propagations: "))
-    print(forward_propagation(expected, number_propagations))
+"""
+from random import seed
+from random import random
+from math import exp
+
+#Initializing Network
+def initialize_network(n_input,n_hidden,n_output):
+  network=list()
+  hidden_layer=[{'weights':[random() for i in range(n_input+1)]} for i in range(n_hidden)]
+  network.append(hidden_layer)
+  output_layer=[{'weights':[random() for i in range(n_hidden+1)]} for i in range(n_output)]
+  network.append(output_layer)
+  return network
+
+
+
+# Forward Propagate
+  # 1.Neuron Activation.
+  # 2.Neuron Transfer.
+  # 3.Forward Propagation.
+
+# Neuron activation is calculated as the weighted sum of the inputs
+def activate(weights,inputs):
+  activation=weights[-1]
+  for i in range(len(weights)-1):
+    activation+=weights[i]*inputs[i]
+  return activation
+def transfer(activation):
+  return 1.0/(1.0+exp(-activation))
+
+
+def forward_propogate(network,row):
+  inputs=row
+  for layer in network:
+    new_inputs=[]
+    for neuron in layer:
+      activation=activate(neuron['weights'],inputs)
+      neuron['output']=transfer(activation)
+      new_inputs.append(neuron['output'])
+    inputs=new_inputs
+
+  return inputs
+
+
+
+#Back Propagation
+  # 1.Transfer Derivative.
+  # 2.Error Backpropagation.
+def transfer_derivative(output):
+  return output*(1.0-output)
+
+
+def back_propogate_error(network,expected):
+  for i in reversed(range(len(network))):
+    layer=network[i]
+    errors=list()
+
+    if i != len(network)-1:
+      for j in range(len(layer)):
+        error=0.0
+        for neuron in network[i+1]:
+          error += (neuron['weights'][j]*neuron['delta'])
+        errors.append(error)
+    else:
+      for j in range(len(layer)):
+        neuron=layer[j]
+        errors.append(neuron['output']-expected[j])
+
+    for j in range(len(layer)):
+      neuron=layer[j]
+      neuron['delta']=errors[j]*transfer_derivative(neuron['output'])
+
+# Once errors are calculated for each neuron in the network via the back propagation method above,
+#  they can be used to update weights.
+def update_weights(network, row, l_rate):
+	for i in range(len(network)):
+		inputs = row[:-1]
+		if i != 0:
+			inputs = [neuron['output'] for neuron in network[i - 1]]
+		for neuron in network[i]:
+			for j in range(len(inputs)):
+				neuron['weights'][j] -= l_rate * neuron['delta'] * inputs[j]
+			neuron['weights'][-1] -= l_rate * neuron['delta']
+
+
+##Training
+
+def train_network(network, train, l_rate, n_epoch, n_outputs):
+	for epoch in range(n_epoch):
+		sum_error = 0
+		for row in train:
+			outputs = forward_propogate(network, row)
+			expected = [0 for i in range(n_outputs)]
+			expected[row[-1]] = 1
+			sum_error += sum([(expected[i]-outputs[i])**2 for i in range(len(expected))])
+			back_propogate_error(network, expected)
+			update_weights(network, row, l_rate)
+		print('>epoch=%d, lrate=%.3f, error=%.3f' % (epoch, l_rate, sum_error))
+
+seed(1)
+dataset = [[2.7810836,2.550537003,0],
+ [1.465489372,2.362125076,0],
+ [3.396561688,4.400293529,0],
+ [1.38807019,1.850220317,0],
+ [3.06407232,3.005305973,0],
+ [7.627531214,2.759262235,1],
+ [5.332441248,2.088626775,1],
+ [6.922596716,1.77106367,1],
+ [8.675418651,-0.242068655,1],
+ [7.673756466,3.508563011,1]]
+n_inputs = len(dataset[0]) - 1
+n_outputs = len(set([row[-1] for row in dataset]))
+network = initialize_network(n_inputs, 2, n_outputs)
+train_network(network, dataset, 0.7, 30, n_outputs)
+for layer in network:
+ print(layer)