Analysis_neural_networks.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Analysis_neural_networks.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.0b5"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "eQwFA1bQqIkb"
      },
      "source": [
        "# Project :Deeper understanding of how neural networks learn (I)\n",
        "## Objective : To go through how the matrix changes its shape and make its way through the layers to the final output layer and how updation of weights effects the overall loss and prediction power of the network !\n",
        "\n",
        "##### The codes have been highly influenced by the one given on  [Andrew Trask's blog](https://iamtrask.github.io/2015/07/12/basic-python-network/)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "UT_A3JWOqT_P"
      },
      "source": [
        "### packages import"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "joBriGROVcr-",
        "colab": {}
      },
      "source": [
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "dERwGmGsqZrz"
      },
      "source": [
        "### Function for training the network made of 3 input nodes and 1 output node"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "SF8lNlB1jd7-",
        "colab": {}
      },
      "source": [
        "# function for feedforward and backpropogation step \n",
        "#if shapes = True, that means we want to print the shapes as well otherwise no shape will be printed, only pass will be done.\n",
        "def feed_forward_backpropogation_train(n, X, y, weights_input_to_hidden, weights_hidden_to_output, shapes = True):\n",
        "    '''\n",
        "    n : number of iterations\n",
        "    X : Input data matrix \n",
        "    y : actual target (label)\n",
        "    weights_input_to_hidden : weights matrix from input to hidden layer\n",
        "    weights_hidden_to_output : weights matrix from hidden to output layer\n",
        "    shapes : if shapes required to be printed or not \n",
        "    '''\n",
        "    if shapes == True:\n",
        "        for j in range(n):\n",
        "            print(\"inside the loop....\")\n",
        "            h1 = 1/(1+np.exp(-(np.dot(X,weights_input_to_hidden))))\n",
        "            print(f\"first output shape : {X.shape} x {weights_input_to_hidden.shape} =  {h1.shape}\")\n",
        "            print(f\"shape of h1 : {h1.shape}\")\n",
        "            print(f\"shape of weights_hidden_to_output : {weights_hidden_to_output.shape}\")\n",
        "            output = 1/(1+np.exp(-(np.dot(h1,weights_hidden_to_output))))\n",
        "            print(f\"second outupt shape {h1.shape} x {weights_hidden_to_output.shape} = {output.shape}\")\n",
        "            print(f\"predicted :\")\n",
        "            print(output)\n",
        "            print()\n",
        "            output_delta = (y - output)*(output*(1-output))\n",
        "            h1_delta = output_delta.dot(weights_hidden_to_output.T) * (h1 * (1-h1))\n",
        "            weights_hidden_to_output += h1.T.dot(output_delta)\n",
        "            weights_input_to_hidden += X.T.dot(h1_delta)\n",
        "    else:\n",
        "        for j in range(n):\n",
        "            # feedforward pass\n",
        "            h1 = 1/(1+np.exp(-(np.dot(X,weights_input_to_hidden))))\n",
        "            output = 1/(1+np.exp(-(np.dot(h1,weights_hidden_to_output))))\n",
        "            # backpropogation of error \n",
        "            output_delta = (y - output)*(output*(1-output))\n",
        "            h1_delta = output_delta.dot(weights_hidden_to_output.T) * (h1 * (1-h1))\n",
        "            # updation of weights\n",
        "            weights_hidden_to_output += h1.T.dot(output_delta)\n",
        "            weights_input_to_hidden += X.T.dot(h1_delta)\n",
        "    return weights_input_to_hidden, weights_hidden_to_output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "VD1Lx8CmqwuS"
      },
      "source": [
        "### Let's understand the code one by one by actually seeing the shapes of matrix as they go from layer to layer and how they are changed in the way...."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nbU0TZYLRY1M",
        "colab_type": "text"
      },
      "source": [
        "Here we are using a neural network consisting of one hidden layer(containing one node) and one output layer (including input layer which we don't necessarily count as a layer)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "W18dO_zrcl8N",
        "outputId": "c25aa559-0ed3-4652-ea10-e1860c297b98",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 995
        }
      },
      "source": [
        "# Here we start our analysis by taking input examples and outputs.\n",
        "X = np.array([[0,0,1],[0,1,1],[1,0,1],[1,1,1]])\n",
        "print(\"X (training examples)\")\n",
        "print(X)\n",
        "print(f\"shape X: {X.shape}\")\n",
        "y = np.array([[0,1,1,0]]).T\n",
        "print(\"Y (actual target)\")\n",
        "print(y)\n",
        "print(f\"shape of Y : {y.shape}\")\n",
        "weights_input_to_hidden = 2*np.random.random((3,4)) - 1\n",
        "print(\"weights initialized from input to hidden:\")\n",
        "print(weights_input_to_hidden)\n",
        "print(f\"shape of weights_input_to_hidden: {weights_input_to_hidden.shape}\")\n",
        "weights_hidden_to_output = 2*np.random.random((4,1)) - 1\n",
        "print(\"weights iniatialized from hidden to output:\")\n",
        "print(weights_hidden_to_output)\n",
        "print(f\"shape of weights_hidden_to_output: {weights_hidden_to_output.shape}\")\n",
        "\n",
        "print(\"Let's play with one time pass and check the shapes of matrix data as they pass from input to output:\")\n",
        "weights_input_to_hidden, weights_hidden_to_output = feed_forward_backpropogation_train(1, X, y, weights_input_to_hidden, weights_hidden_to_output, True)\n",
        "\n",
        "\n",
        "print(\"new weights from input to hidden\")\n",
        "print(weights_input_to_hidden)\n",
        "print(\"new weights from hidden to output\")\n",
        "print(weights_hidden_to_output)\n",
        "\n",
        "print()\n",
        "print(\"Let's now iterate more and check how these weights get updated\")\n",
        "weights_input_to_hidden, weights_hidden_to_output = feed_forward_backpropogation_train(10000, X, y, weights_input_to_hidden, weights_hidden_to_output, False)\n",
        "\n",
        "\n",
        "print(\"new weights from input to hidden\")\n",
        "print(weights_input_to_hidden)\n",
        "print(\"new weights from hidden to output\")\n",
        "print(weights_hidden_to_output)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "X (training examples)\n",
            "[[0 0 1]\n",
            " [0 1 1]\n",
            " [1 0 1]\n",
            " [1 1 1]]\n",
            "shape X: (4, 3)\n",
            "Y (actual target)\n",
            "[[0]\n",
            " [1]\n",
            " [1]\n",
            " [0]]\n",
            "shape of Y : (4, 1)\n",
            "weights initialized from input to hidden:\n",
            "[[ 0.81782303  0.95969135  0.15655927 -0.16690352]\n",
            " [-0.00180481  0.73578928 -0.57032858  0.19066284]\n",
            " [ 0.76723382  0.14705598 -0.98890526 -0.56767396]]\n",
            "shape of weights_input_to_hidden: (3, 4)\n",
            "weights iniatialized from hidden to output:\n",
            "[[ 0.5093118 ]\n",
            " [-0.74293308]\n",
            " [-0.37334562]\n",
            " [-0.95113288]]\n",
            "shape of weights_hidden_to_output: (4, 1)\n",
            "Let's play with one time pass and check the shapes of matrix data as they pass from input to output:\n",
            "inside the loop....\n",
            "first output shape : (4, 3) x (3, 4) =  (4, 4)\n",
            "shape of h1 : (4, 4)\n",
            "shape of weights_hidden_to_output : (4, 1)\n",
            "second outupt shape (4, 4) x (4, 1) = (4, 1)\n",
            "predicted :\n",
            "[[0.3784271 ]\n",
            " [0.34756135]\n",
            " [0.36420226]\n",
            " [0.3448391 ]]\n",
            "\n",
            "new weights from input to hidden\n",
            "[[ 0.82279962  0.94609927  0.14955591 -0.18036304]\n",
            " [ 0.00891509  0.71987164 -0.57365041  0.17392407]\n",
            " [ 0.77872087  0.12715718 -0.99727118 -0.5955438 ]]\n",
            "new wegiths from hidden to output\n",
            "[[ 0.60704923]\n",
            " [-0.64265716]\n",
            " [-0.34252003]\n",
            " [-0.90402795]]\n",
            "\n",
            "Let's now iterate more and check how these weights get updated\n",
            "new weights from input to hidden\n",
            "[[ 7.06885096  2.14403306 -0.32311451  4.09532197]\n",
            " [ 6.49955721 -1.2906453   0.79312209  5.04087269]\n",
            " [-2.97294418  1.7316125  -0.19571147 -6.93288546]]\n",
            "new weights from hidden to output\n",
            "[[ 10.65037375]\n",
            " [ -4.96686006]\n",
            " [ -1.73433319]\n",
            " [-10.6052312 ]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "I4JZd6cDq_tu"
      },
      "source": [
        "### Function to make a single pass through the network with updated weights"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "lrB-UObkoGZY",
        "colab": {}
      },
      "source": [
        "# function to do a single pass run through the network to predict the output \n",
        "def run_network(X, weights_input_to_hidden, weights_hidden_to_output):\n",
        "    '''\n",
        "    X : input data matrix\n",
        "    weights_input_to_hidden : weights matrix from input to hidden layer\n",
        "    weights_hidden_to_output : weights matrix from hidden to output layer\n",
        "    '''\n",
        "    h1 = 1/(1+np.exp(-(np.dot(X,weights_input_to_hidden))))\n",
        "    output = 1/(1+np.exp(-(np.dot(h1,weights_hidden_to_output))))\n",
        "    return output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "cNrc21kJrGwE"
      },
      "source": [
        "### Defining error (here we take [mean squared error](https://en.wikipedia.org/wiki/Mean_squared_error))"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "13wfmGminkJn",
        "colab": {}
      },
      "source": [
        "# function to calculate the error \n",
        "def MSE(y, Y):\n",
        "    '''\n",
        "    y : actual label (matrix)\n",
        "    Y : predicted label (matrix)\n",
        "    '''\n",
        "    return np.mean((y - Y)**2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "iCMwCd0drNTA"
      },
      "source": [
        "### Here we actually see how increasing the training of network decreases overall loss but mind it this is training loss !\n",
        "### Actual testing loss is the one which is over unseen data !"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "CaomDCq3ljNV",
        "outputId": "ea3b5d0d-29d8-4bf6-8844-cfa492991d7d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "weights_input_to_hidden, weights_hidden_to_output = feed_forward_backpropogation_train(10000, X, y, weights_input_to_hidden, weights_hidden_to_output, False)\n",
        "error = MSE(y, run_network(X, weights_input_to_hidden, weights_hidden_to_output))\n",
        "print(error)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "4.0107463780298154e-06\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "dJMP2xWxpocz",
        "outputId": "76caefad-b02e-4160-fa6e-c3db6075226c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "weights_input_to_hidden, weights_hidden_to_output = feed_forward_backpropogation_train(20000, X, y, weights_input_to_hidden, weights_hidden_to_output, False)\n",
        "error = MSE(y, run_network(X, weights_input_to_hidden, weights_hidden_to_output))\n",
        "print(error)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "3.640432255734101e-06\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "cRsvd5ySpvad",
        "outputId": "06a9827e-b915-4339-9732-55ffe82d35ec",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "weights_input_to_hidden, weights_hidden_to_output = feed_forward_backpropogation_train(40000, X, y, weights_input_to_hidden, weights_hidden_to_output, False)\n",
        "error = MSE(y, run_network(X, weights_input_to_hidden, weights_hidden_to_output))\n",
        "print(error)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "3.067497883744055e-06\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "6JlRzLpkqB0Q",
        "outputId": "574d88ca-237d-41ec-ab42-2d47758301e4",
        "scrolled": false,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "weights_input_to_hidden, weights_hidden_to_output = feed_forward_backpropogation_train(100000, X, y, weights_input_to_hidden, weights_hidden_to_output, False)\n",
        "error = MSE(y, run_network(X, weights_input_to_hidden, weights_hidden_to_output))\n",
        "print(error)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.1877647038761036e-06\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rNi48KaZRY1n",
        "colab_type": "text"
      },
      "source": [
        "#### Feel free to explore more as everything is Open Sourced !\n"
      ]
    }
  ]
}