MNIST_MLP.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    },
    "colab": {
      "name": "MNIST_MLP.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mashyko/deep-learning/blob/master/MNIST_MLP.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YdZzWEJeXJnt",
        "colab_type": "text"
      },
      "source": [
        "Obtaining the MNIST dataset:\n",
        "http://yann.lecun.com/exdb/mnist/\n",
        "\n",
        "MNISTデータセットはY. LeCunのWebサイトで公開されています。MNISTデータセットは<br>\n",
        "トレーニングデータセットの画像：train-images-idx3-ubyte.gz <br>\n",
        "トレーニングデータセットのラベル：train-labels-idx1-ubyte.gz <br>\n",
        "テストデータセットの画像：test-images-idx3-ubyte.gz <br>\n",
        "テストデータセットのラベル：test-labels-idx1-ubyte.gz <br>\n",
        "の4種類から構成されています。上記のwebサイトからダウンロードできます。\n",
        "\n",
        "ここでは、Kerasのdatasetsを用いて colab上で簡単にダウンロードします。\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7r-QS3lynLWm",
        "colab_type": "code",
        "outputId": "977ea1dc-d9b7-46f8-8a06-2559d54d67dd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 97
        }
      },
      "source": [
        "%matplotlib inline\n",
        "from tensorflow.keras.datasets import mnist\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "#Kerasの関数でデータの読み込み。データをシャッフルして学習データと訓練データに分割してくれる\n",
        "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
        "\n",
        "X_train = X_train.reshape(60000, 784)\n",
        "X_test = X_test.reshape(10000, 784)\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 0s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "VyY0I60_XJnz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "500d6d4d-bf34-4049-8fa1-429609321914"
      },
      "source": [
        "print('Rows: %d, columns: %d' % (X_train.shape[0], X_train.shape[1]))"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Rows: 60000, columns: 784\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qic2z7NVXJn3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "17e8a35f-e250-423d-d292-6b2c1ee81bbf"
      },
      "source": [
        "print('Rows: %d, columns: %d' % (X_test.shape[0], X_test.shape[1]))"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Rows: 10000, columns: 784\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sGgsw-VxXJn5",
        "colab_type": "text"
      },
      "source": [
        "Visualize the first digit of each class:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iVDDIlQuXJn6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        },
        "outputId": "c41ea49e-f664-4058-ceae-c12027e3bcfa"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "fig, ax = plt.subplots(nrows=2, ncols=5, sharex=True, sharey=True,)\n",
        "ax = ax.flatten()\n",
        "for i in range(10):\n",
        "    img = X_train[y_train == i][0].reshape(28, 28)\n",
        "    ax[i].imshow(img, cmap='Greys', interpolation='nearest')\n",
        "\n",
        "ax[0].set_xticks([])\n",
        "ax[0].set_yticks([])\n",
        "plt.tight_layout()\n",
        "# plt.savefig('./figures/mnist_all.png', dpi=300)\n",
        "plt.show()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAADPCAYAAACgNEWWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAcrElEQVR4nO3dd5zUxf3H8deJgmJDKcbEgiJSEhDE\niBr8SegIgQiCGBEpGgWsgEpIQrfF0BQpClKNvUQTE0UQjURBQCKgAmpAUSkKoiKg6P3+yOMzO8t+\n77w7tszuvp//OI/PbRm+7t3szPcznykoLCxEREQkNAdkugMiIiJRNECJiEiQNECJiEiQNECJiEiQ\nNECJiEiQNECJiEiQDizNg6tUqVJYvXr1FHUley1btuzTwsLCqqV9nq5nNF3P5NL1TK6yXk/QNS1K\nUde0VANU9erVWbp0afJ6lSMKCgo2lOV5up7RdD2TS9czucp6PUHXtChFXVMt8YmISJA0QImISJA0\nQImISJA0QImISJA0QImISJA0QImISJA0QImISJA0QImISJBKtVE3RB9++KFrT5gwAYBx48a52A03\n3ADAdddd52LHH398mnonIiJlpRmUiIgEKWtnUB999BEADRs2dLHPP/8cgIKCAhcbP348ALNmzXKx\nrVu3pqOLOem+++4D4KqrrnKx77//HoA1a9a42KmnnprejgVqz549rv3tt98C8Morr7iYfY4vu+wy\nFzvwwKz9tSyzTz/91LX37t0LwJIlS1ysY8eOABxwQOm/U/fq1QuAqVOnuli5cuXK1E/5n7fffhuA\nFi1auNiKFSsAqFq1TGUKI2kGJSIiQdIAJSIiQcqqtYQNG2IFb5s2bQrA9u3bXcyW9o488kgXq1Ch\nAgBbtmxxsffffx+AE0880cU05S/a/PnzXXvAgAFA9FKLv7Saj2yJGWDMmDEALFiwwMUWL15c5HNt\nqQ9g6NChKehdODZt2uTas2fPBuDee+91MVsy/uCDD1zMPm9l+YzNnDkTgKOOOsrFRo8eDcT+PoRm\n3bp1QPzftzPPPDNT3Ulgn+XmzZun9H00gxIRkSAFO4OyG8oQmzm1adPGxfz08n01aNDAtW+55RYA\nmjRp4mI1a9YE4r+19enTZz97nLvWrl3r2rt3785gT8LhJ9rY9gb7L8CuXbsAKCwsdLGTTjoJgMqV\nK7vYsmXLgPgb+H379gWSe7M5JIMHD3btuXPnpu19/e0nluRTo0aNtL1/adiqxTvvvONimZ5B+Z9l\nm+H5fxtSQTMoEREJkgYoEREJUrBLfDfeeKNrT5w4sVTPfemll1x7586dAFxwwQUu9sQTTwDwxhtv\n7E8Xc95bb70FwPDhwxN+dvrpp7v2888/D8Chhx6aln6lm7+saTfXJ0+e7GI7duwo8rn16tVzbftc\n2j4fgGOOOQaAzZs3J7xeri7x/epXv3LtqCW+H//4xwAMGjTIxSxxIio551//+pdrP/nkk0nrZybd\nddddALRq1SrDPYn56quvXPu2224D4iv0pOLzqhmUiIgEKbgZlCU/+N+s/JtzxmZEnTt3drHu3bsD\n8bX26tSpA8DNN9/sYo899liRr5vv3n33Xdc+//zzAdi2bVvC426//XbX9tP6c9GiRYtc2/93F6du\n3boAvPzyyy52xBFHAPDZZ58lsXfZx1/NiPps2SzpsMMOK9HrXXnlla5tv+9+irrp3bu3a/tbTEL0\n3XffZboLCfzqMcaud6poBiUiIkHSACUiIkEKYonP30VvxV/9Xfm2e/ySSy5xMStaajfy/Vi3bt1c\nrGLFikDsxivElhDmzJnjYrY3I9+P4pg2bZprR+0169SpEwC//OUv09anTLNKBEWxwrjNmjVzMdt/\nZ8t6Pr8iSj7yEx2irk9pLV++3LX9orP7OuGEE1w7xIK8H3/8sWv7fxNDEbUc27Jly5S+p2ZQIiIS\npIx+jbBvO3fccYeLWe0pS7+F2A5822EPUL58eSC+aoTfLomvv/7ate+8804glt6Zb+xa2HWA2Ddd\nv/LBqFGj0tuxAEyaNMm1zz77bCC+qol9VkuaZu/XhZSys2NL/Aoe/u/0vvytKyGy7RpQ/L8j3Wyr\nzsqVKxN+5v9tSAXNoEREJEgaoEREJEhpX+Lzd9HbTnF/z5PtqXnuuedc7JRTTgHiC8gm23//+9+U\nvXao/EQUO7E0il9Jonbt2qnsUpAOP/xw1+7Xr99+v55/BIeUjO0nGzhwoIutXr0agG+++abY5557\n7rlA2U7jTadVq1YlxEp72yIVfv/73wPxSRz169cHYrdaUiXs/2MiIpK30j6D8nd4R9Xheu2114BY\n6q7vkEMOSV3H8pBfw+zf//53ws+7dOkCQM+ePdPVpaxmFUoAvvjiCyC+Woltl7AjNnzt2rVz7ZNP\nPjlVXQyCP3N/5JFHAHj22WeLfc4zzzwD/PCBhZUqVQJiByFC7Kidgw46qPSdzbDGjRun/D327Nnj\n2vbZ9I8ievjhhxOeY8lkBx98cEr7phmUiIgESQOUiIgEKe1LfP3793dtW/7wi0dGLe0lW1Tp/nwq\nHPv6668DcNlllyX8zD8KwSpzpHoan00sUce/YTx06FAgesnaPmsQfZPeKpfMmDGj2Mflgk8++QSA\npk2buth7772X1Pewz68VOs52/nJocfzPo33m/GOHLAnMTyi5++67gfjCtLaXzz/mw37//SS1VBeJ\nNbn5myAiIlkvbTMoOxzQP37Abnjazfh0sW+o/g3XM844I619SDf/m9hZZ51V5OMspR9y9wDCkrJv\nlhs3bnQx+/bv1ym0eo9+Hce2bdsC8OCDD7qYf+CbsW0Xf//7313sN7/5DQDlypXbr/6Hyl+tKOnK\nRXEHFvosOcI/SC+EVO2SsM8RxP42dejQwcVq1apV5HNfffVV17Zr6tcbtKNL/KQL2+ZjafgQu1b+\n7759rq2iBKTvME3NoEREJEgaoEREJEhpW+LbvXs3EJ9zb0dg+HtAks2WUKKKwF544YWuPWTIkJT1\nIQRjxoxx7eKWSfyTh/ORf8N4xYoVQPReFL+AbPPmzQGoUaOGi+3atQuAN99808UWL16c8DqbNm0C\noFevXi5m+6D89w3xeIjSOvbYY4FYkg7Ao48+CsTflC9pdYLp06cDMGzYsGR1MaNGjhzp2vZZWrhw\nYYmeW7NmTde2JWJ/ud4KbpeUvy/NPqOZqCKjGZSIiAQpo1/LLH3RbuAli1/vb/LkyQDcdNNNLla9\nenUgVmMKUl9TKlPs4DO/ykEU+wafrpufobGZk390g/+ZMfbttEePHi5mn2P/iIT27dsDscooABUq\nVADijzSxWZqfZn7eeecB0LVrVxezVPao35XjjjuumH9ZeKzeJsDll19e5texuny5MoPy2RaQqK0g\n6fC3v/0tIda7d++090MzKBERCZIGKBERCVJGl/guvfTSpL6eLWf5J/TazWz/JrRVSMgHtr/LTi/2\ntW7d2rUnTpyYtj6Fwq/yMH78eCA+ScSO2Zg5c6aL2TXzq2ts2LABgCuuuMLFbL9fvXr1XOyhhx4C\n4m82W9LQNddc42L3338/ALNmzXIxK6rqs2SKtWvXFvVPzGnLly/PdBfySqdOndL+nppBiYhIkNI2\ng7Ldzf7Ocftm+sc//rHMr+vv1Ldvodu3b3exa6+9FoBx48aV+T2y2ZYtW4Do1HJ/tpCrSSLF8W8E\n27XwkxDsiIdGjRq52Jo1awCYMmWKi1kNPksth9iM1JIqAI444oiEPljihB0AB7HZXOfOnV0satYf\n8mfakk5WrlzpYj/96U+B/Tv2Yt68ea6d7go0kn6aQYmISJA0QImISJDStsRnxQ/9Aq1WhNPfQd2n\nTx8gdoMaYPXq1QBMnTrVxew02PXr17uY7b7u1q2bi9kSXz6xIpAQnwiwL39ZKR/169cvIebvobN9\ncjt27HCxVatWFfl6tucOYp/j/Tk6wy/i6bdDtW7dOtcePnw4EH8a67Zt24CSL/H5S6ZLliwB4n+3\no4rvWsFVHRGTPHZbxpKBIH2nPmsGJSIiQcpomrndSPVnUFZf6+ijj3Yx/0brvuxYA4A2bdoAcPXV\nVye1n9kiqmqEfYO3m/EQ23mf78dpWEURiNUbs5qRAIsWLUp4Tvfu3QFo2bKli9lnsFKlSi6Wq4cO\nFqdnz56uHVV30JI6opJFoliSCsQO3/NXYIyf/mzVJTJRNy5X2TUvbjUmVfLvt0hERLKCBigREQlS\n2pb4bA9EixYtXOyFF15IeJwlTthyla9atWqu3bdvX2D/9lDlGrtpHHXt/OWsfD9Sw8yfP9+17URS\nf1nPjoe46KKLXMxuvufqabepNGrUqP1+DTuiB2KVaEaMGOFiuXAsSagWLFjg2nbETKppBiUiIkFK\n29cNuzHq38CfPXs28MOp4KNHjwbia51Vrlw52V2UPOMnjjRt2jTuv1J6fkq5HRA6duzYUr9O3bp1\ngfhkCjvQ0P8bYDNcSS2/+k+6aQYlIiJB0gAlIiJBSvsdRb8Yp+3kj9rRL6X3k5/8BIB27dq5mL+X\nRCSV/JN9b731VgD+7//+z8Xs9Fz/6Bc7pbVDhw4uZsusyT5pW0rOL1TsF0ZON82gREQkSMrJzCH2\njfOpp57KcE8k31m6d/v27V3MqnVI+Pw08kxUkDCaQYmISJA0QImISJA0QImISJA0QImISJA0QImI\nSJA0QImISJA0QImISJAKSlMIsKCgYCuw4QcfmH9OLCwsrFraJ+l6FknXM7l0PZOrTNcTdE2LEXlN\nSzVAiYiIpIuW+EREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEga\noEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoERE\nJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEga\noEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoERE\nJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEgaoEREJEga\noEREJEgHlubBVapUKaxevXqKupK9li1b9mlhYWHV0j5P1zOarmdy6XomV1mvJ+iaFqWoa1qqAap6\n9eosXbo0eb3KEQUFBRvK8jxdz2i6nsml65lcZb2eoGtalKKuqZb4REQkSBqgREQkSBqgREQkSKW6\nByX55dNPPwXgF7/4hYvt3bsXgPfeey8jfRKR/KEZlIiIBEkzKIkzYsQI154yZQoAW7dudbEePXqk\nvU8ikp80gxIRkSBpgBIRkSBpiS+P7dy507W7dOkCwHPPPediBQUFADRu3NjF7rnnnjT1TkTynWZQ\nIiISpKyYQX3//fcA7Nmzp9jHzZo1C4ifGbz11lsAjB8/3sWGDBkCwMSJE13skEMOAWDMmDEu1rdv\n3/3pdrAsfXzQoEEu9vzzzyc8bsaMGQD8/Oc/dzG7TiIh++abb1y7TZs2QPzWiP/85z8AVKpUKb0d\nk1LRDEpERIKkAUpERIKU0SW+HTt2APDdd9+5mE29/SWnzz//HIB777231O9hpe0HDhzoYtOnTwfg\nyCOPdLFzzz0XgGbNmpX6PbLNF198AcDcuXOLfZxdu9q1a6e6SyIl8uWXX0a2zaGHHgrAsmXLXGzh\nwoUAnHbaaS6mpersoBmUiIgEKe0zqI0bN7p2gwYNANi+fXtS3+OAA2Ljrs2W/G9Mffr0AaBatWou\ndthhhwFQtWqZziELniVGALRt2xaAwsLChMctXrzYtc8444zUdyzH/eUvf3Ht3bt3A7By5UoXu+uu\nuxKe07BhQ4C8Ojfok08+cW27JuvXr094nD8ziqoHaUlO/jW2z3nNmjVdzBKv8oVdy5kzZ7rYP//5\nTwBef/31hMc/8MADrn388ccDMG/ePBfr2bMnEFtlSRXNoEREJEgaoEREJEhpX+KrXLmyax9zzDFA\n2Zb4WrVqlfB6TzzxBAAVKlRwsaZNm5almznnwQcfdG1bGunevbuL2Z6www8/PL0dywFr164FYnvu\nIFaRY9q0aS4WtaRq1Tp8b775JgCnn366iy1fvjw5nQ3UokWLXPtPf/pTkY87+OCDXfu6664DYr/3\nEJ8MZewa9+/f38XyIUnCv6Zdu3YFYPPmzS5mn8dOnTq52IcffgjE/23Y9/EQKyCd6soymkGJiEiQ\n0j6D8r+52A27xx57zMXOPvtsADp37pzw3CZNmrj2X//6VwDKly/vYps2bQJgwoQJyetwlrOEiJdf\nftnFTj31VADGjh3rYpo5Jfrqq68AuPTSS13MtkH4bAXAT3u2b5v+DP6ll14q0fvaDXzbhpHLJk2a\nBMBNN92U8LMBAwa4tq229OvXz8UqVqwIxM+arOqJP1P40Y9+BMQfvJlr/KQPS4ho166di9ln+de/\n/rWLjR49GohPHrEtP71793axhx56KOH9zjnnnCT0+odpBiUiIkHSACUiIkHKaCUJm47Xr1/fxWzJ\nzp/y203TUaNGJTzOZ1P52267LfmdzSL+/hmryOHfjL/88ssBOOigg9LbsSzgJzrYcsj7779f6tex\n5WbbXwexZZbPPvvMxdq3bw9E7/k566yzSv2+2cauyddff+1ip5xyCgDDhg1zMf86mm3btgGxpSqI\nXXerKAEwefJkAA48MCtqY5fJiy++6NqtW7dO+PlFF10EwP333+9ifjKZeeWVV4DoZT1/z9MFF1xQ\n5r6WhmZQIiISpCC+UkSN5EcddVRCzN91b7XzotJ085VVKpg/f36xj6tSpQoARxxxRIle99FHH3Xt\nqNnEzTffXNIuBm/kyJGuXdzMyU93nj17NgCNGjVysaiKJJYgdPfdd7tY1MzJkljuu+++EvY6e1n6\ns/8Zs5T6oUOHutjtt98OxB+5Y0kUc+bMcTG77n6iVMeOHZPd7WDY38QbbrjBxexvon/97Hc06m+t\n7/rrry/yZw8//LBrW4JKqmkGJSIiQdIAJSIiQQpiiS+KP9VcsmQJAE8++aSLrV69GoCf/exn6e1Y\nwGxqb9cLYvsj/AK6tjwaxa84Ya/n36x+9913E54zePBgIHaMB2TfvqpVq1YBsQKaRalRowYAzz77\nbEKspD744INif96jRw8gfcsomXTccccB0Lx5cxezJT6/QsTFF18MwCWXXOJiUcVibV9V1D7KXDFl\nyhTXtqU9f+muW7duAPzud79zsaiEqL179wLxe/vWrVsHxFeNsGXETBSP1gxKRESCFOwMyk8jt4MK\n/Zv/duPT3xltO8X9FMh8SqKwFGmrsgGxmZP/LT8qOeKjjz4C4q+xX5rf2Mzo5JNPdjH7BtalSxcX\nsxuq/qGQIbvllluAWNqzz9+RbzfrSzprssQViM1sn3766WLfI5dv6u/LUr8rVaqU8DOrCwexlHv/\nm739bvtbUlq2bJmSfobAPkv+dhu7BjZrgvhU8n1Zaj7EUs/9FHVz5ZVXuvYVV1xRxh7vP82gREQk\nSBqgREQkSMEu8fmOPvpoIHaEAUCbNm0AGD9+vItZ25/i2s3SqJ3oucDfFxK1b8dOw7z22mtdzI4o\n8U/ZveOOOwCYMWOGi1mBTn/p7sYbbwTid/7XqVMHgC1btpTxX5F5lpTz8ccfu5jtqfGXOkv7OfJP\n1P3tb3+b8HOrpuKfYJqrn9XiWPWI0rAjIfxisSXd25eNrJCrXwjXjBs3zrV37twJxBfhtiX3V199\n1cUsqcm/DWJtqzYD0VV70kUzKBERCVJWzKDMmWee6dqWZu7voLbd6H6peEtFtW/+kH0p0MV55513\nXNtuevosBfyqq65yMfuGNWjQIBebO3cuEJ/UYN/4//CHP7iYzbr897LndOjQISGWLRo3bgyU/EiM\nH2Kp0ldffXXCz/yUX/v/k4+zJohtg5g3b56LRR3saPyjT2bNmpW6jgWoXLlyQKzmKMRqD9oqExSf\nGHbCCSe4tiWm+MkotmriH5aZSZpBiYhIkDRAiYhIkLJqic937LHHAvE3sG0Zq0WLFi5m+1vWrFnj\nYn7Rw2y3YsWKYn/uL+0ZS3qwozh8r732mmtb0VI/+cJiPrvGuVQ0dn9Z8kPUcsvjjz/u2ueff37a\n+hSivn37AjBt2jQXK26JKp/2Ne7LChTbkRgQ2x+2detWF6tbty4Qvxxq1Un8Y0js5/4Sn/3/CIVm\nUCIiEqSsnUEZ/9iDpk2bArGbiRCrN/XUU0+5mM2matWqlYYeppZ/+J3dXO7Vq1fC46xSBMQSTPyb\n0ZYO7c+QLCGibdu2Ce/hp09HJWfkIz/VN6oGorHZVb758ssvgfgVDDtSxJ8ZnXfeeUD8dfrzn/8M\nxG8DyFf+wYGWJFFSVmsPYn8T/c9o7dq1969zSaYZlIiIBEkDlIiIBClrl/hsqu+X5Ldd0ras5/OX\nC6Ju9OcCWyb5oRvJNqX3H7d06VIgvkT/rl27gPgjTexxP3QyZz6xHf52bSD6GtvOfjvRON8sW7YM\niC9EavzTg+1IDb/qgS3xnXbaaansYs7zixdHfUb95fwQaAYlIiJByooZlKVQ3nPPPS5mNeM2btxY\n7HMtYcK/sZhLqar+cSN27IBfT89mRJYYAbBjx46E17Eb/H7ihO0qv/POO10sl6pw7I9vv/3Wta0K\nQtT2Bb+ShNWPzKXP3w/xt3dEHSJos6p69eq5mB150r9//4THl/ZwSInnX+dsoBmUiIgESQOUiIgE\nKbglPpveP/PMMy42cuRIANauXVui12jWrJlr2wmojRo1SlYXg+IXHrWCo/6psDVr1gRKvqwUVSy2\nQYMG+93PXGHHmwwYMMDFpk6dmvA4W+7zl7XyaWnP/OMf/3Dt7du3A/EnXjds2BCIJZoALFiwAIg/\n/dWWnq2CjJTNypUrM92FUtEMSkREgpTRGZQd++DXgrJDyN54440SvUarVq1ce8SIEUB8Snmuf2u1\nAwkBFi5cCMRq40F8Gv6+/FmAzTDtGy3kbjr+/rAEk6hZk9VAA7jwwgvT1qeQ+VUKorZB2MxpyZIl\nLma1Iv10fKvz2LFjx9R1Ng9EHWoaMs2gREQkSBqgREQkSGlb4rOqBNdff72LWdl4/1TY4vhHEwwd\nOhSIv4HvJwzkI7sWdrKwJId/lMHYsWMTfl6/fn0AXnzxxbT1KVts3rw5IVatWjXXtqXQp59+OuFx\nfoJFKCe8Zjv/VPLiChqHItyeiYhIXkvJDGr9+vUA3HrrrS72wgsvALBhw4YSvUbFihVde9SoUQD0\n69fPxcqXL7+/3RQpEfv8AUyaNCnh58OGDQPiU/Tlf2x26fMTTCx9vGrVqi5mqyPZVvUgG/hp+lZj\n8+2333Yxm/GedNJJ6e1YETSDEhGRIGmAEhGRIKVkie/xxx8HYPr06cU+zm58XnzxxbEOHfi/LlkV\nA4g/NVckXey00qjiukOGDHHtc845J219yjb+viUrYuwX0G3ZsiUQ2/sE0K1btzT1Lr+NHz8egNat\nW7uYFZyeOHGii1nR6EzQDEpERIKUkhnUwIED4/4rko3mzp0LwAMPPOBiVtvwmmuucTH/Br/E81c/\nevToEfdfyawmTZoA0LVrVxd75JFHgPgqHhMmTAAyk5imGZSIiARJA5SIiAQpuOM2RELRrl07AAYP\nHuxic+bMAbSsJ9mvQoUKQPwJ3LVq1QLi9/4NHz4cyEyyhGZQIiISJM2gRIpQp04dAPbu3Zvhnoik\njs2kIFYVxf6baZpBiYhIkDRAiYhIkAqsWGOJHlxQsBUoWbXX/HJiYWFhqe+a63oWSdczuXQ9k6tM\n1xN0TYsReU1LNUCJiIiki5b4REQkSBqgREQkSBqgREQkSBqgREQkSBqgREQkSBqgREQkSBqgREQk\nSBqgREQkSBqgREQkSP8P783lTcv5Hc8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 10 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wyGqKfTJXJn9",
        "colab_type": "text"
      },
      "source": [
        "Implementing a multi-layer perceptron"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cVe0vZCIXJn-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "from scipy.special import expit\n",
        "import sys\n",
        "\n",
        "\n",
        "class NeuralNetMLP(object):\n",
        "    \"\"\" Feedforward neural network / Multi-layer perceptron classifier.\n",
        "\n",
        "    Parameters\n",
        "    ------------\n",
        "    n_output : int\n",
        "        Number of output units, should be equal to the\n",
        "        number of unique class labels.\n",
        "    n_features : int\n",
        "        Number of features (dimensions) in the target dataset.\n",
        "        Should be equal to the number of columns in the X array.\n",
        "    n_hidden : int (default: 30)\n",
        "        Number of hidden units.\n",
        "    l1 : float (default: 0.0)\n",
        "        Lambda value for L1-regularization.\n",
        "        No regularization if l1=0.0 (default)\n",
        "    l2 : float (default: 0.0)\n",
        "        Lambda value for L2-regularization.\n",
        "        No regularization if l2=0.0 (default)\n",
        "    epochs : int (default: 500)\n",
        "        Number of passes over the training set.\n",
        "    eta : float (default: 0.001)\n",
        "        Learning rate.\n",
        "    alpha : float (default: 0.0)\n",
        "        Momentum constant. Factor multiplied with the\n",
        "        gradient of the previous epoch t-1 to improve\n",
        "        learning speed\n",
        "        w(t) := w(t) - (grad(t) + alpha*grad(t-1))\n",
        "    decrease_const : float (default: 0.0)\n",
        "        Decrease constant. Shrinks the learning rate\n",
        "        after each epoch via eta / (1 + epoch*decrease_const)\n",
        "    shuffle : bool (default: True)\n",
        "        Shuffles training data every epoch if True to prevent circles.\n",
        "    minibatches : int (default: 1)\n",
        "        Divides training data into k minibatches for efficiency.\n",
        "        Normal gradient descent learning if k=1 (default).\n",
        "    random_state : int (default: None)\n",
        "        Set random state for shuffling and initializing the weights.\n",
        "\n",
        "    Attributes\n",
        "    -----------\n",
        "    cost_ : list\n",
        "      Sum of squared errors after each epoch.\n",
        "\n",
        "    \"\"\"\n",
        "    def __init__(self, n_output, n_features, n_hidden=30,\n",
        "                 l1=0.0, l2=0.0, epochs=500, eta=0.001,\n",
        "                 alpha=0.0, decrease_const=0.0, shuffle=True,\n",
        "                 minibatches=1, random_state=None):\n",
        "\n",
        "        np.random.seed(random_state)\n",
        "        self.n_output = n_output\n",
        "        self.n_features = n_features\n",
        "        self.n_hidden = n_hidden\n",
        "        self.w1, self.w2 = self._initialize_weights()\n",
        "        self.l1 = l1\n",
        "        self.l2 = l2\n",
        "        self.epochs = epochs\n",
        "        self.eta = eta\n",
        "        self.alpha = alpha\n",
        "        self.decrease_const = decrease_const\n",
        "        self.shuffle = shuffle\n",
        "        self.minibatches = minibatches\n",
        "\n",
        "    def _encode_labels(self, y, k):\n",
        "        \"\"\"Encode labels into one-hot representation\n",
        "\n",
        "        Parameters\n",
        "        ------------\n",
        "        y : array, shape = [n_samples]\n",
        "            Target values.\n",
        "\n",
        "        Returns\n",
        "        -----------\n",
        "        onehot : array, shape = (n_labels, n_samples)\n",
        "\n",
        "        \"\"\"\n",
        "        onehot = np.zeros((k, y.shape[0]))\n",
        "        for idx, val in enumerate(y):\n",
        "            onehot[val, idx] = 1.0\n",
        "        return onehot\n",
        "\n",
        "    def _initialize_weights(self):\n",
        "        \"\"\"Initialize weights with small random numbers.\"\"\"\n",
        "        w1 = np.random.uniform(-1.0, 1.0,\n",
        "                               size=self.n_hidden*(self.n_features + 1))\n",
        "        w1 = w1.reshape(self.n_hidden, self.n_features + 1)\n",
        "        w2 = np.random.uniform(-1.0, 1.0,\n",
        "                               size=self.n_output*(self.n_hidden + 1))\n",
        "        w2 = w2.reshape(self.n_output, self.n_hidden + 1)\n",
        "        return w1, w2\n",
        "\n",
        "    def _sigmoid(self, z):\n",
        "        \"\"\"Compute logistic function (sigmoid)\n",
        "\n",
        "        Uses scipy.special.expit to avoid overflow\n",
        "        error for very small input values z.\n",
        "\n",
        "        \"\"\"\n",
        "        # return 1.0 / (1.0 + np.exp(-z))\n",
        "        return expit(z)\n",
        "\n",
        "    def _sigmoid_gradient(self, z):\n",
        "        \"\"\"Compute gradient of the logistic function\"\"\"\n",
        "        sg = self._sigmoid(z)\n",
        "        return sg * (1.0 - sg)\n",
        "\n",
        "    def _add_bias_unit(self, X, how='column'):\n",
        "        \"\"\"Add bias unit (column or row of 1s) to array at index 0\"\"\"\n",
        "        if how == 'column':\n",
        "            X_new = np.ones((X.shape[0], X.shape[1] + 1))\n",
        "            X_new[:, 1:] = X\n",
        "        elif how == 'row':\n",
        "            X_new = np.ones((X.shape[0] + 1, X.shape[1]))\n",
        "            X_new[1:, :] = X\n",
        "        else:\n",
        "            raise AttributeError('`how` must be `column` or `row`')\n",
        "        return X_new\n",
        "\n",
        "    def _feedforward(self, X, w1, w2):\n",
        "        \"\"\"Compute feedforward step\n",
        "\n",
        "        Parameters\n",
        "        -----------\n",
        "        X : array, shape = [n_samples, n_features]\n",
        "            Input layer with original features.\n",
        "        w1 : array, shape = [n_hidden_units, n_features]\n",
        "            Weight matrix for input layer -> hidden layer.\n",
        "        w2 : array, shape = [n_output_units, n_hidden_units]\n",
        "            Weight matrix for hidden layer -> output layer.\n",
        "\n",
        "        Returns\n",
        "        ----------\n",
        "        a1 : array, shape = [n_samples, n_features+1]\n",
        "            Input values with bias unit.\n",
        "        z2 : array, shape = [n_hidden, n_samples]\n",
        "            Net input of hidden layer.\n",
        "        a2 : array, shape = [n_hidden+1, n_samples]\n",
        "            Activation of hidden layer.\n",
        "        z3 : array, shape = [n_output_units, n_samples]\n",
        "            Net input of output layer.\n",
        "        a3 : array, shape = [n_output_units, n_samples]\n",
        "            Activation of output layer.\n",
        "\n",
        "        \"\"\"\n",
        "        a1 = self._add_bias_unit(X, how='column')\n",
        "        z2 = w1.dot(a1.T)\n",
        "        a2 = self._sigmoid(z2)\n",
        "        a2 = self._add_bias_unit(a2, how='row')\n",
        "        z3 = w2.dot(a2)\n",
        "        a3 = self._sigmoid(z3)\n",
        "        return a1, z2, a2, z3, a3\n",
        "\n",
        "    def _L2_reg(self, lambda_, w1, w2):\n",
        "        \"\"\"Compute L2-regularization cost\"\"\"\n",
        "        return (lambda_/2.0) * (np.sum(w1[:, 1:] ** 2) +\n",
        "                                np.sum(w2[:, 1:] ** 2))\n",
        "\n",
        "    def _L1_reg(self, lambda_, w1, w2):\n",
        "        \"\"\"Compute L1-regularization cost\"\"\"\n",
        "        return (lambda_/2.0) * (np.abs(w1[:, 1:]).sum() +\n",
        "                                np.abs(w2[:, 1:]).sum())\n",
        "\n",
        "    def _get_cost(self, y_enc, output, w1, w2):\n",
        "        \"\"\"Compute cost function.\n",
        "\n",
        "        Parameters\n",
        "        ----------\n",
        "        y_enc : array, shape = (n_labels, n_samples)\n",
        "            one-hot encoded class labels.\n",
        "        output : array, shape = [n_output_units, n_samples]\n",
        "            Activation of the output layer (feedforward)\n",
        "        w1 : array, shape = [n_hidden_units, n_features]\n",
        "            Weight matrix for input layer -> hidden layer.\n",
        "        w2 : array, shape = [n_output_units, n_hidden_units]\n",
        "            Weight matrix for hidden layer -> output layer.\n",
        "\n",
        "        Returns\n",
        "        ---------\n",
        "        cost : float\n",
        "            Regularized cost.\n",
        "\n",
        "        \"\"\"\n",
        "        term1 = -y_enc * (np.log(output))\n",
        "        term2 = (1.0 - y_enc) * np.log(1.0 - output)\n",
        "        cost = np.sum(term1 - term2)\n",
        "        L1_term = self._L1_reg(self.l1, w1, w2)\n",
        "        L2_term = self._L2_reg(self.l2, w1, w2)\n",
        "        cost = cost + L1_term + L2_term\n",
        "        return cost\n",
        "\n",
        "    def _get_gradient(self, a1, a2, a3, z2, y_enc, w1, w2):\n",
        "        \"\"\" Compute gradient step using backpropagation.\n",
        "\n",
        "        Parameters\n",
        "        ------------\n",
        "        a1 : array, shape = [n_samples, n_features+1]\n",
        "            Input values with bias unit.\n",
        "        a2 : array, shape = [n_hidden+1, n_samples]\n",
        "            Activation of hidden layer.\n",
        "        a3 : array, shape = [n_output_units, n_samples]\n",
        "            Activation of output layer.\n",
        "        z2 : array, shape = [n_hidden, n_samples]\n",
        "            Net input of hidden layer.\n",
        "        y_enc : array, shape = (n_labels, n_samples)\n",
        "            one-hot encoded class labels.\n",
        "        w1 : array, shape = [n_hidden_units, n_features]\n",
        "            Weight matrix for input layer -> hidden layer.\n",
        "        w2 : array, shape = [n_output_units, n_hidden_units]\n",
        "            Weight matrix for hidden layer -> output layer.\n",
        "\n",
        "        Returns\n",
        "        ---------\n",
        "        grad1 : array, shape = [n_hidden_units, n_features]\n",
        "            Gradient of the weight matrix w1.\n",
        "        grad2 : array, shape = [n_output_units, n_hidden_units]\n",
        "            Gradient of the weight matrix w2.\n",
        "\n",
        "        \"\"\"\n",
        "        # backpropagation\n",
        "        sigma3 = a3 - y_enc\n",
        "        z2 = self._add_bias_unit(z2, how='row')\n",
        "        sigma2 = w2.T.dot(sigma3) * self._sigmoid_gradient(z2)\n",
        "        sigma2 = sigma2[1:, :]\n",
        "        grad1 = sigma2.dot(a1)\n",
        "        grad2 = sigma3.dot(a2.T)\n",
        "\n",
        "        # regularize\n",
        "        grad1[:, 1:] += self.l2 * w1[:, 1:]\n",
        "        grad1[:, 1:] += self.l1 * np.sign(w1[:, 1:])\n",
        "        grad2[:, 1:] += self.l2 * w2[:, 1:]\n",
        "        grad2[:, 1:] += self.l1 * np.sign(w2[:, 1:])\n",
        "\n",
        "        return grad1, grad2\n",
        "\n",
        "    def predict(self, X):\n",
        "        \"\"\"Predict class labels\n",
        "\n",
        "        Parameters\n",
        "        -----------\n",
        "        X : array, shape = [n_samples, n_features]\n",
        "            Input layer with original features.\n",
        "\n",
        "        Returns:\n",
        "        ----------\n",
        "        y_pred : array, shape = [n_samples]\n",
        "            Predicted class labels.\n",
        "\n",
        "        \"\"\"\n",
        "        if len(X.shape) != 2:\n",
        "            raise AttributeError('X must be a [n_samples, n_features] array.\\n'\n",
        "                                 'Use X[:,None] for 1-feature classification,'\n",
        "                                 '\\nor X[[i]] for 1-sample classification')\n",
        "\n",
        "        a1, z2, a2, z3, a3 = self._feedforward(X, self.w1, self.w2)\n",
        "        y_pred = np.argmax(z3, axis=0)\n",
        "        return y_pred\n",
        "\n",
        "    def fit(self, X, y, print_progress=False):\n",
        "        \"\"\" Learn weights from training data.\n",
        "\n",
        "        Parameters\n",
        "        -----------\n",
        "        X : array, shape = [n_samples, n_features]\n",
        "            Input layer with original features.\n",
        "        y : array, shape = [n_samples]\n",
        "            Target class labels.\n",
        "        print_progress : bool (default: False)\n",
        "            Prints progress as the number of epochs\n",
        "            to stderr.\n",
        "\n",
        "        Returns:\n",
        "        ----------\n",
        "        self\n",
        "\n",
        "        \"\"\"\n",
        "        self.cost_ = []\n",
        "        X_data, y_data = X.copy(), y.copy()\n",
        "        y_enc = self._encode_labels(y, self.n_output)\n",
        "\n",
        "        delta_w1_prev = np.zeros(self.w1.shape)\n",
        "        delta_w2_prev = np.zeros(self.w2.shape)\n",
        "\n",
        "        for i in range(self.epochs):\n",
        "\n",
        "            # adaptive learning rate\n",
        "            self.eta /= (1 + self.decrease_const*i)\n",
        "\n",
        "            if print_progress:\n",
        "                sys.stderr.write('\\rEpoch: %d/%d' % (i+1, self.epochs))\n",
        "                sys.stderr.flush()\n",
        "\n",
        "            if self.shuffle:\n",
        "                idx = np.random.permutation(y_data.shape[0])\n",
        "                X_data, y_enc = X_data[idx], y_enc[:, idx]\n",
        "\n",
        "            mini = np.array_split(range(y_data.shape[0]), self.minibatches)\n",
        "            for idx in mini:\n",
        "\n",
        "                # feedforward\n",
        "                a1, z2, a2, z3, a3 = self._feedforward(X_data[idx],\n",
        "                                                       self.w1,\n",
        "                                                       self.w2)\n",
        "                cost = self._get_cost(y_enc=y_enc[:, idx],\n",
        "                                      output=a3,\n",
        "                                      w1=self.w1,\n",
        "                                      w2=self.w2)\n",
        "                self.cost_.append(cost)\n",
        "\n",
        "                # compute gradient via backpropagation\n",
        "                grad1, grad2 = self._get_gradient(a1=a1, a2=a2,\n",
        "                                                  a3=a3, z2=z2,\n",
        "                                                  y_enc=y_enc[:, idx],\n",
        "                                                  w1=self.w1,\n",
        "                                                  w2=self.w2)\n",
        "\n",
        "                delta_w1, delta_w2 = self.eta * grad1, self.eta * grad2\n",
        "                self.w1 -= (delta_w1 + (self.alpha * delta_w1_prev))\n",
        "                self.w2 -= (delta_w2 + (self.alpha * delta_w2_prev))\n",
        "                delta_w1_prev, delta_w2_prev = delta_w1, delta_w2\n",
        "\n",
        "        return self"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EDvI4VGRXJoA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "nn = NeuralNetMLP(n_output=10, \n",
        "                  n_features=X_train.shape[1], \n",
        "                  n_hidden=50, \n",
        "                  l2=0.1, \n",
        "                  l1=0.0, \n",
        "                  epochs=800, \n",
        "                  eta=0.001,\n",
        "                  alpha=0.001,\n",
        "                  decrease_const=0.00001,\n",
        "                  minibatches=50, \n",
        "                  shuffle=True,\n",
        "                  random_state=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fUMQ36KlXJoC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "65a0dd0e-9bab-43c0-9a95-7a31457ab5b5"
      },
      "source": [
        "nn.fit(X_train, y_train, print_progress=True)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 800/800"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<__main__.NeuralNetMLP at 0x7fc800941668>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "772LxsinXJoE",
        "colab_type": "text"
      },
      "source": [
        "plotting the decreases in cost per epoch "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kvzZzVgKXJoF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 297
        },
        "outputId": "32d16a63-303a-4405-8c32-eca6681e2b75"
      },
      "source": [
        "batches = np.array_split(range(len(nn.cost_)), 800)\n",
        "cost_ary = np.array(nn.cost_)\n",
        "cost_avgs = [np.mean(cost_ary[i]) for i in batches]\n",
        "\n",
        "plt.plot(range(len(cost_avgs)), cost_avgs, color='red')\n",
        "plt.ylim([0, 2000])\n",
        "plt.ylabel('Cost')\n",
        "plt.xlabel('Epochs')\n",
        "plt.tight_layout()\n",
        "#plt.savefig('./figures/cost2.png', dpi=300)\n",
        "plt.show()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deZhU1bX38e8SZFRCM4gIKIOKcQpg\nX5yNxgmNU64DqIkoGqLG2bzON0aNiiZqJKIRFYcoOBu5DlGCxiFX0EYRBEQaQW1ooBEQFESB9f6x\nT1nVczdU1anq+n2ep55zzj5D7SoKFnuffdY2d0dERCTXbBZ3BURERGqiACUiIjlJAUpERHKSApSI\niOQkBSgREclJClAiIpKTMhagzKyHmb1uZjPNbIaZXRiVdzCzCWY2J1oWReVmZiPNrNTMppnZgJRr\nDY2On2NmQzNVZxERyR2WqeegzKwr0NXd3zezLYEpwHHA6cAydx9hZlcARe5+uZkdCZwPHAnsCdzp\n7nuaWQegBCgGPLrOHu6+PCMVFxGRnJCxFpS7l7v7+9H6KmAW0A04Fng4OuxhQtAiKn/Eg0lA+yjI\nHQ5McPdlUVCaAAzKVL1FRCQ3NM/Gm5hZT6A/MBno4u7l0a5FQJdovRvwRcppZVFZbeU1vc9wYDhA\n27Zt99hpp502reJTpkDXrrDNNpt2HRERqdWUKVOWunvnquUZD1BmtgXwDHCRu680sx/2ububWdr6\nGN19NDAaoLi42EtKSjbtgm3bwsknw223paF2IiJSEzP7rKbyjI7iM7PNCcHpMXd/NipeHHXdJe5T\nLYnKFwA9Uk7vHpXVVp55nTrB0qVZeSsREaksk6P4DHgAmOXut6fsGg8kRuINBZ5PKT8tGs23F/BV\n1BX4CnCYmRVFI/4Oi8oyr1MnqKjIyluJiEhlmezi2xf4FTDdzKZGZVcBI4AnzexM4DPgpGjfS4QR\nfKXAauAMAHdfZmY3AO9Fx13v7ssyWO+kzp3VghIRiUnGApS7vw1YLbsPruF4B35by7XGAGPSV7sG\n6tgRSkuz/rYiIqJMEnVr3Rq+/TbuWoiIFCQFqLq0bKkAJSISEwWourRqpQAlIhITBai6tGoFa9fG\nXQsRkYKkAFWXli1h3TpYvz7umoiIFBwFqLq0ahWWakWJiGSdAlRdEgFK96FERLJOAaouLVuGpQKU\niEjWKUDVRV18IiKxUYCqi7r4RERiowBVl9atw/Kbb+Kth4hIAVKAqkuXaC7FxYvjrYeISAFSgKpL\n165huXBhvPUQESlAClB12XrrsCwvr/s4ERFJOwWourRoAe3awbLsTD8lIiJJClD1ad8eVqyIuxYi\nIgVHAao+ClAiIrFQgKqPApSISCwUoOqjACUiEgsFqPooQImIxCJjAcrMxpjZEjP7KKXsCTObGr3m\nm9nUqLynma1J2fe3lHP2MLPpZlZqZiPNzDJV5xopQImIxKJ5Bq/9EHAX8EiiwN0HJ9bN7Dbgq5Tj\n57p7vxqucw/wa2Ay8BIwCHg5A/WtWfv2sHIlbNgAm6nBKSKSLRn7F9fd3wRqfIAoagWdBIyr6xpm\n1hVo5+6T3N0Jwe64dNe1Tu3bgzt8/LGymouIZFFcTYL9gcXuPielrJeZfWBmb5jZ/lFZN6As5Ziy\nqKxGZjbczErMrKSioiI9NW3fPix32QUGD677WBERSZu4AtTJVG49lQPbunt/4BJgrJm1a+xF3X20\nuxe7e3Hnzp3TU9NEgAJ4/vn0XFNEROqVyXtQNTKz5sB/A3skytx9LbA2Wp9iZnOBHYEFQPeU07tH\nZdmTGqBERCRr4mhBHQJ87O4/dN2ZWWczaxat9wZ2AD5193JgpZntFd23Og3IbjNGAUpEJBaZHGY+\nDngH6GtmZWZ2ZrRrCNUHRxwATIuGnT8NnO3uiQEW5wL3A6XAXLI5gg8UoEREYpKxLj53P7mW8tNr\nKHsGeKaW40uAXdNaucZQgBIRiYUe7KlPuypjNb77Lp56iIgUGAWo+jRrVnl76dJ46iEiUmAUoBpi\n4MDk+pw5tR8nIiJpowDVEI8/Dl26hPUDD4y1KiIihUIBqiF69YI33oi7FiIiBUUBqqHSlZlCREQa\nRAGqoYqKkutKGisiknEKUA1lBqNGhXXNDyUiknEKUI2ReGh3+fJ46yEiUgAUoBoj0c2nFpSISMYp\nQDVGogVVVgbr18dbFxGRJk4BqjESLagTT4QLLoi3LiIiTZwCVGOkJo69++746iEiUgAUoBojdag5\nhLx8N9wAGzbEUx8RkSZMAaoxWrasvH322fD73yvLhIhIBihAbYpEZnP3eOshItIEKUA1VvOUOR4T\nLSeN6BMRSTsFqMbaeuvqZatWZb8eIiJNnAJUY519dvWyL77Ifj1ERJo4BajGuuqqMGlh6rxQF10U\nW3VERJqqjAUoMxtjZkvM7KOUsj+Y2QIzmxq9jkzZd6WZlZrZbDM7PKV8UFRWamZXZKq+DWYG228P\nmym2i4hkUib/lX0IGFRD+R3u3i96vQRgZjsDQ4BdonPuNrNmZtYMGAUcAewMnBwdG79EgNptt7Cu\ngRIiImmVsQDl7m8Cyxp4+LHA4+6+1t3nAaXAwOhV6u6fuvt3wOPRsfFLBKjttgsP6n75Zbz1ERFp\nYuLopzrPzKZFXYCJ1AzdgNSRBmVRWW3lNTKz4WZWYmYlFRUV6a531TcLyy5dwnLRosy+n4hIgcl2\ngLoH6AP0A8qB29J5cXcf7e7F7l7cOdNTtJ9/flgOinoxy8sz+34iIgUmqwHK3Re7+3p33wDcR+jC\nA1gA9Eg5tHtUVlt5/H7+85BBon//sD1jRrz1ERFpYrIaoMysa8rmL4DECL/xwBAza2lmvYAdgHeB\n94AdzKyXmbUgDKQYn80616tr9JGU3VxEJK2a13/IxjGzccCBQCczKwOuBQ40s36AA/OB3wC4+wwz\nexKYCawDfuvu66PrnAe8AjQDxrh7bjVV2rSBE06Ap5+G776DFi3irpGISJNg3kQTnRYXF3tJSUl2\n3uyJJ2DIEDjvPPjrX7PzniIiTYSZTXH34qrleto0HU46CfbZBx54IJn26Prr4Yr4nysWEclXClDp\nYAZ9+8KaNbDttjBtGlx7LdxyS9w1ExHJWwpQ6ZI6zPwnP0muf/hh9usiItIEKECly403wp57Vi/v\n1w8+/zz79RERyXMKUOkyYABMmpTcPu645PrHH2e/PiIieU4BKlMuuSS5PmdOfPUQEclTClCZktrd\nN2tWfPUQEclTClDp9uc/w2WXhQd2r7wylM2eHW+dRETyUMYySRSsSy9Nrt90UxhyrkSyIiKNphZU\nprVtC++/H56V+uqruGsjIpI3FKAyLXUiw6lT46uHiEieUYDKtG+/Ta7PnBlfPURE8owCVKalBihN\nCy8i0mAKUJm29dbJ9RUr4quHiEieUYDKtIceggcfhHbtYNy4uGsjIpI3FKAyrVMnOP10WLkSFi6E\nioq4ayQikhcUoLJt8eK4ayAikhcUoLLl+efDcsmSeOshIpInFKCypU+fsFQXn4hIgyhAZUuPHtCy\nJTz3XNieODFMCb9mDSxaBO7x1k9EJMdkLECZ2RgzW2JmH6WU/cnMPjazaWb2nJm1j8p7mtkaM5sa\nvf6Wcs4eZjbdzErNbKSZWabqnFHt2sFFF8ETT8DSpXDIIWFK+EGDoGtXuP32uGsoIpJTMtmCeggY\nVKVsArCru+8OfAJcmbJvrrv3i15np5TfA/wa2CF6Vb1m/hg4MCw7d06WvflmWL74YvbrIyKSwzIW\noNz9TWBZlbJX3X1dtDkJ6F7XNcysK9DO3Se5uwOPAMfVdU5O69at9n0bNmSvHiIieSDOe1DDgJdT\ntnuZ2Qdm9oaZ7R+VdQPKUo4pi8pqZGbDzazEzEoqcnEwQq9elbcTLSpQgBIRqSKWAGVmVwPrgMei\nonJgW3fvD1wCjDWzdo29rruPdvdidy/unNqNliu22gpuvDGsH3ww/PrXyX0aJCEiUknWA5SZnQ4c\nBZwaddvh7mvd/ctofQowF9gRWEDlbsDuUVn+GjYsuezQIVn+9tuwfn08dRIRyUFZDVBmNgi4DDjG\n3VenlHc2s2bRem/CYIhP3b0cWGlme0Wj904Dns9mndNu661Da+mUUyoHKFAyWRGRFJkcZj4OeAfo\na2ZlZnYmcBewJTChynDyA4BpZjYVeBo4290TAyzOBe4HSgktq9T7VvmtaoDSjLsiIj9onqkLu/vJ\nNRQ/UMuxzwDP1LKvBNg1jVXLHT16VN5WgBIR+YEyScSpqAiuuQaOPjps19bFt25dzeUiIk2YAlTc\nbrgBrrsurK9YAbfdBhMmwKxZUFYG06bB5pvDy02nZ1NEpCEy1sUnjdClS1j+/e8hV99WWyWzng8f\nHpYTJ8IRR8RTPxGRGKgFlQu22QZ69kwmkk2dkmP06OQxIiIFRAEqVySyTLRqFW89RERyhAJUrvjm\nm7A85ZS694uIFAgFqFxRFqUcPP74ZFlqsFKAEpECowCVK26+GbbYIuToSzj6aLj44rD+9ddh+fLL\ncPbZ1c8XEWlizJtoktLi4mIvKSmJuxobJzEn4+uvw4EHJrePOQbGj08e10T/7ESksJjZFHcvrlqu\nFlQuq5oKKTU4QZguXkSkiVKAymVFRWGZGGpe1eLF2auLiEiWKUDlskQLqra5rRYtyl5dRESyTAEq\nF731VhgI0aZN2O7YMbnvmmuS64kA9eabMH9+1qonIpINClC5aL/94J57koMjBg6Eyy6D8vKQu++N\nN0L56aeHsp/+NDzo+9RTNV9vzRr48susVF1EJF00ii8frVsXEsjWpKY/z913h+nTax/15w7nnx+e\nu9pnn/TVU0SkATZpFJ+Z/b0hZZIlzevI8XvhhWG5fDnMnRvWp08Py+++q/mctWth1CjYd9/01VFE\nZBM1tItvl9SNaHr2PdJfHWmwYcNqLh85MrSI/uu/YPvtK7eaPv+85nPWrk1//URENlGdAcrMrjSz\nVcDuZrYyeq0ClgDPZ6WGUrOTa5qwOPLJJ8nW06pVyfLy8rBctqzy8d9+m966iYikQZ0Byt1vdvct\ngT+5e7votaW7d3T3K7NUR6nJPvuE7rwHH4Rnnqm8b6edkuuJQAUhQI0aFUYFTp0ayj7/HKZMyXx9\nRUQaqaETFr5gZm3d/Rsz+yUwALjT3T/LYN2kLm3awF/+EtYXLqz9uIEDk+uDByfXZ82Cfv1gu+0y\nUz8RkU3U0HtQ9wCrzewnwKXAXOCR+k4yszFmtsTMPkop62BmE8xsTrQsisrNzEaaWamZTTOzASnn\nDI2On2NmQxv1CQvBVlsl1w8/vPK+detqPqdt28zVR0QkDRoaoNZ5GI9+LHCXu48CtmzAeQ8Bg6qU\nXQFMdPcdgInRNsARwA7RazghKGJmHYBrgT2BgcC1iaAmkcSovqOOgnbtGnbOjBn1T+GxYoWenxKR\n2DQ0QK0ysyuBXwEvmtlmQC0P4iS5+5tAlTvyHAs8HK0/DByXUv6IB5OA9mbWFTgcmODuy9x9OTCB\n6kFPvvkmTBl/000wYkT1gRAAP/5xcv2qq+DQQ6sfs3p1aF09+CD06QOdOmWuziIidWhogBoMrAWG\nufsioDvwp418zy7uHg0nYxHQJVrvBnyRclxZVFZbeTVmNtzMSsyspKKiYiOrl6fatAktqe23h8sv\nD4lmJ0xI7n/6aZg2rfI577xT/TqffhqC1LBhySDXRB/mFpHc1qAAFQWlx4AfmdlRwLfuXu89qAZc\n14G0/evn7qPdvdjdizvXlmC1kBxySBhmfsMNYS6p5s3rz9mXOuovQUlpRSQGDc0kcRLwLnAicBIw\n2cxO2Mj3XBx13REtl0TlC4AeKcd1j8pqK5eG2GKLkGA2kRpp223rPr6mVtXSpdXLpkzR81MiklEN\n7eK7Gvgvdx/q7qcRBiv8z0a+53ggMRJvKMkHfscDp0Wj+fYCvoq6Al8BDjOzomhwxGFRmWyMRALa\n2txyS/WyUaMqb7/9NhQXw623pq9eIiJVNDRAbebuS1K2v2zIuWY2DngH6GtmZWZ2JjACONTM5gCH\nRNsALwGfAqXAfcC5AO6+DLgBeC96XR+Vycb6/PPwoO7o0fDee6Es9eHehFNOCct774UPPkiWT5oU\nlo89BrNnZ7auIlKwGpTN3Mz+BOwOjIuKBgPT3P3yDNZtkzTpbObp9tRTsPfe8P77IRCNHx/Wx48P\n964gDLg45BD48MPwgG+qvfYKs/v26xeu1axZ9j+DiOSt2rKZ15lJwsy2J4y6+39m9t/AftGudwiD\nJqQpOPHEsOzePQSk886DRx+tPPVGWVlY/uIX1c9PtKjmzYMlS6Br18zWV0QKQn3ddH8BVgK4+7Pu\nfom7XwI8F+2Tpqhjx5Dnr0MHuCJ6jnrWrDDT77x5dZ+7fHlYrlkDH0UJRKZODaMHzeC11zJWbRFp\nWuoLUF3cfXrVwqisZ0ZqJLnDDG6+OUzdceutcMAB9Z9z0klQWhqyre+2G/z739C/PxxxRNh///0Z\nrbKINB31Baj2dexrnc6KSA7bccfK2//zPyHo1GTGDDj7bHg+Gpw5cWJYfvxxWG6xBXz9dQhktc1P\nJSJC/dnMS8zs1+5+X2qhmZ0FaI6GQtElSvZx2WXwm99Ajx5w2mlw331h9N9338HkySE9EsB//pM8\nd+bMyte6777wgjBR4vOaVkxEalZfgLoIeM7MTiUZkIqBFkANd8ulSUpkPu/XD3r3Duvbb1/5manT\nT08GqNQHeD/8sPbrLlwIjz8epvzYe++0VllE8l+dAcrdFwP7mNlBwK5R8YvurjvdheTii0M3X+p8\nUlW1bFlzeU2pkxIWL07ODKx8fyJSRUNz8b3u7n+NXgpOhaaoCH75S9isnp/L5Mlw441h/ac/rT7n\n1M9/Dr/7XXI7NcffxInJ+1QiIjQ8k4RI/QYOhAsugP32gzvuCEPNITmtR58+cP75yeO//z65fsgh\nYTqQmTNh3DjqNXQoPPFE+uouIjmnQZkk8pEySeSAp5+G226D118PQeekk0KravXqkGLpwANrP3fD\nhvAM1p57QosW4aHhK68M96suuQRaR4NIm+jvV6SQbFQmCZFNcsIJ4QVwxhnJ8jZtYJdd6j73yy9r\nT0a7fn1yfe3a2u9/iUheUxefxKO+mXq32672fb//fXL9008r71u8eOPrJCI5RQFK4vP663DnnTXv\nW726Ydd45RUYMwZGjgz3sbbeuvqzVyKSlxSgJD4HHhgGVTRUapffH/8YlhdfDGeeGXIHJrJWpE4N\nIiJ5SwFK4jd2LBx0UJiefvbsMPDhmmuqZ5lI5PMDOPjg2q+nOapEmgSN4pPcdvHFYUDFfvtB587J\ne1fffRdG96Xq3z9MT79yJZxzTljefrsGUYjkuNpG8akFJbntjjvgrLNCzr8OHZLlm29eObv65puH\nZ6O23Ra++gpGjIC7766e6LYma9fC4YdXziEoIrHTMHPJH2bw0EMwYEDYfuON8KzVgAEhMDVvDiUl\nlQNNTRnT77wztKrOPjtsz5gBr74Kc+ZUHxUoIrFRC0ryy9ChYZ6phBNOCAlsm0f/16op6ezDD0NF\nBTz3XOgqvOii0AWYmCV4ejTl2bx5YXbh1AwXIhIbtaCkaTnlFPjss5Bp4u9/Dy2i00+v+dh//zvk\nGBw7Nln29NNhvqvdd89GbUWkDllvQZlZXzObmvJaaWYXmdkfzGxBSvmRKedcaWalZjbbzA7Pdp0l\nj7RvH6YB+cMf4J//rPmYU0+FZs3CNPYQktYmphGBMAWIiMQu6wHK3We7ez937wfsAawGnot235HY\n5+4vAZjZzsAQYBdgEHC3mTXLdr0lD+2wQ5gccdddK5cfd1xIl3TTTaGFtWxZ5QEXN98cBmasXx9y\nAtZl/Xq44QZYsSL99RcpcHHfgzoYmOvun9VxzLHA4+6+1t3nAaXAwKzUTvLfWWfBlGiuzT/9KTxj\nlcgPCCHDelkZdOxYuVvvgQfCfa3994epU0OQM0sOrEh46aWQeumSSzL/WUQKTNwBagiQOrfCeWY2\nzczGmFlRVNYN+CLlmLKorBozG25mJWZWUlFRkZkaS/5p0SIEptS5qEpKYMiQ5HaHDiEQpc5RBfB/\n/xeer5oxI2zfe2/lQRSJ9WXLMlN3kQIWW4AysxbAMcBTUdE9QB+gH1AO3NbYa7r7aHcvdvfizp07\np62u0gTtsUdoJSUkWkhdusCwYXWf+8Yb1cvWrUtv/UQk1hbUEcD70bTyuPtid1/v7huA+0h24y0A\neqSc1z0qE9k0bdrAN9+EHH5HH50sX7o0LB95pObzDj00dOk9+CAcf3wo++qrMFrwzTczWmWRQhJb\nqiMzexx4xd0fjLa7unt5tH4xsKe7DzGzXYCxhIC1DTAR2MHd19dyaUCpjmQTzJ4N998fslFMngz7\n7tvwc83CwAmzMIX98uVhCpDjjqt8/eLikNR2++3TX3+RPJNTqY7MrC1wKPBsSvGtZjbdzKYBBwEX\nA7j7DOBJYCbwT+C39QUnkU3St28YUNGsWZjJ93//F15+GX71q5q791K5w2abhXN+/ONw/i9+Aeee\nmzzm3nvh66/hmWcy+zlE8lwsAcrdv3H3ju7+VUrZr9x9N3ff3d2PSbSmon03unsfd+/r7i/HUWcp\nYEcdBYMGhS6/Aw4IraKHHw77zGqeMqTqfax77kk+d5UYkt6+febqLNIExD2KTyT/tG8fMlZcey18\n+GFy2HpqxorEfaxUO+8Me+0VhqYDfPtt5f0VFeGaa9dmpNoi+UbTbYhsKveQLun440OKparuvrty\nF1+qVq1gzZqwfsAB8NZb8Npr4b7X999D27aha3HwYPjHP0IWjIMOytxnEYlBTt2DEmlSzELgaNUK\nevSovn/4cPjXv2o+99tvoTzqzf7447BcujR0K26xRcjM/uijYZThAw/Az36WPPe00+ofEi+Sx9SC\nEkmnFStC0Bk4MLyuuQb69Qv71qwJLaGdd4Yttwz7IMx1lQhODfHRR3D11ckZh1evrrnlJpInamtB\nKZu5SDolBj7UNA9V69Ywfnxy+5JL4Ec/alxwAnjhhWRwgtDlN2lS4+sqkuPUxScSl9atw9xUEOa4\nWr06JLhNOOSQysefdFJYXnFF5fLJk0OqpUWLwszCDz6YuTqLZJEClEicbrwRnn025AZs3Ro++SQ8\nyDtiRLj3lPDEE2F+q7POCttnnBGmFUno2BG6dg0pl4YNC/eq5szJ7mcRSTPdgxLJZWZh+c03ITXT\n2rXw+utw+OFh32efQc+etZ8/bVrlGYghBLHm6t2X3KFRfCL56L77wr2qNm3CdsuW4aHhRODq0SME\nqwEDYOTIMEx9jz2S5ydaYe+9F6YWMQstNU0PInlALSiRpubbb8NIwXnzoKgoDMIYPDhMcZ9q7NjQ\nFbj11mH7+ONh1KiQ0V0ki9SCEikUrVqFmYI//DCkZerSJQSnZtFE1GPHhjmyrrqqcsb2Z55JBqv1\n60N6plWrwnYT/Y+s5DYFKJGmavfd4fLLk9uffw6vvgonnxwe+p0/Hy67rPp5f/xjSHZ77rkwdGjo\nFtxsszB4QySL1MUn0tTNnRuWffokyxYuhN69G5/3r6wsDGXfaqu6j/v8c9h228plCxbAhRfCmDHQ\nrl3j3leaNHXxiRSqPn0qByeAbbYJyWk3bAij+pYtC914n31W8zX22y8su3cPXYZPPRWev/rww+Qx\nixeHbBk33gjbbRf2p7rlltCNmMgEL1IPjTUVKVRbbhmWzZqFwRQQWj3Ll4dWzimnhHtUPXrAkCEh\nH+CLL4bjEg8NQwhYgwaFwLPHHjBlSij/z39gzz2TxyXSMX35ZWY/lzQZ6uITkYZZtQpuvjkMwrj2\n2vqP/+Uvw8PFCddcE1pXEO5/bbddRqop+Ue5+ERk02y5Jdx0U7JLsFevkOj29NNDmqVUP/1peAbr\n0UdDy6q0NAxhTxg2LGzvtFPN77Vype5TiVpQIpIGL78c7nNNmRIyVyxfHua3quq005JD23v2DOmY\nqma1uOmmkK194cKQvkmaPLWgRCRzjjgiLHfcMVm2ahWceSY8+WTYXrIEOneGvn1DAJo/P4wIHDYs\npGQ66KCwfvXV4fjjjgsPEu+yS8gSf//9YaBF375Z/WgSH7WgRCSzli4NXYKpwWv5cujWLTmbcGP1\n7g1/+xscemh66iixyrlh5mY238ymm9lUMyuJyjqY2QQzmxMti6JyM7ORZlZqZtPMbEBc9RaRRurU\nqXJwgjBqsKwsJL7t2BEOOyy57/jjYe+94fzzK59z7LHJ9U8/DecsXBiGt48YEVpoK1Yo60UTElsL\nyszmA8XuvjSl7FZgmbuPMLMrgCJ3v9zMjgTOB44E9gTudPc9a7puglpQInnCPWSrmDkzJMcdMSIk\nxU2YOTOMHOzdOwxzP+qohl33xRdDEGvePASv998Pw+El59TWgsq1ADUbONDdy82sK/Bvd+9rZvdG\n6+OqHlfb9RWgRJqoDz4I96R6927Y8bvtBtOnJ7fnzk2eu2FDSOP0ySfhuoMHp7++Uq9cHCThwKtm\n5sC97j4a6JISdBYBibTK3YAvUs4ti8oqBSgzGw4MB9i2apoVEWka+vcPy8QDv2PGhBbXTjvBFlvA\nCy+EkYAJ8+ZVPr9PnxC0+vSBN94Isxi/+27YN2BAyAa/224hT2GzZuFaielNJKvibEF1c/cFZrYV\nMIHQhTfe3dunHLPc3YvM7AVghLu/HZVPBC5391qbSGpBiRSwd9+F0aPh7rtDF9+6daF81KiNmwvr\niivgnHOq5xeUtMi5QRLuviBaLgGeAwYCi6OuPaLlkujwBUCPlNO7R2UiItUNHBiGpbdoEbrwWrQI\nr4svDve8xo8PQaehGdpHjAiZL8yguDjcFxszBr7/vvJxDz8cEuKuX5/+z1SAYmlBmVlbYDN3XxWt\nTwCuBw4GvkwZJNHB3S8zs58D55EcJDHS3QfW9R5qQYlIgzzySJhW5O67YcYM2H//MH/We++FiR9T\n0zXVZNddoW1buOACOPXUZPmkSZVzEZaUhIeTO3XKxKfIazk1SMLMehNaTRDug4119xvNrCPwJLAt\n8BlwkrsvMzMD7gIGAauBMzbnu4IAAAqrSURBVOrq3gMFKBFphMRgibq4h67DBQvCUPiGuO66EJDe\neSekfQLYZ5/Q/di7dwhiBx20aXVvAnIqQGWDApSIZMzTT4eh60cfHZ692msvWL06jAKsqIDXXgsZ\nLxrahdi8eQiAZ5wR7pcVFYXnu845JznVSROmACUikg3r14cW2fffh0D19ddw++3wl7+EUYKlpWGU\nYENtthmceGK4r9ahQ5j+5KWXQp7CSy9tEiMMFaBERHLFhg0hzdO778Ixx4Quw++/h7Fjw/5LL4Un\nngjZNuqz5ZZhAEinTiF/4aGHhgebH3oIfvc7aNMmBLUcpgAlIpIP1q8Pz18BvPkmvPVWaIW9+mrI\nhrGxjjkmTGFy/fXw8cchl+Fxx4UBIpMmhfRSMbXGFKBERPLZhg3h4eTOncO2O3z0UQhoEyaEGYxX\nrgyzHZ9zTuOv37176Ho88siQJuquu0I+xKKiMGS/vDwk+G3ePNSlefO0BTQFKBGRQjF/fuj6e+ed\nMFhj7NiQUWPcONh99zDk/brrQuaN2bPDqMKN8eMfhwefzzprk6qrACUiIjVbuDA8A/bll3DrrSEv\n4bnnwrPPhlZSea1pT0OQmjFjk1pTClAiIrLxVq8OAy7cw1D68vIwFH7//aF16026dC4mixURkXzR\npk1YmsFWW4VXhsWWi09ERKQuClAiIpKTFKBERCQnKUCJiEhOUoASEZGcpAAlIiI5SQFKRERykgKU\niIjkJAUoERHJSQpQIiKSkxSgREQkJylAiYhITsp6gDKzHmb2upnNNLMZZnZhVP4HM1tgZlOj15Ep\n51xpZqVmNtvMDs92nUVEJPviyGa+DrjU3d83sy2BKWY2Idp3h7v/OfVgM9sZGALsAmwD/MvMdnT3\n9VmttYiIZFXWW1DuXu7u70frq4BZQLc6TjkWeNzd17r7PKAUGJj5moqISJxivQdlZj2B/sDkqOg8\nM5tmZmPMrCgq6wZ8kXJaGbUENDMbbmYlZlZSUVGRoVqLiEg2xBagzGwL4BngIndfCdwD9AH6AeXA\nbY29pruPdvdidy/u3LlzWusrIiLZFUuAMrPNCcHpMXd/FsDdF7v7enffANxHshtvAdAj5fTuUZmI\niDRhcYziM+ABYJa7355S3jXlsF8AH0Xr44EhZtbSzHoBOwDvZqu+IiISjzhG8e0L/AqYbmZTo7Kr\ngJPNrB/gwHzgNwDuPsPMngRmEkYA/lYj+EREmr6sByh3fxuwGna9VMc5NwI3ZqxSIiKSc5RJQkRE\ncpIClIiI5CQFKBERyUkKUCIikpMUoEREJCcpQImISE5SgBIRkZykACUiIjlJAUpERHKSApSIiOQk\nBSgREclJClAiIpKTFKBERCQnKUCJiEhOUoASEZGcpAAlIiI5SQFKRERykgKUiIjkJAUoERHJSQpQ\nIiKSk/ImQJnZIDObbWalZnZF3PUREZHMyosAZWbNgFHAEcDOwMlmtnO8tRIRkUzKiwAFDARK3f1T\nd/8OeBw4NuY6iYhIBjWPuwIN1A34ImW7DNiz6kFmNhwYHm1+bWazN+E9OwFLN+H8pkLfQ5K+i0Df\nQ6DvIUjH97BdTYX5EqAaxN1HA6PTcS0zK3H34nRcK5/pe0jSdxHoewj0PQSZ/B7ypYtvAdAjZbt7\nVCYiIk1UvgSo94AdzKyXmbUAhgDjY66TiIhkUF508bn7OjM7D3gFaAaMcfcZGX7btHQVNgH6HpL0\nXQT6HgJ9D0HGvgdz90xdW0REZKPlSxefiIgUGAUoERHJSQpQNSiktEpm1sPMXjezmWY2w8wujMo7\nmNkEM5sTLYuicjOzkdF3M83MBsT7CdLLzJqZ2Qdm9kK03cvMJkef94lokA5m1jLaLo3294yz3ulk\nZu3N7Gkz+9jMZpnZ3oX4ezCzi6O/Ex+Z2Tgza1UovwczG2NmS8zso5SyRv8GzGxodPwcMxva2Hoo\nQFVRgGmV1gGXuvvOwF7Ab6PPewUw0d13ACZG2xC+lx2i13DgnuxXOaMuBGalbN8C3OHu2wPLgTOj\n8jOB5VH5HdFxTcWdwD/dfSfgJ4Tvo6B+D2bWDbgAKHb3XQmDs4ZQOL+Hh4BBVcoa9Rswsw7AtYSk\nCgOBaxNBrcHcXa+UF7A38ErK9pXAlXHXK4uf/3ngUGA20DUq6wrMjtbvBU5OOf6H4/L9RXi+biLw\nM+AFwAhPyDev+tsgjCjdO1pvHh1ncX+GNHwHPwLmVf0shfZ7IJm9pkP05/sCcHgh/R6AnsBHG/sb\nAE4G7k0pr3RcQ15qQVVXU1qlbjHVJauibon+wGSgi7uXR7sWAV2i9ab8/fwFuAzYEG13BFa4+7po\nO/Wz/vA9RPu/io7Pd72ACuDBqKvzfjNrS4H9Htx9AfBn4HOgnPDnO4XC+z2kauxvYJN/GwpQAoCZ\nbQE8A1zk7itT93n470+Tfh7BzI4Clrj7lLjrErPmwADgHnfvD3xDsisHKJjfQxEhIXUvYBugLdW7\nvApWtn4DClDVFVxaJTPbnBCcHnP3Z6PixWbWNdrfFVgSlTfV72df4Bgzm0/Ilv8zwr2Y9maWeKA9\n9bP+8D1E+38EfJnNCmdIGVDm7pOj7acJAavQfg+HAPPcvcLdvweeJfxGCu33kKqxv4FN/m0oQFVX\nUGmVzMyAB4BZ7n57yq7xQGLUzVDCvalE+WnRyJ29gK9Smv15y92vdPfu7t6T8Gf+mrufCrwOnBAd\nVvV7SHw/J0TH532rwt0XAV+YWd+o6GBgJgX2eyB07e1lZm2ivyOJ76Ggfg9VNPY38ApwmJkVRS3S\nw6Kyhov7RlwuvoAjgU+AucDVcdcnw591P0JTfRowNXodSeg/nwjMAf4FdIiON8Iox7nAdMIop9g/\nR5q/kwOBF6L13sC7QCnwFNAyKm8VbZdG+3vHXe80fv5+QEn0m/gHUFSIvwfgOuBj4CPg70DLQvk9\nAOMI996+J7Sqz9yY3wAwLPpOSoEzGlsPpToSEZGcpC4+ERHJSQpQIiKSkxSgREQkJylAiYhITlKA\nEhGRnKQAJZIlZrbezKamvNKWKd/MeqZmnhZpCvJiyneRJmKNu/eLuxIi+UItKJGYmdl8M7vVzKab\n2btmtn1U3tPMXovm2JloZttG5V3M7Dkz+zB67RNdqpmZ3RfNYfSqmbWOjr/Awnxf08zs8Zg+pkij\nKUCJZE/rKl18g1P2feXuuwF3EbKqA/wVeNjddwceA0ZG5SOBN9z9J4Q8eTOi8h2AUe6+C7ACOD4q\nvwLoH13n7Ex9OJF0UyYJkSwxs6/dfYsayucDP3P3T6PEvYvcvaOZLSXMv/N9VF7u7p3MrALo7u5r\nU67RE5jgYTI5zOxyYHN3/6OZ/RP4mpC26B/u/nWGP6pIWqgFJZIbvJb1xlibsr6e5D3mnxNypQ0A\n3kvJxi2S0xSgRHLD4JTlO9H6/xEyqwOcCrwVrU8EzgEws2Zm9qPaLmpmmwE93P114HLCNBDVWnEi\nuUj/kxLJntZmNjVl+5/unhhqXmRm0witoJOjsvMJM9v+P8Ist2dE5RcCo83sTEJL6RxC5umaNAMe\njYKYASPdfUXaPpFIBukelEjMontQxe6+NO66iOQSdfGJiEhOUgtKRERyklpQIiKSkxSgREQkJylA\niYhITlKAEhGRnKQAJSIiOen/Az/FazlNtcqCAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BklRgcB4XJoH",
        "colab_type": "text"
      },
      "source": [
        "showing the training and test accuracy"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qjWJX9H-XJoI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "61f15381-9e74-4ac0-f541-b7b79407a908"
      },
      "source": [
        "y_train_pred = nn.predict(X_train)\n",
        "\n",
        "if sys.version_info < (3, 0):\n",
        "    acc = ((np.sum(y_train == y_train_pred, axis=0)).astype('float') /\n",
        "           X_train.shape[0])\n",
        "else:\n",
        "    acc = np.sum(y_train == y_train_pred, axis=0) / X_train.shape[0]\n",
        "\n",
        "print('Training accuracy: %.2f%%' % (acc * 100))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training accuracy: 97.22%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PFncZWqqXJoL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "babe7991-8021-4043-9bc7-401d4bcc08e6"
      },
      "source": [
        "y_test_pred = nn.predict(X_test)\n",
        "\n",
        "if sys.version_info < (3, 0):\n",
        "    acc = ((np.sum(y_test == y_test_pred, axis=0)).astype('float') /\n",
        "           X_test.shape[0])\n",
        "else:\n",
        "    acc = np.sum(y_test == y_test_pred, axis=0) / X_test.shape[0]\n",
        "\n",
        "print('Test accuracy: %.2f%%' % (acc * 100))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test accuracy: 95.82%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B7wm9S2gXJoO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}