DigitClassificationNet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# necessary libraries\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import sys"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparation of training and validation datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The data was taken from <a href=\"https://www.kaggle.com/c/digit-recognizer/data\">kaggle competition</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42000, 785)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = np.genfromtxt('mnist/train.csv', delimiter=',', skip_header=1)\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_size = 32000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32000, 784)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train = data[:train_size, 1:]\n",
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32000, 1)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train = data[:train_size, :1]\n",
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 784)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_validation = data[train_size:, 1:]\n",
    "x_validation.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 1)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_validation = data[train_size:, :1]\n",
    "y_validation.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3.])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAOTUlEQVR4nO3de4xc9XnG8edhWZtgA8EYiGsggHEbUKRAs4FSQoHSpEClQCQoQQ1yKypHKkig0CqUtAqKUsmtmqC0TVA2gOKmBIIECEtF3Cxaitoglkuw8QYMFhdjyy5xwObmy+7bP3ZIF7Pnt+uZM3MGv9+PtJqZ887Z82rsZ8/M/M45P0eEAOz99mm6AQC9QdiBJAg7kARhB5Ig7EAS+/ZyY7M8O/bTnF5uEkjlXb2lHbHdU9U6CrvtcyR9V9KApBsjYlnp+ftpjk7x2Z1sEkDBo7Gystb223jbA5K+J+lcSSdIusT2Ce3+PgDd1cln9pMlPR8R6yJih6TbJJ1fT1sA6tZJ2BdKemXS4/WtZe9je6ntEdsjO7W9g80B6EQnYZ/qS4APHHsbEcMRMRQRQ4Oa3cHmAHSik7Cvl3TkpMdHSNrQWTsAuqWTsD8mabHtY2zPkvQlSSvqaQtA3doeeouIXbavkHSfJobebo6IZ2rrDECtOhpnj4h7JN1TUy8AuojDZYEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IoqdTNu+tPLs80807n/tUsf7yH3W4/f13VdbW/sGNxXUHXP57f9XGoWL9vhUnF+vHDq+rrI2/+VZx3fFt24p17Bn27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQhCOiZxs70PPiFJ/ds+3Vad9jj66sPfutjxbXHT3jppq72Tsc/5MrivVFf/U/Pepk7/ForNTW2OKpah0dVGP7RUnbJI1J2hUR5SMwADSmjiPozoqI12r4PQC6iM/sQBKdhj0k3W/7cdtLp3qC7aW2R2yP7NT2DjcHoF2dvo0/LSI22D5M0gO2fxERD09+QkQMSxqWJr6g63B7ANrU0Z49Ija0bjdLuktS+RQoAI1pO+y259g+4L37kj4vaXVdjQGoVydv4w+XdJft937PTyLi3lq66kNr/vKwytp3P/NvxXU3jb1TrB8+8JFi/W82f7pY3zVe/Td7dOvHiuu++sZBxfrlv/WfxfqfHfhKsV7yF+feV6x//zfOKNYX/cmTbW87o7bDHhHrJJWvygCgbzD0BiRB2IEkCDuQBGEHkiDsQBKc4lqDgeMXF+vPXju3WD/kwf2K9Xm3PFasx67qS0l3at8jFhbro399RLH+7AXfb3vb//52eVjwhsXHtf2791alU1zZswNJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEkzZXIOx0bXF+nGXdvb7m7y8T+xfPgZgyWf/q0edoFPs2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZUfTaqdWX0Jaka+ff3qNO0Cn27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBOPsyXn27GL9nUOnvAT5rz25Y7xYP2kW+5N+Me2/hO2bbW+2vXrSsnm2H7C9tnV7cHfbBNCpmfzZ/ZGkc3Zbdo2klRGxWNLK1mMAfWzasEfEw5K27Lb4fEnLW/eXS7qg5r4A1KzdD1SHR8RGSWrdVh5AbXup7RHbIzu1vc3NAehU1789iYjhiBiKiKFBlb8MAtA97YZ9k+0FktS63VxfSwC6od2wr5C0pHV/iaS762kHQLdMO85u+1ZJZ0qab3u9pG9IWibpdtuXSXpZ0kXdbBJlA/MPqayNLjumuO63Tr+rWB+LF4r1WSqPs3fySfGEWZuK9XXLyhfkP+6bP6+sjb/9dls9fZhNG/aIuKSidHbNvQDoIg5vApIg7EAShB1IgrADSRB2IAlOcd0L+IC5lbXnzv1Bl7de/i/09I6xytrOGCiu++nZ5emi11z6L8X6xb+7+/lb/+/1bx5fXHfwwceL9Q8j9uxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7HuB8c2vVdY+8dCfF9f9/cXP1d3O+7zw9U9U1ma9saO47obTDyjWH7/6n4v1ny66t7J2+lcvLq570IPF8ocSe3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9r3A+FtvVdaO+/KTxXVfrruZ3Qyq+rzwmGbddy48td5mkmPPDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJMM6Oxuz4w6Fi/Y6Lr5/mNwzW10wC0+7Zbd9se7Pt1ZOWXWf7VdtPtX7O626bADo1k7fxP5I01dQa10fEia2fe+ptC0Ddpg17RDwsaUsPegHQRZ18QXeF7adbb/MPrnqS7aW2R2yP7NT2DjYHoBPthv0GSYsknShpo6RvVz0xIoYjYigihgY1u83NAehUW2GPiE0RMRYR45J+KOnketsCULe2wm57waSHX5S0uuq5APrDtOPstm+VdKak+bbXS/qGpDNtn6iJU5JflPSVLvaIvdRL55X/+x0/yDh6naYNe0RcMsXim7rQC4Au4nBZIAnCDiRB2IEkCDuQBGEHkuAUVxR5cFaxvs/cOcX681+rnrL5rFNWtdXTTA2/cXRlbd5V5QtZj9XcSz9gzw4kQdiBJAg7kARhB5Ig7EAShB1IgrADSTDOntw+++9frD9/428W62vOmO4EyAf3sKOZ+97ri4r1+y/8TGVt7Lm1dbfT99izA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASjLPP0D6frD4v+xeXH1hcd8F/lP+mHnT3U8X6+LvvFusDi4+trG391KHFdT925QvF+ppjm7uQ8JM7xov1+y8qz00yNvpcne186LFnB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkGGdvGTjumGL9mhW3VdZOnT3NVca/UC5fdtVZxfrrOz5arC9ZUH3O+Bfm/Kq88S47c9VFlbUlH/9Zcd3rf3pBsX7Umv9uq6espt2z2z7S9kO2R20/Y/vK1vJ5th+wvbZ1e3D32wXQrpm8jd8l6eqIOF7S70i63PYJkq6RtDIiFkta2XoMoE9NG/aI2BgRT7Tub5M0KmmhpPMlLW89bbmk8nsuAI3aoy/obB8t6SRJj0o6PCI2ShN/ECQdVrHOUtsjtkd2antn3QJo24zDbnuupDskXRURW2e6XkQMR8RQRAwNanY7PQKowYzCbntQE0G/JSLubC3eZHtBq75A0ubutAigDtMOvdm2pJskjUbEdyaVVkhaImlZ6/burnTYIzH3I8X6mncXVtZOnf1yR9u+6aiHOlq/n+3/dwdV1u5+tXyK6lHrGFqr00zG2U+TdKmkVbbfO/H6Wk2E/Hbbl0l6WVL1gCqAxk0b9oh4RJIrymfX2w6AbuFwWSAJwg4kQdiBJAg7kARhB5LgFNeWGC1fUnn4n6rPUz30q7cU1+32aaabxt6prJ35yBXFdf9+6M5ifTp/++MvF+tH/WyksrZr546Oto09w54dSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JwRPRsYwd6XpziD+eJcm9deEplbb9f7iyuO3btL4v1lzYeUqzPX1m+ws/8e6uPERjbVL6myMDBnV0UeOxXzV6qGu/3aKzU1tgy5Vmq7NmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2YG9COPsAAg7kAVhB5Ig7EAShB1IgrADSRB2IIlpw277SNsP2R61/YztK1vLr7P9qu2nWj/ndb9dAO2aySQRuyRdHRFP2D5A0uO2H2jVro+If+xeewDqMpP52TdK2ti6v832qKSF3W4MQL326DO77aMlnSTp0daiK2w/bftm21Ne38j2Utsjtkd2antHzQJo34zDbnuupDskXRURWyXdIGmRpBM1sef/9lTrRcRwRAxFxNCgytdSA9A9Mwq77UFNBP2WiLhTkiJiU0SMRcS4pB9KOrl7bQLo1Ey+jbekmySNRsR3Ji1fMOlpX5S0uv72ANRlJt/GnybpUkmrbD/VWnatpEtsnygpJL0o6Std6RBALWbybfwjkqY6P/ae+tsB0C0cQQckQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiip1M22/5fSS9NWjRf0ms9a2DP9Gtv/dqXRG/tqrO3j0fEoVMVehr2D2zcHomIocYaKOjX3vq1L4ne2tWr3ngbDyRB2IEkmg77cMPbL+nX3vq1L4ne2tWT3hr9zA6gd5reswPoEcIOJNFI2G2fY/tZ28/bvqaJHqrYftH2qtY01CMN93Kz7c22V09aNs/2A7bXtm6nnGOvod76YhrvwjTjjb52TU9/3vPP7LYHJD0n6XOS1kt6TNIlEbGmp41UsP2ipKGIaPwADNu/J+lNSf8aEZ9sLfsHSVsiYlnrD+XBEfG1PuntOklvNj2Nd2u2ogWTpxmXdIGkP1WDr12hrz9WD163JvbsJ0t6PiLWRcQOSbdJOr+BPvpeRDwsactui8+XtLx1f7km/rP0XEVvfSEiNkbEE6372yS9N814o69doa+eaCLsCyW9MunxevXXfO8h6X7bj9te2nQzUzg8IjZKE/95JB3WcD+7m3Ya717abZrxvnnt2pn+vFNNhH2qqaT6afzvtIj4bUnnSrq89XYVMzOjabx7ZYppxvtCu9Ofd6qJsK+XdOSkx0dI2tBAH1OKiA2t282S7lL/TUW96b0ZdFu3mxvu59f6aRrvqaYZVx+8dk1Of95E2B+TtNj2MbZnSfqSpBUN9PEBtue0vjiR7TmSPq/+m4p6haQlrftLJN3dYC/v0y/TeFdNM66GX7vGpz+PiJ7/SDpPE9/IvyDp6030UNHXsZJ+3vp5puneJN2qibd1OzXxjugySYdIWilpbet2Xh/19mNJqyQ9rYlgLWiot89q4qPh05Keav2c1/RrV+irJ68bh8sCSXAEHZAEYQeSIOxAEoQdSIKwA0kQdiAJwg4k8X+cnD3H9Vtb1gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Image size 28x28\n",
    "plt.imshow(np.array([x_train[9][i-28:i] for i in range(28,785, 28)]));\n",
    "y_train[9]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Activation function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperbolic tangent\n",
    "def tanh(x):\n",
    "    return (math.e**x - math.e**-x) / (math.e**x + math.e**-x)\n",
    "\n",
    "# Derivative of hyperbolic tangent\n",
    "def d_tanh(x):\n",
    "    return 1 - tanh(x)**2\n",
    "\n",
    "# Softmax in output layer\n",
    "def softmax(x):\n",
    "    exps = np.exp(x - np.max(x))\n",
    "    return exps / np.sum(exps, axis=1)[:, np.newaxis]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DigitRecognizer:\n",
    "    def __init__(self, n_hidden_neurons):\n",
    "        self.n_hidden_neurons = n_hidden_neurons\n",
    "        \n",
    "        self.layer1 = np.random.random((n_hidden_neurons, 784)) - 0.5\n",
    "        self.b1 = np.random.random((1, n_hidden_neurons)) - 0.5\n",
    "        \n",
    "        self.act1 = tanh\n",
    "        self.d_act1 = d_tanh\n",
    "        \n",
    "        self.layer2 = np.random.random((10, n_hidden_neurons)) - 0.5\n",
    "        self.b2 = np.random.random((1, 10)) - 0.5\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = x / 255\n",
    "        # (n_examples, n_hidden_neurons) = (n_examples, 784) @ (n_hidden_neurons, 784).T\n",
    "        x = x @ self.layer1.T\n",
    "        # (n_examples, n_hidden_neurons) += (1, n_hidden_neurons)\n",
    "        x += self.b1\n",
    "        self.sum1 = x\n",
    "        # (n_examples, n_hidden_neurons)\n",
    "        x = self.act1(x)\n",
    "        # (n_examples, 10) = (n_examples, n_hidden_neurons) @ (10, n_hidden_neurons).T\n",
    "        x = x @ self.layer2.T\n",
    "        # (n_examples, 10) += (1, 10)\n",
    "        x += self.b2\n",
    "        self.sum2 = x\n",
    "        # (n_examples, 10)\n",
    "        return softmax(x)\n",
    "    \n",
    "n_neurons = 100 # You can change number of neurnons\n",
    "net = DigitRecognizer(n_neurons) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def accuracy(x, y):\n",
    "    return np.sum(net.forward(x).argmax(axis=1)[:,np.newaxis] == y) / len(y)\n",
    "accuracy(x_validation, y_validation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cross entropy loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy(y_pred, y):\n",
    "    m = y.shape[0]\n",
    "    log_likelihood = -np.log(y_pred[range(m),y.reshape(m).astype(int)])\n",
    "    loss = np.sum(log_likelihood) / m\n",
    "    return loss\n",
    "\n",
    "# derivative of cross entropy with softmax\n",
    "def d_ce(y_pred, y):\n",
    "    ans = np.copy(y_pred)\n",
    "    ans[range(len(y)), y.reshape(len(y)).astype(int)] = -1 + y_pred[range(len(y)), y.reshape(len(y)).astype(int)]\n",
    "    return ans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.096144850461014"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_entropy(net.forward(x_validation), y_validation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learning process (back propagation algorithm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epoch = 50\n",
    "batch_size = 500\n",
    "lr = 0.001\n",
    "\n",
    "ce_hist = np.zeros(n_epoch) # history of accuracy changes\n",
    "ac_hist = np.zeros(n_epoch) # history of cross-entropy changes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(net, x, y, lr):\n",
    "    # (n_examples, 10)\n",
    "    y_pred = net.forward(x)\n",
    "    \n",
    "    # (n_examples, 10)\n",
    "    error_2 = d_ce(y_pred, y)\n",
    "    # (1, 10) -= () * (1, 10)\n",
    "    net.b2 -= lr * np.sum(error_2, axis=0)[np.newaxis,]\n",
    "    # (10, n_neurons)-=()*(n_examples, 10).T @ (n_examples, n_neurons)\n",
    "    net.layer2 -= lr * error_2.T @ net.act1(net.sum1)\n",
    "    \n",
    "    # (n_examples, n_neurons) = (n_examples, 10) @ (10, n_neurons) * (n_examples, n_neurons)\n",
    "    error_1 = error_2 @ net.layer2 * net.d_act1(net.sum1)\n",
    "    # (1, n_neurons)-=()*(1, n_neurons)\n",
    "    net.b1 -= lr * np.sum(error_1 ,axis=0)[np.newaxis,]\n",
    "    # (n_neurons, 784) -= lr * (n_examples, n_neurons).T @ (n_examples, 784)\n",
    "    net.layer1 -= lr * error_1.T @ (x / 255)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 50 Accuracy: 0.9539 CE: 0.1513339001530272     "
     ]
    }
   ],
   "source": [
    "for i in range(n_epoch):\n",
    "    for j in range(batch_size, train_size + 1, batch_size):\n",
    "        train(net, x_train[j - batch_size: j], y_train[j - batch_size: j], lr)\n",
    "    \n",
    "    ce_hist[i] = cross_entropy(net.forward(x_validation), y_validation)\n",
    "    ac_hist[i] = accuracy(x_validation, y_validation)\n",
    "    \n",
    "    sys.stdout.write(\"\\r\" + 'Epoch: '+str(i + 1)+' Accuracy: '+str(ac_hist[i])+' CE: '+str(ce_hist[i])+'    ')\n",
    "    sys.stdout.flush()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross-entropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxddZ3/8dfn3uxptrZpuqRtWlqWAt2IyCaIClJ2FARkBndkfjCiMy7oLI7bjI46gyIOIuA2AjIiWgFBBAQRBVIoLaUUSteQLumSJk2a/fP7456U2/Q2PSk5ucm97+fjcR/3nO895+Zz+mj7zjnfc75fc3dERET6i6W7ABERGZkUECIikpICQkREUlJAiIhISgoIERFJKSfdBQyl8ePHe01NTbrLEBEZNZYsWbLN3StTfZZRAVFTU0NdXV26yxARGTXMbP2BPtMlJhERSUkBISIiKSkgREQkJQWEiIikpIAQEZGUFBAiIpKSAkJERFLK+oDo7XW+9+irPPFKY7pLEREZUbI+IGIx4wdPrOHRl7emuxQRkREl6wMCoKq0gM272tNdhojIiKKAACaWFrClRQEhIpIs0oAws7PMbJWZrTaz6wfY7i1m1mNmFye1rTOz5Wa21MwiHWBpQmk+W3QGISKyj8gG6zOzOHATcAZQDzxrZovd/aUU230DeCjF15zu7tuiqrHPxNICtrZ00NvrxGIW9Y8TERkVojyDOB5Y7e5r3L0TuAu4IMV2fw/cA6Stl7iqtIDuXmd7a2e6ShARGXGiDIgpwMak9fqgbS8zmwJcBNycYn8Hfm9mS8zsqgP9EDO7yszqzKyusfHQblWtKi0AYEuzLjOJiPSJMiBSXavxfus3AJ9z954U257s7guBRcA1ZnZqqh/i7re4e62711ZWppzz4qCqSvMB2KqOahGRvaKcMKgemJq0Xg009NumFrjLzADGA2ebWbe7/9rdGwDcfauZ3UviktUTURTadwaxeVdHFF8vIjIqRXkG8Sww28xmmFkecBmwOHkDd5/h7jXuXgP8Evh/7v5rMys2sxIAMysGzgRejKrQypJ8zHSJSUQkWWRnEO7ebWbXkrg7KQ7c7u4rzOzq4PNU/Q59qoB7gzOLHOAOd38wqlpz4zHGFecrIEREkkQ6J7W7PwA80K8tZTC4+weTltcA86Ksrb+JZQoIEZFkepI6UFVSwOZm9UGIiPRRQASqygrYqjMIEZG9FBCBqpICtrd20tGd6o5bEZHso4AITCxLPAvR2KLLTCIioIDYa4KephYR2YcCIjBxb0DoDEJEBBQQe73xNLXOIEREQAGxV0VRLnnxmCYOEhEJKCACZsaE0ny26hKTiAiggNiH5qYWEXmDAiKJ5qYWEXmDAiKJ5qYWEXmDAiLJxNICWjt72N3Rne5SRETSTgGRRLe6ioi8QQGRpC8gNGifiIgCYh99c1NvVkCIiCggklVpuA0Rkb0UEEmK83Moyc/RgH0iIigg9jOhVFOPioiAAmI/E8sKFBAiIigg9lNVUqA+CBERFBD7qSorYGtLO729nu5SRETSSgHRT1VJPl09zo62znSXIiKSVpEGhJmdZWarzGy1mV0/wHZvMbMeM7t4sPsOtYllmnpURAQiDAgziwM3AYuAOcDlZjbnANt9A3hosPtGQXNTi4gkRHkGcTyw2t3XuHsncBdwQYrt/h64B9h6CPsOOc1NLSKSEGVATAE2Jq3XB217mdkU4CLg5sHuG5XKknzMNGCfiEiUAWEp2vrfGnQD8Dl37zmEfRMbml1lZnVmVtfY2HgIZe4rNx5jXHE+WzVxkIhkuZwIv7semJq0Xg009NumFrjLzADGA2ebWXfIfQFw91uAWwBqa2uH5N7UqtJ8nUGISNaLMiCeBWab2QzgdeAy4P3JG7j7jL5lM/sxcJ+7/9rMcg62b5Q0N7WISISXmNy9G7iWxN1JK4G73X2FmV1tZlcfyr5R1dpfVamG2xARifIMAnd/AHigX1v/Dum+9g8ebN/hUlWaz/bWTjq7e8nL0bOEIpKd9L9fCn23ujbu1q2uIpK9FBApaG5qEREFREqam1pERAGRkuamFhFRQKQ0tjiP3LhpuA0RyWoKiBTMjAklutVVRLKbAuIANPWoiGS7UAFhZoVmdkTUxYwkVaX56oMQkax20IAws/OApcCDwfp8M1scdWHpNqGkgK3qgxCRLBbmDOLfSMzP0ATg7kuBmuhKGhkmlhWwu6Ob3R3d6S5FRCQtwgREt7vvirySEabvVlf1Q4hItgoTEC+a2fuBuJnNNrMbgacirivtqjT1qIhkuTAB8ffA0UAHcCfQDHwyyqJGAgWEiGS7g47m6u5twD8Fr6xRpbmpRSTLHTQgzOwxUkz36e7viKSiEWJMfg5j8nM0YJ+IZK0w80F8Omm5AHgvkBW39lSVam5qEcleYS4xLenX9GczezyiekaUKRVFrNvWlu4yRETSIsyDcmOTXuPN7N3AxGGoLe3mVZexaksLbZ1ZccIkIrKPMJeYlpDogzASl5bWAh+JsqiRYsG0cnp6neX1u3jrzHHpLkdEZFiFucQ0YzgKGYkWTK0A4LkNTQoIEck6BwwIM3vPQDu6+6+GvpyRpaI4jxnji3l+w850lyIiMuwGOoM4b4DPHMj4gABYMLWcP63ehrtjZukuR0Rk2BwwINz9Q8NZyEi1YHoFv3r+dep37mHq2KJ0lyMiMmzCdFJjZueQGG6joK/N3b8cYr+zgO8AceBWd/96v88vAL4C9JLoAP+kuz8ZfLYOaAF6SAwYWBum1qG2YGo5AM9vbFJAiEhWCXOb683ApSTGZDLgEmB6iP3iwE3AImAOcLmZzem32SPAPHefD3wYuLXf56e7+/x0hQPAkRNLKMyNqx9CRLJOmMH6TnL3K4Gd7v4l4ERgaoj9jgdWu/sad+8E7gIuSN7A3Xe7e98wHsWkGNIj3XLiMeZWl/HchqZ0lyIiMqzCBMSe4L3NzCYDXUCYW1+nABuT1uuDtn2Y2UVm9jJwP4mziD4O/N7MlpjZVSF+XmQWTKvgpYZdtHf1pLMMEZFhFSYg7jOzcuCbwHPAOhLDfh9Mqlt+Ug36d6+7HwlcSKI/os/J7r6QxCWqa8zs1JQ/xOwqM6szs7rGxsYQZQ3egmnldPU4Kxqybt4kEcliBw0Id/+Kuze5+z0k+h6OdPd/DfHd9ex7KaoaaBjg5zwBHGZm44P1huB9K3AviUtWqfa7xd1r3b22srIyRFmDt2Ba0FGty0wikkXCdFK/YGZfMLPD3L1jENOPPgvMNrMZZpYHXAYs7vfdsyx4uMDMFgJ5wHYzKzazkqC9GDgTeDH8YQ2tCSUFVFcUKiBEJKuEuc31fBJ3Md1tZr3AL4C73X3DQDu5e7eZXQs8ROI219vdfYWZXR18fjOJocOvNLMuEn0dl7q7m1kVcG+QHTnAHe7+4KEd4tBYOK2CZ9ftSGcJIiLDyt64iSjExmazgX8BrnD3eGRVHaLa2lqvq6uL5Lt/9Oe1fOm3L/GXz7+DSWWFkfwMEZHhZmZLDvQoQZhOasysxsw+S+JW1SOBzw5hfaPCgmmJgfuW6jKTiGSJMH0QT5MYdykGXOLux7v7tyOvbISZM6mUvJwYz+mBORHJEmH6ID7g7i9HXskIl5cT49gpZeqoFpGsEeY216wPhz4Lppaz/PVddHb3prsUEZHIheqDkISF0yvo6O5l5abmdJciIhI5BcQgvPHAnPohRCTzhemkrjOza8ysYjgKGskmlRUyqayA5zeqH0JEMl+YM4jLgMnAs2Z2l5m9u+/p52y0YFq57mQSkawQppN6tbv/E3A4cAdwO7DBzL5kZmOjLnCkWTC1go079tDY0pHuUkREIhX2Qbm5wLdJjOh6D3Ax0Aw8Gl1pI9PC6eqHEJHscNDnIMxsCdAE3AZc7+59vzo/bWYnR1ncSHT05DJy48bzG5s48+iJ6S5HRCQyYR6Uu8Td16T6wN3fM8T1jHgFuXHmTCrVGYSIZLwwl5h2mdl3zey5YHa375jZuMgrG8EWTq9g6cYm9nRqhjkRyVxhAuIuoJHE0NwXB8u/iLKoke6Mo6po7+rl0Ze3prsUEZHIhAmIscGscmuD11eB8qgLG8neOnMclSX5/PaFA06QJyIy6oUJiMfM7DIziwWv9wH3R13YSBaPGeccO4lHV22lub0r3eWIiEQiTEB8nMTzD53B6y7gH8ysxcyydlCi8+dPprO7l4dXbEl3KSIikQjzoFyJu8fcPSd4xYK2EncvHY4iR6IFU8uprihksS4ziUiGCvug3Plm9q3gdW7URY0GZsZ58ybz5Opt7GjtTHc5IiJDLsxgfV8HrgNeCl7XBW1Z77y5k+npdR5YvindpYiIDLkwZxBnA2e4++3ufjtwVtCW9Y6aVMKsCWN0N5OIZKSw80Ek39ZaFkUho5GZcd7cyTyzbgebd7WnuxwRkSEVJiD+HXjezH5sZj8BlgRtApw3bxLucN8ynUWISGYZMCDMLAb0AicAvwpeJ7r7XWG+3MzOMrNVZrbazK5P8fkFZrbMzJYGExOdEnbfkWJm5RiOmVLKb5epH0JEMsuAAeHuvcC17r7J3Re7+2/cfXOYLzazOHATsAiYA1xuZnP6bfYIMM/d5wMfBm4dxL4jxvnzJvPCxibWb29NdykiIkMmzCWmh83s02Y21czG9r1C7Hc8sNrd17h73wN2FyRv4O673d2D1WLAw+47kpwzdzIA9+ksQkQySJiA+DBwDfAEif6HJUBdiP2mABuT1uuDtn2Y2UVm9jKJ4Ts+PJh9R4op5YW8paaCxUvVDyEimSNMQBzl7jOSXyQu+xxMqnmrfb8G93vd/UjgQuArg9kXwMyuCvov6hobG0OUFY3z5k1m1ZYWVm1uSVsNIiJDKUxAPBWyrb96YGrSejVwwF+x3f0J4DAzGz+Yfd39FnevdffaysrKEGVFY9Exk4gZeiZCRDLGAQPCzCaa2XFAoZktMLOFwevtQFGI734WmG1mM8wsD7gMWNzvZ8wyMwuWFwJ5wPYw+440lSX5nDxrPL9d1sAb3SoiIqPXQFOOvhv4IInf3v8rqb0F+MLBvtjdu83sWuAhIA7c7u4rzOzq4PObSUxCdKWZdQF7gEuDTuuU+w724IbbefMm89lfLuPptTs4YWZWT7onIhnADvbbrpm9193vGaZ63pTa2lqvqwvTfx6N9q4eTvnGo8yZXMZPP3x82uoQEQnLzJa4e22qzwY6g+hzn5m9H6hJ3t7dvzw05WWOgtw4Hz5lBv/54CqW1TcxtzqrJ94TkVEuTCf1b0g8g9ANtCa9JIW/PWE6pQU53PTY6nSXIiLypoQ5g6h297MiryRDlBTk8sGTavjuo6t5dUsLs6tK0l2SiMghCXWbq5kdG3klGeRDJ8+gKC/O9//4WrpLERE5ZGEC4hRgSTBw3jIzW25my6IubDSrKM7j/cdPY/ELDWzY3pbuckREDkmYgFgEzAbOBM4Dzg3eZQAfO3UmcTP+53GdRYjI6HTQgHD39SSean5HsNwWZr9sV1VawCW11dyzpF6TCYnIqBRmTuovAp8DPh805QL/G2VRmeLq0w6jx50f/mlNuksRERm0MGcCFwHnE9za6u4NgG7NCWHq2CIumDeZO57ewI7WznSXIyIyKGECojMY/sIBzKw42pIyy/87/TDau3v40Z/XprsUEZFBCRMQd5vZD4ByM/sY8Afgh9GWlTlmTSjhrKMn8uOn1tHc3pXuckREQgvTSf0t4JfAPcARwL+6+41RF5ZJrjl9Fi3t3fyPnosQkVEk1N1I7v6wu38G+KO7PxxxTRnnmCllvHdhNT98Yg2vbNGEQiIyOgz2dlUN0HeIvnD2kYwpyOGf731R80WIyKgw2IBINRWohDBuTD6fX3Qkz6zbwS+X1Ke7HBGRgxpsQHw8kiqyxCXHTaV2egX//sBK3fYqIiNemAflLjGzvuce3m1mvwqmB5VBisWMr110LC3t3Xz9dyvTXY6IyIDCnEH8i7u3mNkpwBnAT4D/ibaszHXExBI++raZ3F1XzzNrd6S7HBGRAwoTED3B+znAze7+GyAvupIy3yfeOYsp5YX886+X09ndm+5yRERSChMQrwcPyr0PeMDM8kPuJwdQlJfDly84mle27ObWJzVOk4iMTGH+o38f8BBwlrs3AWOBz0RaVRZ451FVvPvoKr77yKts3KE5I0Rk5AkTEJOA+939VTN7O3AJ8EykVWWJfzv/aOJmfPr/XqC7R5eaRGRkCRMQ9wA9ZjYLuA2YAdwRaVVZYlJZIV++4BieXruD//7DK+kuR0RkH2ECotfdu4H3ADe4+6dInFXIEHjvcdVc9pap3PTYazz28tZ0lyMisleYgOgys8uBK4H7grbcMF9uZmcFc1mvNrPrU3x+RTDP9TIze8rM5iV9ti6Y/3qpmdWF+Xmj1b+dfzRHTSrlU3cv5fWmPekuR0QECBcQHwJOBL7m7mvNbAYhZpQzszhwE4k5recAl5vZnH6brQVOc/e5wFeAW/p9frq7z3f32hB1jloFuXG+f8VCunuca37+nG59FZERIcxw3y8BnwaWm9kxQL27fz3Edx8PrHb3Ne7eCdwFXNDvu59y953B6l+B6kFVn0FmjC/mmxfPZenGJv5DT1mLyAgQZqiNtwOvkjgb+D7wipmdGuK7pwAbk9brg7YD+Qjwu6R1B35vZkvM7KoB6rvKzOrMrK6xsTFEWSPXomMn8aGTa/jRn9fxwPJN6S5HRLJcTohtvg2c6e6rAMzscOBO4LiD7Jdq5NeU41yb2ekkAuKUpOaT3b3BzCYAD5vZy+7+xH5f6H4LwaWp2traUT+O9ucXHcXzG5r47C+XcdSkUmaM1wyvIpIeYfogcvvCAcDdXyFcJ3U9MDVpvRpo6L+Rmc0FbgUucPftST+nIXjfCtxL4pJVxsvLiXHTFQvJiRsf/1kdu9o0TamIpEeYgFhiZreZ2duD1w+BJSH2exaYbWYzzCwPuAxYnLyBmU0DfgX8bRA8fe3FfSPImlkxcCbwYrhDGv2mlBfy/fcvZO22Vj7602dp7+o5+E4iIkMsTEBcDawAPgFcB7wUtA0oeHbiWhLDdKwE7nb3FWZ2tZn17f+vwDjg+/1uZ60CnjSzF0g8tX2/uz84iOMa9U6aNZ7/vnQ+det3cu0dz+tJaxEZdjbQ9JdmFgOWufsxw1fSoautrfW6usx6ZOInT63ji4tXcNlbpvIf7zkWM03qJyJDx8yWHOhRggE7qd2918xeMLNp7r4hmvJkIB84qYZtuzu48dHVjB+Tz6fffUS6SxKRLBHmLqZJwAozewZo7Wt09/Mjq0r28Q9nHE5jSwffe2w148fk8cGTZ6S7JBHJAmEC4kuRVyEDMjO+euExbG/t5Ev3vcS4MfmcN29yussSkQx3wIAIRm+tcvfH+7WfCrwedWGyr5x4jBsvX8Df3vY0n/rFUszg3LkKCRGJzkB3Md0AtKRobws+k2FWkBvntg++hQXTyvnEnc9z97MbD76TiMghGiggatx9Wf9Gd68DaiKrSAZUWpDLTz/8Vk6ZXcln71nGbU+uTXdJIpKhBgqIggE+KxzqQiS8wrw4P7zyOBYdM5Gv3PcSN/zhFQa6XVlE5FAMFBDPmtnH+jea2UcI9yS1RCg/J86Nly/g4uOqueEPr/K1+1cqJERkSA10F9MngXvN7AreCIRaIA+4KOrC5OBy4jH+871zGZOfw61PrmV3RzdfvfAYcuJhHpAXERnYAQPC3bcAJwUjrfY9SX2/uz86LJVJKLGY8cXz5lBSkMONj67m9aY9fO/yhZQVhZr0T0TkgAYcamO0ycShNgbjrmc28C+/eZEp5YXc+oFaZk0oSXdJIjLCDTTUhq5FZJDLjp/GnR87gd0d3Vx401M8snJLuksSkVFMAZFhamvGsvjaU6gZX8RHf1rHTY+tVue1iBwSBUQGmlxeyP99/CTOnzeZbz60imvvfJ62zu50lyUio4wCIkMV5sW54dL5XL/oSB5Yvolzb3ySF1/fle6yRGQUUUBkMDPj6tMO4+cfeSttHT1c9P0/c/Pjr9HTq0tOInJwCogscNKs8Tz4ybfxrqOq+PrvXuaKW/9KQ9OedJclIiOcAiJLlBfl8f0rFvKfF89lWf0uzrrhCe5ftindZYnICKaAyCJmxvtqp/LAJ97GjMoxXHPHc1x31/Ns292R7tJEZARSQGShmvHF/PLqE7nunbN5YPkm3vGtP3LnMxvoVd+EiCRRQGSp3HiMT51xOL+77m0cOamUz/9qOe/7wV9YtTnVFCAiko0UEFlu1oQSfnHVCXzz4rm81ribc777J77+u5fZ09mT7tJEJM0UEIKZcUntVB75x7dz0YIp3Pz4a7zrvx7n3ufrddlJJItFGhBmdpaZrTKz1WZ2fYrPrzCzZcHrKTObF3ZfGXpji/P45iXz+MVVJ1BRnMunfvEC5974JE++ui3dpYlIGkQWEGYWB24CFgFzgMvNbE6/zdYCp7n7XOArwC2D2Fci8taZ41h8zSl857L57NrTxd/c9jRX3v4MLzU0p7s0ERlGUZ5BHA+sdvc17t4J3AVckLyBuz/l7juD1b8C1WH3lWjFYsYF86fwyD+exj+fcxQvbGzinBv/xD/cvZS121rTXZ6IDIMoA2IKsDFpvT5oO5CPAL8b7L5mdpWZ1ZlZXWNj45soV1IpyI3z0bfN5InPnM7H3jaT+5dt4p3f/iOfuPN53fEkkuGiDAhL0ZayxzOYte4jwOcGu6+73+Lute5eW1lZeUiFysGVFeXyhbOP4k+fSwTFH1Zu4d03PMHHf1bH8noNAiiSiQaak/rNqgemJq1XAw39NzKzucCtwCJ33z6YfWX4TSgp4PNnH8XVpx3Gj55ax4//vJaHVmzhtMMr+djbZnLyrHGYpcp3ERltIpty1MxygFeAdwKvA88C73f3FUnbTAMeBa5096cGs28q2T7laDq0tHfxs7+u57Y/rWV7ayezJ4zhAyfV8J6FUyjKi/L3DxEZCgNNORrpnNRmdjZwAxAHbnf3r5nZ1QDufrOZ3Qq8F1gf7NLdV2iqfQ/28xQQ6dPe1cP9yzbx46fWsfz1XZQU5HBp7VSuPLGGaeOK0l2eiBxA2gJiuCkg0s/deW7DTn781Hp+t3wTPe6cOruS99VO5V1zJpCfE093iSKSRAEhabF5Vzt3PL2e/1tSz6Zd7ZQX5XLh/ClcUlvN0ZPL0l2eiKCAkDTr6XX+vHobd9dt5PcrttDZ08vRk0t5z8Jqzp07iarSgnSXKJK1FBAyYjS1dfKbpQ3cXbeRFQ3NmMHxNWM5b95kFh0zkXFj8tNdokhWUUDIiLR6627uW9bAb19o4LXGVuIx46TDxnHu3Em886gqxissRCKngJARzd15eXMLv32hgfuWbWLDjjbMYOG0Ct51VBVnzKli1oQx6S5TJCMpIGTUcHde2tTMH17aysMrN/Pi64kBAmeOL+Zdc6o47fBKamsqdDeUyBBRQMio1dC0hz+s3MLDL23hr2u209XjFObGOWHmWE49vJJTD69k5vhiPb0tcogUEJIRWju6+ctr23ni1UaeeKWRddvbAKiuKOSkw8ZxwsxxnHjYOCaVFaa5UpHRQwEhGWnD9jYeD8Li6TXbaW7vBmD6uCJOnJkIjLfMGMuUcgWGyIEoICTj9fQ6L29u5i+vbeeva3bw9NrttASBMamsgOOmV3Dc9Apqp4/lqEkl5MQ1264IKCAkC/X0Ois3NVO3bgd163fy3PqdNOxqB6AwN87c6jLmTS1PvFeXU11RqH4MyUoKCBESHd5L1u9kyfqdPL+xiZUNzXT29AKJ+bjnVpcxt7qcoyeXcvTkUqaUKzQk8w0UEBqPWbLG5PJCJpcXct68yQB0dveyanMLS+ubWLaxiWX1u3jilVfpDX5nKivMZc6kRFgcPaWUI6pKOWxCsW6xlayhgJCslZcT49jqMo6tLoMTpgOwp7OHlZubeamhmRUNzbzUsIuf/XU9Hd2JM414zJgxvpgjJpZwRFUJR0wsYfaEMUwbW6R+Dck4CgiRJIV5cRZOq2DhtIq9bd09vazd1srLm1tYtbmFVVtaWF6/i/uXbdq7TW7cqBlXzKwJY/a+DqscQ834Ysbk65+ZjE76mytyEDnxGLOrSphdVcJ5895ob+3o5tWtu3l1SwurG3fz2tbdrNzUzEMrNu+9TAVQWZLPjPHFzBhXzIzKYmrGFTFtbDHTxhUpPGRE099OkUNUnJ/D/KnlzJ9avk97e1cP67a3sraxlbXB+7rtrTzy8ha21XXus+244jymji1i+rgipo0torqikOqKxPukskLycnTZStJHASEyxApy4xw5sZQjJ5bu91lzexcbtrexYUcb67e3sWFHKxt2tLFk/U5++0LDPmceZjCxtIApQef6pPICJpcFy2UFTC4vpKIoV3daSWQUECLDqLQgl2OmlHHMlP1n1Ovq6WXzrnbqd+6hfmcb9Tv38HrTHjbuaGPpxiYefLF97225ffJzYlSVFjCxtICqsgImluZTVVrAhNICJpTkJ16lBbqUJYdEf2tERojceIypY4uYOrYIGLff5729zvbWTjbt2kND0x4amtrZ0tzO5uZ2Nu9qZ3l9Ew83t9Pe1bvfvkV58SAwChhfksf4MfmMK87fu5xYz2PsmDxK8nN0ViKAAkJk1IjFjMqSfCpL8plbXZ5yG3dn154utrZ0sLW5g60t7fssN7Z0sGpzC0+1bqeprSvld+TFY1QU5zK2OBEaFcV5VBTlUlEUvBfnUVGUR3lRLuWFeZQV5VKSn0MsplDJNAoIkQxiZpQX5VFelMfhVSUDbtvZ3cuO1k627e6gcXcHO3Z3sqO1k+2tnexo7WD77sRy/c42drZ1sWtP6kABiBmUFuZSXphLWWEupYW5lBYE74U5bywX5FBSkMOY/FxKguWSglzG5OcQV8CMOAoIkSyVlxNjYlkBE8sKQm3f3dPLrj1d7GzrYmdbJ01BaDS1dQbvifbm9m6a93TxetMemvcktunqOfiQPoW5ccYU5DAmP/Eqzo8zJj+HorzEcuI9h+K8OEV5cQrzcoL3OEW5ic8L83e9HwwAAAeRSURBVGIU5MYpzI1TELwUPIcu0oAws7OA7wBx4FZ3/3q/z48EfgQsBP7J3b+V9Nk6oAXoAboPNFaIiAyPnHiMcWPyGTfIucLdnY7uXpr3dNHc3k1Lexe7O7ppCZZb2rvZ3dHN7vZuWjsT7a0dibaGpnbaOrtp7eyhrSPxPlh5OTEKchLBkZ8boyCnLzxi5OfEyc+JkR8s58X7lmPk5cTIjSfe85Lec+MxcuK233JOsJwbixGPGblxS7TFjJy4kRNLLMfjlniPGXFLvI/UPp/IAsLM4sBNwBlAPfCsmS1295eSNtsBfAK48ABfc7q7b4uqRhGJnpnt/W1+wv53/g5Kb6+zp6uHts4e9nT20NbVzZ6+5c4e2rp6aA9eezp72NPVQ3tX79629q4eOrr71ntp7exmZ1svnd29dHT3vffsXe7uHZ7BTGOWGMYlZm8ERyxme9vNksMEYpb4LGaJ9XHF+dx99YlDXleUZxDHA6vdfQ2Amd0FXADsDQh33wpsNbNzIqxDRDJELGaJy0zDdNtub6/T2dObePWFRk+irbu3l67uYLmnl64ep6s38XlPb2K9O3jv6XW6e52enkTo7F1PfrnTm7TsTtJyor3XoTf4rNffWC+J6M8jyj/lKcDGpPV64K2D2N+B35uZAz9w91tSbWRmVwFXAUybNu0QSxUR2V8sZhTEEmc/2SjK5/hTXVQbzPnaye6+EFgEXGNmp6bayN1vcfdad6+trKw8lDpFRCSFKAOiHpiatF4NNITd2d0bgvetwL0kLlmJiMgwiTIgngVmm9kMM8sDLgMWh9nRzIrNrKRvGTgTeDGySkVEZD+R9UG4e7eZXQs8ROI219vdfYWZXR18frOZTQTqgFKg18w+CcwBxgP3Brd+5QB3uPuDUdUqIiL7i/RWAHd/AHigX9vNScubSVx66q8ZmJeiXUREhokGmxcRkZQUECIikpICQkREUjL34XmUfDiYWSOw/hB3Hw9k47AeOu7souPOLmGOe7q7p3yILKMC4s0ws7psHBBQx51ddNzZ5c0ety4xiYhISgoIERFJSQHxhpSDAWYBHXd20XFnlzd13OqDEBGRlHQGISIiKSkgREQkpawPCDM7y8xWmdlqM7s+3fVEycxuN7OtZvZiUttYM3vYzF4N3ivSWeNQM7OpZvaYma00sxVmdl3QnunHXWBmz5jZC8Fxfyloz+jj7mNmcTN73szuC9az5bjXmdlyM1tqZnVB2yEfe1YHRNK82YtIjCJ7uZnNSW9VkfoxcFa/tuuBR9x9NvBIsJ5JuoF/dPejgBNITD41h8w/7g7gHe4+D5gPnGVmJ5D5x93nOmBl0nq2HDfA6e4+P+n5h0M+9qwOCJLmzXb3TqBv3uyM5O5PADv6NV8A/CRY/glw4bAWFTF33+TuzwXLLST+05hC5h+3u/vuYDU3eDkZftwAZlYNnAPcmtSc8cc9gEM+9mwPiFTzZk9JUy3pUuXumyDxnykwIc31RMbMaoAFwNNkwXEHl1mWAluBh909K44buAH4LNCb1JYNxw2JXwJ+b2ZLzOyqoO2Qjz3S+SBGgTc7b7aMEmY2BrgH+KS7NweTUWU0d+8B5ptZOYkJuI5Jd01RM7Nzga3uvsTM3p7uetLgZHdvMLMJwMNm9vKb+bJsP4N4U/NmZ4gtZjYJIHjfmuZ6hpyZ5ZIIh5+7+6+C5ow/7j7u3gT8kUT/U6Yf98nA+Wa2jsQl43eY2f+S+ccNgLs3BO9bgXtJXEY/5GPP9oA45HmzM8hi4APB8geA36SxliFniVOF24CV7v5fSR9l+nFXBmcOmFkh8C7gZTL8uN398+5e7e41JP49P+ruf0OGHzeAmRWbWUnfMnAm8CJv4tiz/klqMzubxDXLvnmzv5bmkiJjZncCbycxBPAW4IvAr4G7gWnABuASd+/fkT1qmdkpwJ+A5bxxTfoLJPohMvm455LokIyT+EXwbnf/spmNI4OPO1lwienT7n5uNhy3mc0kcdYAie6DO9z9a2/m2LM+IEREJLVsv8QkIiIHoIAQEZGUFBAiIpKSAkJERFJSQIiISEoKCJFBMLOeYKTMvteQDfpmZjXJI+2KpFu2D7UhMlh73H1+uosQGQ46gxAZAsE4/N8I5mB4xsxmBe3TzewRM1sWvE8L2qvM7N5gvoYXzOyk4KviZvbDYA6H3wdPQYukhQJCZHAK+11iujTps2Z3Px74Homn8wmWf+ruc4GfA98N2r8LPB7M17AQWBG0zwZucvejgSbgvREfj8gB6UlqkUEws93uPiZF+zoSE/SsCQYH3Ozu48xsGzDJ3buC9k3uPt7MGoFqd+9I+o4aEsNyzw7WPwfkuvtXoz8ykf3pDEJk6PgBlg+0TSodScs9qJ9Q0kgBITJ0Lk16/0uw/BSJUUUBrgCeDJYfAf4O9k7sUzpcRYqEpd9ORAanMJilrc+D7t53q2u+mT1N4hevy4O2TwC3m9lngEbgQ0H7dcAtZvYREmcKfwdsirx6kUFQH4TIEAj6IGrdfVu6axEZKrrEJCIiKekMQkREUtIZhIiIpKSAEBGRlBQQIiKSkgJCRERSUkCIiEhK/x8sFKCtOGeRSwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(ce_hist)\n",
    "plt.ylabel('Cross-entropy value')\n",
    "plt.xlabel('Epoch');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZxcZb3n8c+v9707S6dJ0tmEEBICWWwCIuNlcQEFAblXwEEUFwYExQWv6J0Z5rrMqHNdcBm5iCgZUASEKyqKigKySEhIQsgCZN+T6obe0l3d1VW/+8c5nVSa6qQSurq6q77v16tfXWep6t+BV863zvOc8zzm7oiIiAxUkO0CRERkZFJAiIhISgoIERFJSQEhIiIpKSBERCSlomwXMJTGjx/v06dPz3YZIiKjxrJly5rdvT7VtpwKiOnTp7N06dJslyEiMmqY2ZbBtqmJSUREUlJAiIhISgoIERFJSQEhIiIpKSBERCQlBYSIiKSkgBARkZRy6jkIEZHRyt3p6UsQjcVp7+6jtbuXtu4YrV0xWrtjtHX1Ek9AWXEBZcWFlBYFv8uKC6gqLeaMmeOHvCYFhIhIkmgszuaWfRQVFFBXUUxteTHFhYdvbOnpi9PWHaMtPKG3dsVo7Tpwkm/r7l/fS3t3sNwdixONBaHQ05c46prHV5Wy9L+//ajfPxgFhIjkLHeno6ePtq5Yyu3RWJz1ezt5aU8HL+0Ofja37CMxYB61qtIiasuDsCgoYP9JPRpL0BOLE+2LE4sPPvlagUFteTF1FSXUlhczprKEaeMqKQ+vAMqKCyntf11USHVZEXUVJdRVFFNXXkxtGFSFZkT7+v9mfw1xMjXvmwJCREacvniC7lic9mjf/m/h/d/M27pj9MVTf9vu6OljT1uU3e1R9rb3sLs9Sldv/LB/zwymj6vk+IYqzp83iZkTqki4057UxBNcBfSScPafyEuTTvCVJYX7T+q15cXUlQeva8qLqS4toqDAhuS/TVVhAVWlw3PqVkCIyCElEgfaxqN9B5pEOsKTd+v+k3cvrV0x4gkPvxEHJ9H+dnLgoOaWtv7mlmgf3b1xepI+u2/gV/g0lRQWMKGmlGNqypg9sYYzZ03gmNpS6ipKKLDXn6CLC403ja/iuAlVlJcUvqH/TrlIASGSh3r7Emx7rYutLV1sbtnHjte6D/qWnHwST7dtvLDAqC0vpqjAwjBJ0JvivUHzyYFv2MfUloUhUhgGyoHO15qy4vAb+YFv5rXlxZQUpe4TKCowLEUQyNFRQIiMEF29fWHHZvBtvO2gpo2DT9ytXTHaozHGVZYwdVwl08dVMHVsBdPHVzJtbAUYQRNL2NyyJ/zZ0drNlpYudrZ2H9TOXl5cyJiKYmorSqgrL+bY+qr9zSPlxQeuAsqS2slryg+csOsqiqkqLXrdyTn56sOBmrIiitLo8JWRQQEhMkzcnebOXra+uo/NzV1sadnH5pYutrwavG4dpCMVgqaT2v4Oy/JiJtWVMausmubOHlZua+V3L+x8XcdqsgKD+upSJtaW0zRtDFMXNjJ9XAXTxlUybVwF4ypLMvLNu6DAKC8pVPPNKKWAEBlEPOGs2NbKEy9HeH7ra3T1xvffNRKNJejpC25NHFNRQkNNKQ01ZRxTU8YxtWVMqCljX08fW1oOBMHWln3sS+owLTCYVFfO9HGVvPukiTSOKWfs/k7Okv3fzGvLi6koKTzkCby3LxFeHexjS0sXZuyvp6GmjPFVJfrmLkdMASGSZHdblCdejvD4yxGeXN9MW3cMM5h9TA1jKoNv8Ps7YIsLKSks4LWuXna3RXlxRxt/XruHaOxAu3tJYQGNY4MQOHXG2IO+tTeOqRi0Lf1IlRQVMGN8JTPGVw7J54mAAkLyXDQW57nNr/L4SxGeeCXCy3s6AZhQXco75zTwtuPrOeO48YypLEnr89yd9mgfe9qjVJQUMrG2nMIhur1RZLgpICSv9MUTvLK3k2c2tPD4yxGe3dRCNJagpLCAU2aM4ZKFjfzDrHpmNVQfVZu8me3vuBUZ7RQQkrPcnS0tXazc3srKbW28sL2VF3e27W8CelN9JZedMpV/OL6eU980looS/XMQSaZ/ETKqrd/byUMrdrC7PXrwmDddMV7r6t1/D39pUQFzJ9dy+aKpzJ9Sx8KpY5gytiLL1YuMbAoIGXX64gn+vHYP///vW3hqfQuFBcb4qhLqykuorShmytgKTpocNPMcO6GKkxtrOb6hOq0B10TkAAWEjBqRjh7uWbKVny/Zyq62KJNqy/j8u2Zx6SlTGF9Vmu3yRHKOAkJGlObOHp7e0LJ/wLVg0LXg987WKPGEc8Zx4/lf7z2Rc06YoHv7RTJIASEjQndvnNv/tpFbH9+w/2Gy8uJCjqkto6GmlIVTx3DR/AouWjCZY+urslytSH5QQEhWJRLOA8t38G+PvMTu9ijvOrGB6846jmnjKqkpe/3YPiIyfBQQkjVPr2/mq79by5pd7cxrrOV7ly9g0Yyx2S5LREIKCMmorS1dbGjuZE9blD3hBC572qPseK2bl/Z0MLmunFsum88FJ08asglVRGRoKCBkyHVEY/zuhV3cu3Qbz29tPWjbuMoSGmrKmDymnEvePJkr3zKdsmKN9CkyEikgZEi4O89uepV7l27j96t20x2LM3NCFf/y7tksnFZHQ00ZE6rLhmxwOhHJPAWEvGF/eyXC//z1ajY176O6tIiLF07m/U1TmNdYq05mkVFMASFHrbs3zjf+sI6fPb2Z4yZU8Z1L53HuiRM1OYxIjlBAyFFZtb2NT/9yORsi+7jqrdP5wrknqC9BJMcoIOSI9MUT/OixDdzy6CuMryrlro+eyhkzx2e7LBHJAAWEpMXdeX5rK1/73Rqe39rKe+dN4isXzqW2QvMeiOSqjAaEmZ0L3AIUAre7+9cHbB8D3AEcC0SBj7j7i0nbC4GlwA53Pz+TtUpq3b1xHlq5g8XPbGH1znZqy4u55bL5XDh/crZLE5EMy1hAhCf3HwLvALYDz5nZQ+6+Jmm3LwEr3P1iMzsh3P+cpO03AGuBmkzVKaltbt7HXX/fwn3LttPWHWNWQzVfvWguFy2YTFWpLjxF8kEm/6UvAta7+0YAM7sHuBBIDog5wP8BcPd1ZjbdzBrcfY+ZNQLvAb4GfDaDdUqSve1RvvzbNfz2hV0UFRjnzj2GD542jUUzxuqWVZE8k8mAmAxsS1reDpw6YJ+VwPuAJ81sETANaAT2AN8F/hmoPtQfMbOrgasBpk6dOiSF56NEwrl7yVa++ft19MQTXH/WcVz5lmlMqCnLdmkikiWZDIhUXzd9wPLXgVvMbAWwClgO9JnZ+cBed19mZmce6o+4+23AbQBNTU0DP1/SsG53O198YBXLt7Zy+rHj+NrFJzFjfGW2yxKRLMtkQGwHpiQtNwI7k3dw93bgKgAL2i82hT+XAe81s3cDZUCNmd3l7ldksN68090b53t/eYUfP7GRmvJivv3+eVy8YLKakkQEyGxAPAfMNLMZwA6Ck/4Hkncwszqgy917gY8BT4Sh8cXwh/AK4kaFw9Batb2NT92znE3N+3h/UyNfPG82YypLsl2WiIwgGQsId+8zs+uBRwhuc73D3Veb2TXh9luB2cBiM4sTdF5/NFP1SCCRcO54ahPf+MM6xleV8vOPn8rpx+pBNxF5PXPPnWb7pqYmX7p0abbLGLGaO3u48b6VPPZShHfOaeCb/3gydRW6ahDJZ2a2zN2bUm3TDe154qn1zXz6lyto647xlQtP5IrTpqmvQUQOSQGR42LxBN/508v86PENHFtfxeKPLGL2RD13KCKHp4DIYdte7eJT9yxn+dZWLm2aws3vnUNFif6Xi0h6dLbIUb97YRc3PfACOHz/8gVcMG9StksSkVFGAZFjunvjfPm3q/nFkm3Mn1LH9y9fwJSxFdkuS0RGIQVEDlm3u51P/nw56yOdXHvmsXz2HcdTXKg5oEXk6CggckAi4dz5zGa+/vt1VJcVs/gji/gvM+uzXZaIjHIKiFFuV1s3n7/vBZ5c38xZs+r55j/Oo766NNtliUgOUECMYr9ZuZN/eXAVsbjztYvn8oFFU/Vsg4gMGQXEKNTWFeN//PpFHlq5kwVT6/j2++dr9FURGXIKiFFmxbZWrr1rGZGOHj73juO59sxjKVJHtIhkgAJiFFm3u50P3bGEmvIiHvjE6ZzcWJftkkQkhykgRomtLV188CdLKCsu4OcfO03PNohIxikgRoG97VGu+MmzxOIJ7v1vb1E4iMiwUOP1CNfWFePKO5bQ3NnDTz98Csc3HHKKbhGRIaOAGMG6evu46mdL2BjZx20fbGLB1DHZLklE8ogCYoTq7UtwzV3Ps2JbK7dcNp8zZmrWNxEZXuqDGIE6ojE+fc8Knng5wjcuOYnzTpqY7ZJEJA8pIEaYDZFOrl68lM0tXXzlorlcesrUbJckInlKATGC/HnNHj7zyxWUFBVw98dO5bQ3jct2SSKSxxQQI0Ai4Xz/L+v5zp9f5qTJtdz6wTczua4822WJSJ5TQGRZRzTGZ+9dyZ/W7OF9Cyfzvy8+ibLiwmyXJSKigMimeML5wI+fZc2udm6+YA4fPn26RmMVkRFDAZFF/7F8B6t2tPHdS+dz0YLJ2S5HROQgeg4iS3r7Evv7HC6cPynb5YiIvI4CIkt++dxWtr/WzY3vmqVmJREZkRQQWdDdG+d7f1nPohljeZuekBaREUoBkQV3PrOZSEcPn9fVg4iMYAqIYdYejfGjxzZw5qx6Tpk+NtvliIgMSgExzG5/YiNt3TFufOesbJciInJICohh1NLZw0+e3MR7TprI3Mm12S5HROSQFBDD6EePbaA7Fucz7zg+26WIiByWAmKY7GrrZvHft3DJwkaOm1CV7XJERA5LATFMvvfoetydG94+M9uliIikRQExDLa2dHHv0m18YNFUGsdUZLscEZG0ZDQgzOxcM3vJzNab2U0pto8xswfN7AUzW2Jmc8P1U8zsr2a21sxWm9kNmawz0/7fY+spLDA+cdZx2S5FRCRtGQsIMysEfgicB8wBLjezOQN2+xKwwt1PBq4EbgnX9wGfc/fZwGnAdSneOyrsaO3mV89v59KmKTTUlGW7HBGRtGXyCmIRsN7dN7p7L3APcOGAfeYAjwK4+zpgupk1uPsud38+XN8BrAVG5XCn//74BgCuOfPYLFciInJkMhkQk4FtScvbef1JfiXwPgAzWwRMAxqTdzCz6cAC4NlUf8TMrjazpWa2NBKJDEnhQ2Vve5R7ntvGJQsbNUOciIw6mQyIVIMM+YDlrwNjzGwF8ElgOUHzUvABZlXAr4BPu3t7qj/i7re5e5O7N9XX1w9N5UPktic2Ek841+rqQURGoUxOGLQdmJK03AjsTN4hPOlfBWDBqHWbwh/MrJggHO529wcyWGdGtHT2cPezW7lw3iSmjavMdjkiIkcsk1cQzwEzzWyGmZUAlwEPJe9gZnXhNoCPAU+4e3sYFj8B1rr7tzNYY8bc/uQmon1x3bkkIqNWxq4g3L3PzK4HHgEKgTvcfbWZXRNuvxWYDSw2sziwBvho+Pa3Ah8EVoXNTwBfcveHM1XvUGrt6mXx05t5z0kT9dS0iIxahw0IMzsfeNjdE0f64eEJ/eEB625Nev0M8LpHi939SVL3YYwKP31qM/t641x/tq4eRGT0SqeJ6TLgFTP7ppnNznRBo11HNMZPn9rEO+c0cMIxNdkuR0TkqB02INz9CoLbTDcAPzWzZ8JbS6szXt0otPiZLbRH+/jk2RpzSURGt7Q6qcO7jX5F8LDbROBi4Hkz+2QGaxt1unr7uP1vGzlrVj0nNWq+BxEZ3Q4bEGZ2gZk9CPwFKAYWuft5wDzgxgzXN6rc+9w2XuuKqe9BRHJCOncx/RPwHXd/Inmlu3eZ2UcyU9bok0g4dz6zhQVT63jzNM01LSKjXzpNTDcDS/oXzKw8HP4Cd380M2WNPo+/EmFT8z4+fPr0bJciIjIk0gmI+4DkW1zj4TpJcufTm5lQXcp5cydmuxQRkSGRTkAUhaOxAhC+LjnE/nlnY6STx16KcMVp0ygp0hxMIpIb0jmbRczsvf0LZnYh0Jy5kkafxc9soaSwgMsXTc12KSIiQyadTuprgLvN7AcETzdvI5jcRwgejLtv6TbOP3ki9dWl2S5HRGTIHDYg3H0DcFo49LaFE/hI6P5l29nXG+dD6pwWkRyT1mB9ZvYe4ESgLBhoFdz9yxmsa1RIJJw7n97Mwql1zJtSl+1yRESGVDoPyt0KXEowoY8RPBcxLcN1jQqPvxxhc0sXH37rjGyXIiIy5NLppD7d3a8EXnP3fwXewsETAeWtnz29mYaaUs6be0y2SxERGXLpBEQ0/N1lZpOAGJD3X5k3RDp5/OUIV5w6jeJC3doqIrknnT6I35hZHfB/gecJ5pX+cUarGgUWP705uLX1VN3aKiK56ZABYWYFwKPu3gr8ysx+C5S5e9uwVDdCtUdj3L9sOxfMm8T4Kt3aKiK56ZBtI+Esct9KWu7J93AAePD5HezrjWvcJRHJaek0nv/RzC6x/vtbhSWbX2XK2HLN+SAiOS2dPojPApVAn5lFCW51dXfP2/k01+5qZ7amExWRHJfOlKPV7l7g7iXuXhMu5+3Zsbs3zubmfcyemLf/CUQkTxz2CsLM3pZq/cAJhPLFy3s6SDjMnqgpuUUkt6XTxPT5pNdlwCJgGXB2Rioa4dbuagfQFYSI5Lx0Buu7IHnZzKYA38xYRSPcut0dVJYUMmVMRbZLERHJqKN5BHg7MHeoCxkt1uxqZ9Yx1RQU6KYuEclt6fRBfJ/g6WkIAmU+sDKTRY1U7s66Xe2cP29StksREcm4dPoglia97gN+4e5PZaieEW1nW5T2aJ/6H0QkL6QTEPcDUXePA5hZoZlVuHtXZksbedb1d1AfozuYRCT3pdMH8ShQnrRcDvw5M+WMbP13MJ2gKwgRyQPpBESZu3f2L4Sv8/IWnrW7Opg6toKq0rQm4hMRGdXSCYh9Zrawf8HM3gx0Z66kkWvt7nZOUPOSiOSJdL4Kfxq4z8x2hssTCaYgzSv9Q2xccLLuYBKR/JDOg3LPmdkJwCyCgfrWuXss45WNMBpiQ0TyzWGbmMzsOqDS3V9091VAlZl9IvOljSwaYkNE8k06fRAfD2eUA8DdXwM+ns6Hm9m5ZvaSma03s5tSbB9jZg+a2QtmtsTM5qb73uGmITZEJN+kExAFyZMFmVkhUHK4N4X7/RA4D5gDXG5mcwbs9iVghbufDFwJ3HIE7x1WGmJDRPJNOgHxCHCvmZ1jZmcDvwB+n8b7FgHr3X2ju/cC9wAXDthnDsFzFrj7OmC6mTWk+d5h4+7BJEFqXhKRPJJOQHyB4CR+LXAd8AIHPzg3mMnAtqTl7eG6ZCuB9wGY2SJgGtCY5nsJ33e1mS01s6WRSCSNso7czrYoHdE+PSAnInklnRnlEsDfgY1AE3AOsDaNz07VFuMDlr8OjDGzFcAngeUE4z2l897++m5z9yZ3b6qvr0+jrCO3dmfQQT1HdzCJSB4Z9DZXMzseuAy4HGgBfgng7mel+dnbgSlJy43AzuQd3L0duCr8ewZsCn8qDvfe4bRudxAQszQPtYjkkUNdQawjuFq4wN3PcPfvA/Ej+OzngJlmNsPMSgjC5qHkHcysLtwG8DHgiTA0Dvve4aQhNkQkHx3qjHcJwYn5r2b2B4KO4rRv4XH3PjO7nqCTuxC4w91Xm9k14fZbgdnAYjOLA2uAjx7qvUd8dENEQ2yISD4aNCDc/UHgQTOrBC4CPgM0mNmPgAfd/Y+H+3B3fxh4eMC6W5NePwPMTPe92aAhNkQkX6XTSb3P3e929/MJ+gJWAFl/cG24vKQhNkQkTx3RnNTu/qq7/7u7n52pgkaadRpiQ0Ty1BEFRD5au6tdQ2yISF5SQBzG2t0dGmJDRPKSAuIQNMSGiOQzBcQhaIgNEclnCohD0BAbIpLPFBCH0D9JkIbYEJF8pIA4hHW7NcSGiOQvBcQhbH21ixnjK7NdhohIViggDiHS0cOE6tJslyEikhUKiEEkEk5zZw/1CggRyVMKiEG0dsfoS7gCQkTylgJiEJGOHgAFhIjkLQXEIPYHRJUCQkTykwJiEJHOKKArCBHJXwqIQaiJSUTynQJiEHvbeygvLtRDciKStxQQg4iEt7iaaZhvEclPCohBRDr0DISI5DcFxCAiHT26g0lE8poCYhARPUUtInlOAZFCT1+c1q6YAkJE8poCIoWWzl5At7iKSH5TQKSwN3wGQiO5ikg+U0CkoIfkREQUECkpIEREFBAp9QfEuEoFhIjkLwVECpHOKGMqiikp0n8eEclfOgOmoKeoRUQUECkpIEREFBApRTp7mFBdlu0yRESySgExgLuzt11XECIiCogBOnr66OlLaKA+Ecl7GQ0IMzvXzF4ys/VmdlOK7bVm9hszW2lmq83sqqRtnwnXvWhmvzCzYWnz0TMQIiKBjAWEmRUCPwTOA+YAl5vZnAG7XQescfd5wJnAt8ysxMwmA58Cmtx9LlAIXJapWpMpIEREApm8glgErHf3je7eC9wDXDhgHweqLZi2rQp4FegLtxUB5WZWBFQAOzNY634KCBGRQCYDYjKwLWl5e7gu2Q+A2QQn/1XADe6ecPcdwL8BW4FdQJu7/zHVHzGzq81sqZktjUQib7jo/QGhPggRyXOZDIhUkzn7gOV3ASuAScB84AdmVmNmYwiuNmaE2yrN7IpUf8Tdb3P3Jndvqq+vf8NFRzp7KC406iqK3/BniYiMZpkMiO3AlKTlRl7fTHQV8IAH1gObgBOAtwOb3D3i7jHgAeD0DNa63972YKrRoNVLRCR/ZTIgngNmmtkMMysh6GR+aMA+W4FzAMysAZgFbAzXn2ZmFWH/xDnA2gzWup+mGhURCRRl6oPdvc/MrgceIbgL6Q53X21m14TbbwW+AvzMzFYRNEl9wd2bgWYzux94nqDTejlwW6ZqTRbp6GFynZ6iFhHJWEAAuPvDwMMD1t2a9Hon8M5B3nszcHMm60sl0tHD/Cm1w/1nRURGHD1JnSSecF7d16M7mEREUEAcpGVfDwnXMxAiIqCAOMiBh+TUByEiooBIoqeoRUQOUEAk2RsGxAQFhIiIAiJZ/xXEeHVSi4goIJJFOnqoLi2ivKQw26WIiGSdAiKJnqIWETlAAZEk0tHDeAWEiAiggDhIc0ePOqhFREIKiCSRDjUxiYj0U0CEunvjdPT0KSBEREIKiJBmkhMROZgCIhTpjAJ6ilpEpJ8CIqRhNkREDqaACCkgREQOpoAIRTp6KDAYV6mAEBEBBcR+kc4exlWVUlhg2S5FRGREUECEIh2aSU5EJJkCIrRXD8mJiBxEARHSU9QiIgdTQACJhNOskVxFRA6igADaumPE4q4+CBGRJAoIgjuYQM9AiIgkU0Bw4CE5DfUtInKAAgI9RS0ikooCAtjboYH6REQGUkAQXEGUFRdQVVqU7VJEREYMBQQHnoEw0zAbIiL9FBAEdzHpFlcRkYMpINBT1CIiqSggCAJiQnVZtssQERlR8j4g3J0zZ01g4bS6bJciIjKi5P1tO2bGdy6dn+0yRERGnIxeQZjZuWb2kpmtN7ObUmyvNbPfmNlKM1ttZlclbaszs/vNbJ2ZrTWzt2SyVhEROVjGAsLMCoEfAucBc4DLzWzOgN2uA9a4+zzgTOBbZlYSbrsF+IO7nwDMA9ZmqlYREXm9TF5BLALWu/tGd+8F7gEuHLCPA9UWPIBQBbwK9JlZDfA24CcA7t7r7q0ZrFVERAbIZEBMBrYlLW8P1yX7ATAb2AmsAm5w9wTwJiAC/NTMlpvZ7WZWmeqPmNnVZrbUzJZGIpEhPwgRkXyVyYBI9ViyD1h+F7ACmATMB34QXj0UAQuBH7n7AmAf8Lo+DAB3v83dm9y9qb6+fsiKFxHJd5kMiO3AlKTlRoIrhWRXAQ94YD2wCTghfO92d3823O9+gsAQEZFhksmAeA6YaWYzwo7ny4CHBuyzFTgHwMwagFnARnffDWwzs1nhfucAazJYq4iIDJCx5yDcvc/MrgceAQqBO9x9tZldE26/FfgK8DMzW0XQJPUFd28OP+KTwN1huGwkuNoQEZFhYu4DuwVGLzOLAFuO8u3jgebD7pV7dNz5RcedX9I57mnunrIDN6cC4o0ws6Xu3pTtOoabjju/6Ljzyxs97rwfi0lERFJTQIiISEoKiANuy3YBWaLjzi867vzyho5bfRAiIpKSriBERCQlBYSIiKSU9wFxuDkrcomZ3WFme83sxaR1Y83sT2b2Svh7TDZrHGpmNsXM/hrOKbLazG4I1+f6cZeZ2ZKkuVb+NVyf08fdz8wKw4E+fxsu58txbzazVWa2wsyWhuuO+tjzOiDSnLMil/wMOHfAupuAR919JvAogwyKOIr1AZ9z99nAacB14f/jXD/uHuDscK6V+cC5ZnYauX/c/W7g4Dlk8uW4Ac5y9/lJzz8c9bHndUCQ3pwVOcPdnyCYcyPZhcCd4es7gYuGtagMc/dd7v58+LqD4KQxmdw/bnf3znCxOPxxcvy4AcysEXgPcHvS6pw/7kM46mPP94BIZ86KXNfg7rsgOJkCE7JcT8aY2XRgAfAseXDcYTPLCmAv8KdwdOScP27gu8A/A4mkdflw3BB8CfijmS0zs6vDdUd97BkbrG+USGfOCskBZlYF/Ar4tLu3B5MY5jZ3jwPzzawOeNDM5ma7pkwzs/OBve6+zMzOzHY9WfBWd99pZhOAP5nZujfyYfl+BZHOnBW5bo+ZTQQIf+/Ncj1DzsyKCcLhbnd/IFyd88fdL5yu9zGC/qdcP+63Au81s80ETcZnm9ld5P5xA+DuO8Pfe4EHCZrRj/rY8z0g0pmzItc9BHwofP0h4NdZrGXIhfOd/wRY6+7fTtqU68ddH145YGblwNuBdeT4cbv7F9290d2nE/x7/ou7X0GOHzeAmVWaWXX/a+CdwIu8gWPP+yepzezdBG2W/XNWfC3LJWWMmf0COJNgCOA9wM3AfwD3AlMJJnD6J3cf2JE9apnZGcDfCOY872+T/hJBP5Hb1/kAAAHgSURBVEQuH/fJBB2ShQRfBO919y+b2Thy+LiThU1MN7r7+flw3Gb2JoKrBgi6D37u7l97I8ee9wEhIiKp5XsTk4iIDEIBISIiKSkgREQkJQWEiIikpIAQEZGUFBAiR8DM4uFImf0/Qzbom5lNTx5pVyTb8n2oDZEj1e3u87NdhMhw0BWEyBAIx+H/RjgHwxIzOy5cP83MHjWzF8LfU8P1DWb2YDhfw0ozOz38qEIz+3E4h8Mfw6egRbJCASFyZMoHNDFdmrSt3d0XAT8geDqf8PVidz8ZuBv4Xrj+e8Dj4XwNC4HV4fqZwA/d/USgFbgkw8cjMig9SS1yBMys092rUqzfTDBBz8ZwcMDd7j7OzJqBie4eC9fvcvfxZhYBGt29J+kzphMMyz0zXP4CUOzuX838kYm8nq4gRIaOD/J6sH1S6Ul6HUf9hJJFCgiRoXNp0u9nwtdPE4wqCvBfgSfD148C18L+iX1qhqtIkXTp24nIkSkPZ2nr9wd377/VtdTMniX44nV5uO5TwB1m9nkgAlwVrr8BuM3MPkpwpXAtsCvj1YscAfVBiAyBsA+iyd2bs12LyFBRE5OIiKSkKwgREUlJVxAiIpKSAkJERFJSQIiISEoKCBERSUkBISIiKf0nPfE/Df9KeC0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(ac_hist)\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epoch');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test it"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### N-th image from mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9  - answer\n",
      "9  - network output\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAMhUlEQVR4nO3dfYwcdR3H8c/HerSxQKWW1gpNRR4lRos5CgRiMESF9o9CAob+YWqClhAwkCiRYBAS/yE+ESSGcEBDMQohAqHhQWkaCJIg9iC1FKtSsdY+2IPUBNRwtOXrHzc1Z7mdvduZ3dn2+34lm9md38zOl+E+ndn9zezPESEAh78PNF0AgN4g7EAShB1IgrADSRB2IIkP9nJjR3h6zNDMXm4SSOUd/VvvxqgnaqsUdtsXSrpd0jRJ90TErWXLz9BMneULqmwSQIkXY13Lto5P421Pk/RTSRdJOl3Sctund/p+ALqrymf2xZK2RMTrEfGupAclLaunLAB1qxL24yT9fdzr7cW8/2N7pe1h28N7NVphcwCqqBL2ib4EeN+1txExFBGDETE4oOkVNgegiiph3y5pwbjXx0vaWa0cAN1SJezrJZ1s+wTbR0i6XNKaesoCULeOu94iYp/tayT9WmNdb6si4tXaKgNQq0r97BHxpKQna6oFQBdxuSyQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgiUpDNtveKultSfsl7YuIwTqKAlC/SmEvfD4i3qzhfQB0EafxQBJVwx6Snrb9ku2VEy1ge6XtYdvDezVacXMAOlX1NP7ciNhpe66ktbb/GBHPjV8gIoYkDUnS0Z4dFbcHoEOVjuwRsbOYjkh6VNLiOooCUL+Ow257pu2jDjyX9EVJm+oqDEC9qpzGz5P0qO0D7/OLiPhVLVXhkHHq8EBp++9GFrZsm7VkS93loETHYY+I1yV9psZaAHQRXW9AEoQdSIKwA0kQdiAJwg4kUceNMOhj0049qbR9z23l6990yuOl7Us/9E75G3xsfcumL2lR+bqoFUd2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCfvbDwOhFZ7Zse/beuyu992n3XFXavvRrd1Z6f/QOR3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJ+9kNAu3vSr7/jZx2/99kbLi1tX/jdF8rXHyxf/7eLftmy7c0rzyldd85d5dvG1HBkB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEk6Gc/BGz+1jGl7WW/3d7ufvR2/eg4fLQ9stteZXvE9qZx82bbXmv7tWJa/tcIoHGTOY2/T9KFB827QdK6iDhZ0rriNYA+1jbsEfGcpD0HzV4maXXxfLWki2uuC0DNOv2Cbl5E7JKkYjq31YK2V9oetj28V6Mdbg5AVV3/Nj4ihiJiMCIGBzS925sD0EKnYd9te74kFdOR+koC0A2dhn2NpBXF8xWSHqunHADd0raf3fYDks6XNMf2dkk3S7pV0kO2r5C0TdJl3SzycFf2u++S9Nel5b/9XnZPOv3oOKBt2CNieYumC2quBUAXcbkskARhB5Ig7EAShB1IgrADSXCLax945xv/rLT+jDu46RDtcWQHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSToZ++BdkMulw1rLLUfVnnWU+unXFNdbjrl8ca2janhyA4kQdiBJAg7kARhB5Ig7EAShB1IgrADSdDP3gO7zz+22vo7yu9Xn1Xhvdv9jPWnv7ehtL1suGj0F47sQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AE/ew9MOeu8mGTn7h+Rml7uyGbtXOqFY1X3o/ezhP/Ka+dfvj+0fbIbnuV7RHbm8bNu8X2DtsbiseS7pYJoKrJnMbfJ+nCCebfFhGLiseT9ZYFoG5twx4Rz0na04NaAHRRlS/orrG9sTjNb3nxtu2VtodtD+/VaIXNAaii07DfKelESYsk7ZL0o1YLRsRQRAxGxOCApne4OQBVdRT2iNgdEfsj4j1Jd0taXG9ZAOrWUdhtzx/38hJJm1otC6A/tO1nt/2ApPMlzbG9XdLNks63vUhSSNoq6cou1njY+8lJp5W2/6DNPef/OKf1/8aZOzoq6X/mPftGafue28rXX1rym/hHbdvXSUnoUNuwR8TyCWbf24VaAHQRl8sCSRB2IAnCDiRB2IEkCDuQBLe4HgKmtxmSeeFT3dv2/rZLlA9HXabdfxfqxZEdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Kgnx2V3HTK46XtZ2+4tGXbLG2puxyU4MgOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0nQz45K2g3JfM2OliODaVbdxaAUR3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJ+dpR688pz2iyxobR19jB/Yv2i7ZHd9gLbz9jebPtV29cW82fbXmv7tWLa+uoJAI2bzGn8PknfjIhPSjpb0tW2T5d0g6R1EXGypHXFawB9qm3YI2JXRLxcPH9b0mZJx0laJml1sdhqSRd3q0gA1U3pCzrbH5d0hqQXJc2LiF3S2D8Ikua2WGel7WHbw3s1Wq1aAB2bdNhtHynpYUnXRcRbk10vIoYiYjAiBgc0vZMaAdRgUmG3PaCxoP88Ih4pZu+2Pb9ony9ppDslAqhD234R25Z0r6TNEfHjcU1rJK2QdGsxfawrFaJRA8veqLT+Udv21VQJqppMJ+i5kr4i6RXbBzpVb9RYyB+yfYWkbZIu606JAOrQNuwR8bwkt2i+oN5yAHQLl8sCSRB2IAnCDiRB2IEkCDuQBPcfolS7IZmf+M+M0vbpT62vsxxUwJEdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Kgnz250YvOLG1f+qHyn4o+7Z6rStsX6oUp14Tu4MgOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0nQz57cP86p9icwc0dNhaDrOLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKTGZ99gaT7JX1U0nuShiLidtu3SPq6pAMDeN8YEU92q1B0x4cHy8dfb/e78POeLV9//5QrQrdM5oqKfZK+GREv2z5K0ku21xZtt0XED7tXHoC6TGZ89l2SdhXP37a9WdJx3S4MQL2m9Jnd9sclnSHpxWLWNbY32l5l+5gW66y0PWx7eK9GKxULoHOTDrvtIyU9LOm6iHhL0p2STpS0SGNH/h9NtF5EDEXEYEQMDmh6DSUD6MSkwm57QGNB/3lEPCJJEbE7IvZHxHuS7pa0uHtlAqiqbdhtW9K9kjZHxI/HzZ8/brFLJG2qvzwAdXFElC9gnyfpN5Je0VjXmyTdKGm5xk7hQ9JWSVcWX+a1dLRnx1m+oGLJAFp5Mdbprdjjidom823885ImWpk+deAQwhV0QBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJNrez17rxuw3JP1t3Kw5kt7sWQFT06+19WtdErV1qs7aFkbEsRM19DTs79u4PRwRg40VUKJfa+vXuiRq61SvauM0HkiCsANJNB32oYa3X6Zfa+vXuiRq61RPamv0MzuA3mn6yA6gRwg7kEQjYbd9oe0/2d5i+4YmamjF9lbbr9jeYHu44VpW2R6xvWncvNm219p+rZhOOMZeQ7XdYntHse822F7SUG0LbD9je7PtV21fW8xvdN+V1NWT/dbzz+y2p0n6s6QvSNouab2k5RHxh54W0oLtrZIGI6LxCzBsf07SvyTdHxGfKuZ9X9KeiLi1+IfymIj4dp/UdoukfzU9jHcxWtH88cOMS7pY0lfV4L4rqevL6sF+a+LIvljSloh4PSLelfSgpGUN1NH3IuI5SXsOmr1M0uri+WqN/bH0XIva+kJE7IqIl4vnb0s6MMx4o/uupK6eaCLsx0n6+7jX29Vf472HpKdtv2R7ZdPFTGDegWG2iunchus5WNthvHvpoGHG+2bfdTL8eVVNhH2ioaT6qf/v3Ij4rKSLJF1dnK5iciY1jHevTDDMeF/odPjzqpoI+3ZJC8a9Pl7SzgbqmFBE7CymI5IeVf8NRb37wAi6xXSk4Xr+p5+G8Z5omHH1wb5rcvjzJsK+XtLJtk+wfYSkyyWtaaCO97E9s/jiRLZnSvqi+m8o6jWSVhTPV0h6rMFa/k+/DOPdaphxNbzvGh/+PCJ6/pC0RGPfyP9F0neaqKFFXZ+Q9Pvi8WrTtUl6QGOndXs1dkZ0haSPSFon6bViOruPavuZxob23qixYM1vqLbzNPbRcKOkDcVjSdP7rqSunuw3LpcFkuAKOiAJwg4kQdiBJAg7kARhB5Ig7EAShB1I4r9y5LzMbugUqgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "n = 71\n",
    "plt.imshow(np.array([x_train[n][i-28:i] for i in range(28,785, 28)]));\n",
    "print(int(y_train[n][0]), \" - answer\")\n",
    "print(net.forward(x_train[n][np.newaxis,]).argmax(), \" - network output\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Your image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.image as mpimg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAANdElEQVR4nO3db4gc933H8c+nbsIJJw+k6mwdjvClwZiYQpWwyAWXoBIabD+R8yBFehAsY6pI2BCbQCu5D85+5KNtGudBESiNfUpJHQKJiR8YNUYITB44eG1UW65o7Zptolh/VhUiDkiktr99cONylm9nTjszO3v5vl9w7O78dne+Gt3nZne/s/NzRAjA777f67oAAJNB2IEkCDuQBGEHkiDsQBK/P8mVbd68Oebn5ye5SiCVwWCgCxcueLWxWmG3faekb0u6TtI/RcRi2f3n5+fV7/frrBJAiV6vN3Js7Jfxtq+T9I+S7pJ0m6Tdtm8b9/kAtKvOe/btkt6MiLci4reSfiBpZzNlAWhanbDfJOmXK26fLpZ9iO29tvu2+8PhsMbqANRRJ+yrfQjwkWNvI+JwRPQiojc7O1tjdQDqqBP205K2rrj9KUlv1ysHQFvqhP0lSbfY/rTtj0vaJenZZsoC0LSxW28R8a7tByX9q5Zbb09GxOuNVQagUbX67BHxnKTnGqoFQIs4XBZIgrADSRB2IAnCDiRB2IEkCDuQxES/z47xLC0tlY4/9thjI8cGg0HpY6vOL7CwsFA6vmfPntJxTA/27EAShB1IgrADSRB2IAnCDiRB2IEkaL0Vjh49Wjq+f//+kWNV7a1pVlV72b97LWjNTQ/27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQRJo+e1Uffffu3aXjly5darKcdePKlSul43X68PTgJ4s9O5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4k4YiY2Mp6vV70+/2JrW+lubm50vGzZ89OqJLm7dq1a+TYrbfeWvrYstNQN2FmZmbk2OXLl1tdd0a9Xk/9ft+rjdU6qMb2QNI7kt6T9G5E9Oo8H4D2NHEE3Z9FxIUGngdAi3jPDiRRN+wh6ae2X7a9d7U72N5ru2+7PxwOa64OwLjqhv2OiPi8pLskPWD7C1ffISIOR0QvInqzs7M1VwdgXLXCHhFvF5fnJT0jaXsTRQFo3thht3297U9+cF3SlySdbKowAM2q82n8jZKesf3B8/xLRJR/abxDbfbR9+3bVzp+6NCh1tZdV1kfXJIOHjxY6/mrvg+PyRk77BHxlqQ/brAWAC2i9QYkQdiBJAg7kARhB5Ig7EASaU4lXdViqtMiuvnmm0vH60wHvRYLCwsjx6pO13zgwIHS8bqttzJLS0ul45xqulns2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgiTSnkn744YdLx5944omxn7uqh79ly5bS8cFgMPa6q9Zf93TNGzZsKB2vc3xC1XbjVNPXruxU0uzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiCJNH32qn5wVT+5jscff7x0vM3vjNf9/11cXCwdn+baM6LPDoCwA1kQdiAJwg4kQdiBJAg7kARhB5JI02ev0ub3trvU9v9vMWV3K+izX7tafXbbT9o+b/vkimWbbD9v+43icmOTBQNo3lpexi9JuvOqZQckHYuIWyQdK24DmGKVYY+IFyRdvGrxTklHiutHJN3TcF0AGjbuB3Q3RsQZSSoubxh1R9t7bfdt94fD4ZirA1BX65/GR8ThiOhFRG92drbt1QEYYdywn7M9J0nF5fnmSgLQhnHD/qyke4vr90r6STPlAGhL5fzstp+WtEPSZtunJS1IWpT0Q9v3S/qFpK+0WeQk7Nu3r3S8znnlgWlQGfaI2D1i6IsN1wKgRRwuCyRB2IEkCDuQBGEHkiDsQBKVn8ZnUXW650uXLo0cW1paariaa1PVNgQk9uxAGoQdSIKwA0kQdiAJwg4kQdiBJAg7kAR99sLMzEzp+FNPPTXWGDAt2LMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBL02VHq6NGjXZeAhrBnB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEk6LMnV9VH37171CS+WG8q9+y2n7R93vbJFcsetf0r2yeKn7vbLRNAXWt5Gb8k6c5Vln8rIrYVP881WxaAplWGPSJekHRxArUAaFGdD+getP1q8TJ/46g72d5ru2+7PxwOa6wOQB3jhv2QpM9I2ibpjKRvjrpjRByOiF5E9GZnZ8dcHYC6xgp7RJyLiPci4n1J35G0vdmyADRtrLDbnltx88uSTo66L4DpUNlnt/20pB2SNts+LWlB0g7b2ySFpIGkr7VY45pU9Yv3799fOj4YDBqs5trMz8+Xji8sLJSO79mzZ+RY3T562bz0dTGv/GRVhj0iVvtt+G4LtQBoEYfLAkkQdiAJwg4kQdiBJAg7kMS6+oprWRupyxZSXVVtv6q24ZYtW0aOdb1dytprhw4danXd+DD27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQxLrqs993330jx6a5j17XlStXSsfLeuld9tEleunThD07kARhB5Ig7EAShB1IgrADSRB2IAnCDiSxrvrsZ8+ebe25d+zYUTp+/PjxsZ97w4YNpeNVffQqXZ7umT76+sGeHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSWFd99pmZmZFjdXvVL774Yun40tLSyLGy87ZL5XVL9Wuvo6q222+/fUKVXLu603SXqTNN9rSq3LPb3mr7uO1Ttl+3/fVi+Sbbz9t+o7jc2H65AMa1lpfx70r6RkR8VtKfSHrA9m2SDkg6FhG3SDpW3AYwpSrDHhFnIuKV4vo7kk5JuknSTklHirsdkXRPW0UCqO+aPqCzPS/pc5J+LunGiDgjLf9BkHTDiMfstd233R8Oh/WqBTC2NYfd9ick/UjSQxHx67U+LiIOR0QvInqzs7Pj1AigAWsKu+2PaTno34+IHxeLz9meK8bnJJ1vp0QATXBElN/Btpbfk1+MiIdWLP87Sf8TEYu2D0jaFBF/VfZcvV4v+v3+2MUuLi6OHDt48ODYz4vxzc/Pl46XtbCq2ldVrbU2p6Ouaklevnx57OduU6/XU7/f92pja+mz3yHpq5Jes32iWPaIpEVJP7R9v6RfSPpKE8UCaEdl2CPiZ5JW/Ush6YvNlgOgLRwuCyRB2IEkCDuQBGEHkiDsQBLr6iuuBw6M/10b+vDtGAwGpeNl02yXjXVtPX6FtQp7diAJwg4kQdiBJAg7kARhB5Ig7EAShB1IYl312cvU6cFL3fbh25wWuewcAFLe4w8yTkXNnh1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkqg8b3yT6p43HpO3nvv0Zb3038U+ulR+3nj27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQxFrmZ98q6XuStkh6X9LhiPi27Ucl/aWkYXHXRyLiubLnos8OtKvu/OzvSvpGRLxi+5OSXrb9fDH2rYj4+6YKBdCetczPfkbSmeL6O7ZPSbqp7cIANOua3rPbnpf0OUk/LxY9aPtV20/a3jjiMXtt9233h8PhancBMAFrDrvtT0j6kaSHIuLXkg5J+oykbVre839ztcdFxOGI6EVEb3Z2toGSAYxjTWG3/TEtB/37EfFjSYqIcxHxXkS8L+k7kra3VyaAuirDbtuSvivpVET8w4rlcyvu9mVJJ5svD0BT1vJp/B2SvirpNdsnimWPSNpte5ukkDSQ9LVWKgTQiLV8Gv8zSav17Up76gCmC0fQAUkQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkpjolM22h5L+e8WizZIuTKyAazOttU1rXRK1javJ2m6OiFXP/zbRsH9k5XY/InqdFVBiWmub1rokahvXpGrjZTyQBGEHkug67Ic7Xn+Zaa1tWuuSqG1cE6mt0/fsACan6z07gAkh7EASnYTd9p22/8P2m7YPdFHDKLYHtl+zfcJ2p/NLF3Ponbd9csWyTbaft/1GcbnqHHsd1fao7V8V2+6E7bs7qm2r7eO2T9l+3fbXi+WdbruSuiay3Sb+nt32dZL+U9KfSzot6SVJuyPi3ydayAi2B5J6EdH5ARi2vyDpN5K+FxF/VCz7W0kXI2Kx+EO5MSL+ekpqe1TSb7qexruYrWhu5TTjku6RtEcdbruSuv5CE9huXezZt0t6MyLeiojfSvqBpJ0d1DH1IuIFSRevWrxT0pHi+hEt/7JM3IjapkJEnImIV4rr70j6YJrxTrddSV0T0UXYb5L0yxW3T2u65nsPST+1/bLtvV0Xs4obI+KMtPzLI+mGjuu5WuU03pN01TTjU7Ptxpn+vK4uwr7aVFLT1P+7IyI+L+kuSQ8UL1exNmuaxntSVplmfCqMO/15XV2E/bSkrStuf0rS2x3UsaqIeLu4PC/pGU3fVNTnPphBt7g833E9/2+apvFebZpxTcG263L68y7C/pKkW2x/2vbHJe2S9GwHdXyE7euLD05k+3pJX9L0TUX9rKR7i+v3SvpJh7V8yLRM4z1qmnF1vO06n/48Iib+I+luLX8i/1+S/qaLGkbU9YeS/q34eb3r2iQ9reWXdf+r5VdE90v6A0nHJL1RXG6aotr+WdJrkl7VcrDmOqrtT7X81vBVSSeKn7u73nYldU1ku3G4LJAER9ABSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBL/B6DvihIhnLxLAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "image = mpimg.imread('mnist/2.png')\n",
    "vector_img = np.array(1 - image[:,:, 0]) * 255\n",
    "plt.imshow(image);\n",
    "net.forward(vector_img.reshape(784)[np.newaxis,]).argmax()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}