ml-ex3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi-class classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This exercise is described in [ex3.pdf](data/ml-ex3/ex3.pdf)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy as sp\n",
    "import scipy.io as sio\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression, LogisticRegressionCV\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hand-written digits dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load the hand-written digits dataset\n",
    "digits = sio.loadmat('data/ml-ex3/ex3data1.mat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Digit image data (5000 images with 400 features/pixels)\n",
    "X = digits['X']\n",
    "\n",
    "# Digit classes (1-10) where digit 0 is assigned class 10\n",
    "y = digits['y'].ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAALsAAAJBCAYAAAAeBsQ8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXmYFNXV/z9nhvcXIogRUImKYEAUFUERFJW4BJeocYt7\nUAloXKNJXN7EGNS4E0VJDGrexF2Dihh3cQsqYhRwQRAENwRBZBsGRjDK3N8fXV/vdDHDzPT0Ut19\nv89Tz0xXVVffc6vqe88959xzzDlHQEA5oKLQDQgIyBfCwx5QNggPe0DZIDzsAWWD8LAHlA3Cwx5Q\nNsj4YTezIWY2MZuNSSqCrKWBomd2MzvWzCaZ2ZdmNqGe433MbGp0fKqZ9SlAM7OCIGva8WbLWvQP\nO7AMuAm4Nn7AzP4f8ChwL7AxcBfwaLS/GBFkpQWyOuca3YDOwDhgMbAUuBkYAkysc84oYB5QDUwF\nBtY51h+YEh1bBIyM9reOGrwUqAImA5s1pU31tPFUYEJs3wHAZ4DV2fcpcFCQtbxkdc41zuxmVgk8\nAcwFugJbAGPqOXUy0AdoD9wPPGRmraNjo4BRzrl2QDfgwWj/KcBGUad3AM4AVke/O9rMqhrYpjXW\n7gg7ANNcekzEtGh/kLVMZBWaosb0BzYHLnTO1Tjn1jjn1pnAOOfudc4tdc5945y7AfgOsG10+Gug\nu5l1dM6tcs79p87+DkB359xa59xU51x1dL2znHPfa2DbqQntBmgLrIjtWwFsGGQtK1mBpj3snYG5\nzrlv1neSmV1gZjPNbIWZVZF6sztGh4cBPYBZZjbZzA6N9t8DjAfGmNkCMxthZv/ThDY1FauAdrF9\n7YCVDZwfZI2hRGRNoQk60wDgC6BVbP8QIt0OGBid0wuoiPYtBwbFvlMBHA2sAdrEjnUF3gOGRZ9v\njYSqb5vRDN1uPum63Vwa0O2CrKUpq7amMPsbwELgWjNrY2atzWzP2DkbAt+Qmui0MrPh1HnzzGyw\nmW3inKslNWEBqDWzfc2sV6Q/VpMa/moBnHNnOOfaNrDtUOfalZEO2QqoiNonFpkArAXONbPvmNk5\n0f4Xg6xlJWsKjTF79NZsBfyL1Ox6CfBn0hmgErg9EmwhcBHwCREDkJqZf0H09gJHRPtPAN4HakjN\n5v9MjGma0LYhgIttd9Y5vjMpK8Jq4E1g5yBr+cnqnEsNAwEB5YBScCoFBDQJ4WEPKBuEhz2gbBAe\n9oCyQXjYA8oGrVryZTNzFRXJfV9qa2uXOOc2yca1gqzJQaaytuhhr6iooE2bNi25RE6xcuXKudm6\nVkVFBd/97nezdbmso6amJsjaCFr0sOcKtbW1AKxduxaAyspKIHUTShWSuSEUg+zx+yaYGeDvoz7n\nG8nvwYCALCFRzC5GaNu2LQCbbJJSy7744gsAvvzyS6A4WK4xiAX/+9//AnyrDrZu3TrtvDVr1gBe\n9u985zv5amKTofu25ZZbAvCHP/wBgH79+gFw3XXXATB27FgAufvXGblzzfjF/9QEBDQRiWB2sdsO\nO6SC3i655BIA9ttvPwD+/e9/A/C73/0OgE8++QSA//mfbIZI5xZi8m++SYWPb7zxxgD07dsXgMMO\nOwzw7KjRa+HChQDcfvvtALz99ttA4fTe+tCqVeoxuuKKKwA45phjAM/gw4cPB7wsr776KgA9e/YE\n4OOPPwb8c5CrkTswe0DZoCDMrjf+q6++AqBHjx4A3HjjjQDsscceACxZsgSAAw44AIAJEyYA8I9/\n/CPtOkliuTjUxo022giA/fffH4CTTz4ZgB133BGADTbYAPDsJ7Y77rjjAGjXLhVG/otf/ALwfVfI\n+UtcV5eOrtFrzpw5ADz88MMAVFdXA15HP/300wE/R/v9738PwOLFi4HsyxaYPaBskFdmF8vpzT7o\noIMAuPDCCwHYc8/UQpmvv/4agPfffz/t75VXXgnA6tWrAbjrrrsA+H//L3mpUaR/du7cGYBx48YB\nsPXWWwOetZ566ikA/vznPwPw4YcfAp4FxeyjRo0CPIuKNQvJ7NLV1SbdB1mMRo4cCcA999yTtl/3\n76233gLg4osvBmCrrbYCoKamJu28bI3cgdkDygZ5ZXYx9vbbbw/AbbfdBnh7+htvvAFA+/btAdhs\ns83Szttnn30AzyRxT10SIBk7deoEwK9//WvAs5YY+e677wbggQceAGDZsmWAZ0tZb158MbWscvLk\nyQD89Kc/BeCaa67JoRTrh2T8wQ9+AMAZZ5wBeCbWqLR06dJ6v6/RaN68eYAfEc4880wArr76agA+\n+OADwPdJSxGYPaBskBdmFwNLf5XV5fvf/z4AL7zwAgAXXXQRAP379wfgvffeA+C1114D4Oc//zkA\nP/zhDwHPnmJFzQUKAVlHtt02lT9I7BS3JMlXMGPGDMC3OT7vELNrniM2/PGPfwwUltmlQ2uOpTbJ\n76H7OWXKFGBdZpYsixYtAqCqKpWYQL6Gxx9/HICZM2fW+/1MEZg9oGyQU2YXo2+66aaA96Rtt912\nANx6660APPPMMwDMmjUL8G90/DrSFXfffXcAdt11VwCefPJJoDDMLgaWTi4ZDz00lRxL3l/ZxxXn\nIxZsyNKg62qk6NWrF+CtMoWARhmF/5599tmAl0X75eFesGABwDph4JJZFivdV/kadt55Z8AzfLb8\nKYHZA8oGOWV2edKkt5544omAj325+eabAa/DNcTMYgCxnXQ4RQgWMveN2nbaaacBXu+Uviobshi9\nqfqnZO3evTvgZXz66aeBwnqNxcDxCM1Vq1YBfq7VkKySTd+X/V0ybb755oCfxyjyMzB7QEATkRNm\n15sra8svf/lLwL/x9913H+CZXwzQ0JvbkD1d3y8ExOjSL4844ggAVq5MJZKV91AM39RlbtJf5Wkd\nOnQo4CMFP/roI6AwnlO1TbZ+Wdd03xSR+cgjjwDrWpjilqVDDjkE8LLm+n4GZg8oG+SU2RXRJ0/b\n6NGjAVi+fHnqxxtgdDGAmKRbt26At3i8/PLLgGfNQsTGqG2K9JOF6bHHHgPgiSeeANbVaxuCRoou\nXboAcP3116d9luUq2/EizYFG2IEDBwJe19b9fu655wDvOdV90XH1maxzio3Scc3Z5DfR+SE2JiCg\nmcgJs8ftoppNv/POO4BnJ83qhTgDaES44YYbAM8s8h4q9ruQUY+yMattsqvLo9rYmlGdJwaXrn/g\ngQcCcMsttwA+crCx+U0+oPs7f/58AD777DMA3n33XWBd3Vt2dq1TuOyyywC/Mk0yKYb/X//6F+D7\nJlv3NzB7QNkgp3Z2sY9WxktXj+cXkb4qi8XPfvYzAM45J1VQYe7cVE6cww8/HPA26yTFsUtWySD2\ni+eDie/v3bs34NlOeqzi3DWKyYJRSEaXTj1xYqrO2MEHHwx4ZteILJkUOyMdX3/F9Br1FO2q9QqT\nJk0Csn9/A7MHlA1yyuxiMcWnn3TSSYBnAkEeVr35WnGvlSyXX3454KPkkpQ7Jb4WVGwnT6csE3Hv\n8KBBgwAfyakYGEVFKsdKfN1mIaH5iVZdKTZGlijNrdQXssOLobUCSTEz0tXlUX/llVeA3N3fwOwB\nZYOcMLv0yvhaQuncilcXW22xxRaAj/FWZJ9W86xYkarvmiRGl24ub6E8qWJqWWXi3l+xnv5qlBOj\n33HHHWnHk5QbR/dVdnCNuMceeyzgV5DJ+qZYGTH41KlTAW+V00okeYdzfX8DsweUDVpULa+ystLV\nl7Ja19SbqvgOrTFULITWII4ZMwaA+++/H/D2WnkfM40DWbly5VTn3K4ZfTmGyspKV198Szw/5bBh\nwwAYMGAA0HBEptaWSk+dPXt22vHmWl1qampyLmscsijpPnfo0AHw1jV9FrNPmzYt7Xsa2Zu7EilT\nWQOzB5QNcsLsgt5gsZTWjIoFZX/X7Fyet2zpqflgdkEML5mVASwO9bfmIeqblnpGC8HsQkM+hfjn\nbN3XwOwBAY0gp3Z26dp682V5iMfOSHdLkuWhuYhXB5EloiEkyfvbUsTvo5AE30BdBGYPKBvkJW9M\nXC8tZUjWJGcWLlcEZg8oG7TIGmNmiwGV6esAdATez0K7soUuWawNGmRNDjKT1TmXlQ0YAkzM1vVa\n0I5jgUnAl8CEHP1GkLUIZS1FJXoZcBOwHbBfgduSawRZm4GMdHYz62xm48xssZktNbOb6zlnlJnN\nM7NqM5tqZgPrHOtvZlOiY4vMbGS0v7WZ3Rtds8rMJpvZZs1pm3Pueefcg8CCTGSrR44ga/o5RStr\nsx92M6sEniCl03UFtgDG1HPqZKAP0B64H3jIzLTUfhQwyjnXDugGPBjtPwXYCOhMSlc8A1gd/e7o\nqKPq26Y1V46mIMhaYrJmoDsNABYDrZqj2wHLgd7R/y8DlwMdY+cMJaWX7ZQFHe9UWqjHBllLS9ZM\n1JjOwFzn3HrTN5nZBWY208xWmFkVqTe7Y3R4GNADmBUNaYdG++8BxgNjzGyBmY0ws0K6VYOsMRS1\nrBkywBeshwGAgdE5vYCKOgwwKPadCuBoYA3QJnasK/AeMCz6fCuwqoFtRg7ZLshaIrJmwuxvAAuB\na82sTTT52DN2zobAN0TDopkNB9rpoJkNNrNNnHO1QFW0u9bM9jWzXpH+WA18DdQCOOfOcM61bWDb\noc61KyMdshVQEbUvUxYJspaQrM1+2J1za4GfAN2BT4H5wHGx08YDzwCzSU141gDz6hw/CJhhZqtI\nTWqOd86tBjoBY0l1yEzgJVJDYHNwEqnJzy2kmGg18H/NvAYQZKXEZG2RBzUgoJgQYmMCygbhYQ8o\nG4SHPaBsEB72gLJBeNgDygYtino0M1eI2j5NRW1t7RKXvRjvspLVErzSyjmXkawtetgrKiqaXBir\nEKipqZnb+FlNQ0VFRZNLxhQCX375ZdZkNbNEy7p69eqMZE1EPLtYRDZ/5RuJ5yQPPoHihHLqFDrH\nfHLH5YCALCMRzC7G1l/VWopXti5lSNZ49t5SGM2UCU7ZnFUrNt8MH5g9oGyQCGaPV8e79tprAbj0\n0ksBeP/91ML2pGWYygakz6qSXFVVKliwmGXWaPS9730P8JU6brrpJgDuvfdeoOlVv7OFwOwBZYNE\nMLv01JNPPhmAnXbaCfAVHEoR0tF/9KMfAb769/jx4wG44IILgHXzYhYDNFqpzqkqqyjXZ6H8FYHZ\nA8oGBWV2sZYcGLvssgsAn376KeArxRUTqzUGWZaUo/6UU04B/HxF1feUq74Y82NKxj59+gA+//70\n6dOBwmVrDsweUDZIBG3oTZdup9qhpcToghi7R48egLfCLFy4EICXXnoJ8HpvMTK7RuzGqn3nG4HZ\nA8oGBaUNveli9M02S2VEU53NUsY222wDeNlVNU/WmCTVfM0Uqi6SlBE6MHtA2aCgzC69dN999wWg\nffv2gPceqi5RUpihJZDequqCqg0re/vEiROB7FcMLAQ0Yvfs2RPw845Cx/kEZg8oGyRiqh+veCxm\nV3ScoiBLAfIeSmdXJOBjjz0GlMYoJrS0tmu2EZg9oGyQCGaPv/li+KQwQjag+Unfvn0B2HDDDQF4\n4oknAPjggw+AwsWN5AIasTfeeGPAx7XPnj0byL+spdOzAQGNIBHMXmjPWj6gUeqkk04CvHfx5Zdf\nBmDFihVp+4sZklVzr6OOOgqAfv36ATBr1qyCtCswe0DZoKDMLgYoJWtLHLKj9+/fH4C99toL8Owm\nz2kx29XjkCxjx44FfGz+0UcfDXjL05dffgnkT3cPzB5QNigIs8uTptiJAQMGAH795YcffgiUhmVC\n8xGtvtp8880Bvw5TDF9Ko5vu25w5cwC/PqFDhw5A4TInFP/TFBDQRBSE2aWrK6vAtGmpcpdas/j5\n55+nnVeMkF1dkZzHHZeq2CKP6f333w/40a0UoVFt5syZgGd2IeSNCQjIERIRzy5dXbbmJUuWAMWt\ns0sfVbx6r169AL/C/p133gGKMy9MU6E+kEVKunqhcncW79MUENBMJMKDquwC0tXledMK/ELHQWcC\ntVleQ2XH+uc//wn4nDjFPC9pCJJJsfmK+znggAMAf1/zvSItMHtA2aCgzC5LxJ133gnAmDFjAG9z\nLkZGFzTfkKVp6tSpgPceyhJVSp7TONQH119/PQD33XcfAIsXLwbynzkhMHtA2SARsTHS3cTkSVmz\n2BLIyiJGV0yM9pcyowu6v8rstnz5ciBkBAsIyDkSkesxbk8vZkaPQ7IVs8+gpUhKH5TvHQgoO1hL\nWNTMFgMq09cB6Ai8n4V2ZQtdslgbNMiaHGQmq3MuKxswBJiYreu1oB3XA3OAlcAs4OQc/EaQtQhl\nTYQHNcuoAX4CzAb6Ac+Y2QfOuUmFbVZOEGRthqwZ6exm1tnMxpnZYjNbamY313POKDObZ2bVZjbV\nzAbWOdbfzKZExxaZ2chof2szuze6ZpWZTTazzZrTNufcpc65Wc65Wufc68ArwIBM5AyylpisGQwn\nlcA7wI1AG6A1sBex4Q4YTErfawWcD3wOtI6OvQacFP3fFtg9+v904HFgg+h3+gLtomOjgaoGtmkN\ntPW7wELgoAyHziBrCcmaSacMABYDrWL70zqlnu8tB3pH/78MXA50jJ0zFJgE7JTJDavnN+8CniGa\niAdZy1vWTNSYzsBc59w36zvJzC4ws5lmtsLMqoCNSM3qAYYBPYBZ0ZB2aLT/HmA8MMbMFpjZCDPL\nyN1mZn8CdgSOdVEPZYAg67q/VbyyZsgAX7AeBgAGRuf0AirqMMCg2HcqgKOBNUCb2LGuwHvAsOjz\nrcCqBrYZse9eDkwHOrSQQYKsJSRrJsz+Bil96VozaxNNPvaMnbMh8A3RsGhmw4F2Omhmg81sE+dc\nLSndDKDWzPY1s15mVglUA18DtQDOuTOcc20b2Haoc+3fASdGN2BpBvIFWUtU1mY/7M65taRMQN2B\nT4H5wHGx08aT0qlmk3JOrAHm1Tl+EDDDzFYBo4DjnXOrgU7AWFIdMhN4idQQ2BxcDWwFfGBmq6Lt\n4mZeAwiyUmKytsiDGhBQTAixMQFlg/CwB5QNwsMeUDYID3tA2SA87AFlgxZFPZqZK/Tqk/WhtrZ2\nictejHeQNSHIVNYWPewVFRWJLotSU1Mzt/GzmoaKiopEp5VetWpVkLUR5CWeXRltG6udJDYp5vyH\n8ltI5rgfI74e00ooI5hk1X3W58Z8ObrfuR5NkjtWBQRkGTll9q+++gqAfffdF4Dhw4cDviKc8sMo\n/+Edd9wB+Gy++c4Y1RKIySXbFVdcAcAPf/hDwFfaUF52ZSoWink0E9QHqnvapk0bwOfyjI9myoqm\nzMbKDZmr0S4we0DZoEWxMZWVla6+Care0P322w+AX/3qVwAMHJhawSUGEPSGT5w4EYCLLroI8LWV\nMq1OUVNTM9U5t2tGX46hsrLSrW/SJhkGDRoEwC233AL4ahPKQT9//nzAj2Zi+pqamrTrNZfdVq1a\nlTdZ49D9lOznn38+AN26dQNgyy23BHwfCRr5r7vuOsDnhGyswnmmsgZmDygb5EQpVqWFYcOGAbD3\n3nsDMGHCBABefPFFAFauXAnAWWedBcA+++wDwMiRIwE/IqjaWjHotartqXmH8rKLKXv27An4+Uv7\n9u0BGDVqFOBrLhWTlUbMfsoppwDQpUsXAJ599tm08zRyC3/84x8BOP300wF44oknAJ+fP9v3OzB7\nQNkgJ8yuLK1PPvkkAP/6178AePrppwFYtWoV4BnhrbfeAuAf//gH4HV7VUY+55xzgGQzuyxH//nP\nfwA44ogjALjssssA+NGPfgR4C4Vw7rnnArDzzjsDfpRbujS1GCfhnkzA13ZV/ShZ1UaMGAH4+6YR\n/zvf+Q7gK3Ecc8wxAOy6a0oNf++999K+ly0ktycDArKMnDK7Ki18+2MR++mv3tx3330XgAULFgCw\n1VZbAdCuXTuKBdKxJZNsxxqdjjrqKABOO+00ALp27Zr2/e233x6ATTfdFPDVKYqB2dVmtVVzMx1v\nKCe98rXny5uc3J4MCMgycuqibGqFBdnq4/b0WbNmZb1N+YJGL1llVDfq1VdfBbwdXlW9ZbXp378/\nAG+//XbadZIIMbasJ2effTbg/SPxuqeCLE6qjijE7fDZRmD2gLJBQWlDs3PZWWWDll197NixQHHZ\nnOMQM8urLDaT91Ae7A033BDwfRD3MicRFqt3On36dKBhD6hk6tOnD+Dt8vIqy5KVK6tbYPaAskFB\nmF1sJkbYfffdAe9NfOqppwCYNy+VfyfJFonGID1U85KrrroKgN69e6edt2jRIgDGjx8PeHYTG4ol\nk9wXjTGy7nv37t0B2HrrrQFfAVt9kCsZk9tzAQFZRl6ZPc7oJ5xwAgB9+/YFvK7+wAMPAD4SUB63\nYoIYWbr40KFDATj00FRi27ht+YUXXgDgpZdeAvwoJwuV4ojUd4VkeMmmtsRXGsm+Hrezx6FR77nn\nnkv7nCsEZg8oG+SV2fXmypuouJHNNktVHLnhhhsAHxVZjIwuC5O8v5dffjngI0A1WokdZa3RaHbQ\nQQcBcPjhhwPQsWMq9fnzzz8PwN///nfA92U+GT4eC9OjRw/A+wbkM5DdfcqUKYD3l2i/4t7lMb/9\n9tsB33eZrl9oDIHZA8oGeWF26XYDBqTqPV166aWAZ4hp06YBMGbMGKA47epivV122QXw0YzS0cXo\n8ZVhYvhrrrkGgE02SaVDka6vKEnFyhSib+JZAgYPHgzAmWeemXaeRuI990yldT/uuFTGa/WN7OmK\nan3wwQcBmDlzJpB7b3Fg9oCyQV7zxhx88MEAdO7cGfBvvFa0iP3WrFkDNF8fFTM0NSYnG5BsWmuq\nmHxFbkoPbWitr/bL5qw+0XW18kl9UwgrjNqoft1jjz0AH+Pym9/8BvCRnvHoVvkWFNMvy5L8KJKx\nbdu2OZQiMHtAGSGv1hi9wXEMGTIE8G/+J598AjSsn8aZW7riY489BsDdd9/d0qY2CjGwrC4XXngh\n4FfraJ4Sh9gubpMWk2tUk7Vl9OjRgJ/PiOHzOXqprYrnkU+gX79+gO//N954I+17nTp1AtYdpf73\nf/8XgFdeeQUgbykUA7MHlA3ywuxiM61F1SqdnXbaKe08rUrXbF2sJ4bX2tU482ul0+zZs4F140py\nAemxYiVZYTbaaCPAWx7E/MouoDbKuvLOO+8A8PHHHwN+na6Ox7MN5JPR4xDDK3bp2GOPBeAvf/kL\n4PtgxowZgF9Pq/nIJZdcAvjYfvVhvmL2A7MHlA1ykhGsIYhpxXKys6sN+rzddtsB68aBKKuWmDxu\nvxULiilymRFMbRbjKnJTXkSNPopxEcO//vrrgI9rVxYBtT2+uif+tyHkMyOY5hOKS5cO/uMf/xjw\nHlBlCVCGt0mTJqV9P1PLUsgIFhDQCPLK7ELcAiHWikfTxdGQ3iomj7NfPnI9qv9kTxdrad4Qz1ku\n1ovnom+pZ7QQuR51v+TllfVFMmkkVgyMZC+UrIHZA8oGBVmpFK8+8W1jIoYupmhHsZTaXExtbynE\n4LKfz5kzJ+14vG8KjcDsAWWD5CYlCSgaNDRSJw3Jbl1AQBbRImuMmS0GVKavA9AReD8L7coWumSx\nNmiQNTnITFbnXFY2YAgwMVvXa0E7RgDzgGpSN+ziHPxGkLUIZS1FNeYfwHbOuXbAHsDPzOyoArcp\nVwiyNgMZPexm1tnMxpnZYjNbamY313POKDObZ2bVZjbVzAbWOdbfzKZExxaZ2chof2szuze6ZpWZ\nTTazzZrTNufc+865utW4aoHumcgZZC0xWTMYTiqBd4AbgTZAa2AvYsMdMJiUvtcKOB/4HGgdHXsN\nOCn6vy2we/T/6cDjwAbR7/QF2kXHRgNVDWzTYm38LbAKcMBHwJYZDp1B1hKSNZNOGQAsBlrF9qd1\nSj3fWw70jv5/Gbgc6Bg7ZygwCdgpkxsWu5YBO0e/s2GG1wiylpCsmagxnYG5zrn6A1gimNkFZjbT\nzFaYWRWwEalZPcAwoAcwKxrSDo323wOMB8aY2QIzG2FmGQVwuxTeAlaT6phMEGSNoahlzZABvmA9\nDAAMjM7pBVTUYYBBse9UAEcDa4A2sWNdgfeAYdHnW0kNYfVtM9bT3kuAR1vAdkHWEpE1E2Z/A1gI\nXGtmbaLJx56xczYEviEaFs1sOPBtgSQzG2xmmzjnaknpZgC1ZravmfUys0pSJqavSU1EcM6d4Zxr\n28C2Q3TdCjM73cw2thT6A2cDL2QgZ5C1xGRt9sPunFsL/ITUTPhTYD5wXOy08cAzwGxSNtE1pGyk\nwkHADDNbBYwCjnfOrQY6AWNJdchM4CVSQ2BzcCTwIbASuBf4S7Q1G0HW0pK1RR7UgIBiQik6lQIC\n6kV42APKBuFhDygbhIc9oGwQHvaAskGLViqZmUvy6pTa2tolLnsx3kHWhCBTWVv0sFdUVOQtKWUm\nqKmpmdv4WU1DRUVFzlMqtwTV1dVZlVXpMZKIlStXZiRrWIMakDU0lA9Ivpz4WlXLcxWR5I5VAQFZ\nRmD2HENspyxn8Yxhym7WWHXoJEPMLTVP1Q8Fi2Vhrq6uBnwu+nwhMHtA2SBRzB6vjKy/Yo74Z7Fi\nIXOWNwQxuNhu2223BWDnnXcGfJZfVYpTPaK4bPnWazOBdPDf/e53AJx33nnAujq86qBef/31gK8m\nkqu6p+u0My+/EhCQABSE2aW/6q9yPH7/+98HfKU5VeYQy4kVZe58/PHHAXj00UcBz6aFZENl8z36\n6KMBX9l6yy23BKB799QaYVWKU4ZbVdoYO3YsAE888UTa9ZLM8GqbZFL1POWgP/HEEwHYZ599AN8n\nqjoSr5iSKwRmDygb5JXZpY+qdpIqbey2226Ar5zcrVs3wL/pqkqhz8oKO2jQIMBXafvoo4+Awujw\nkk36Z48ePQDYe++9AW+JeOihhwAYN24c4OunHnLIIQDceOONgLdUqAJgEuclgnTzv/3tb4AfYSWD\nKgiK2VVjSfOZfK2pCMweUDbIC7Przf/e974HwOWXpxaFH3HEEWnH33zzTQDuuSe1YmvRokWAZ7Wh\nQ4cCfnaJ3PhEAAAgAElEQVR/yy23AF43LKStOu4tVAU51XtSbSXp5LLGaN4iGVSXSFXAk6yrxxEf\n3fr37w/Accelr+774IMPAD/a5UvGwOwBZYO8MLvssHqTH3nkEQAmTJgAwKuvvgp4i4SYWzq5GF2W\niZtuugmAu+66C1i3Sl4hIZaSvqo2ap4hJhf7ab5ywQUXAN7urr4ppjXCku34448H4LLLLgNgww03\nTDvvr3/9KwCzZs0C8jcfCcweUDbIKxWKsVXpWkzdsWMqodT+++8P+MrJe+6ZSlvy2muvAXDyyScD\n8J///Cftuklg9DjiXl9ZKPr27Qt468sxxxwD+L649NJLAc96SZQtDsmo6t5HHZVKriu/iUZe9cl+\n++0HwPTp0wH49NNPgWBnDwjIGvJCG2J0vdlaGHD44YcD8POf/xzwrKc3/eqrrwZ8DMWyZctSjW6g\n7mkSIFllSz711FMB71vYZZddAF8zVJgyZQrgdXaxZTEwu+Zkqpr34IMPAt5bLEuU5im/+MUv0j6r\nQrbmOblaJRWYPaBskFPa0Oxcb7Z0cVlZdt99d8B70nS+Yl1uvjmVCz8e+51kiJHlHT700FQiW7Ge\nvL2vv/464GU67bTTAC+z2E92+mJgeI3csrY9/fTTALRu3RrwurqiIn/6058C8MwzzwA+Higwe0BA\nC5ETuhBD9+vXD4Df//73APzwhz8EPJvFox/FDOeccw7gdfcHHnggF83MCSSbGFyMvXDhQsBHBkpm\n6fgrV64E/DxFkZ9z56bWFieJ2dXm+FrS+N+vvvoK8NaYhx9+GPAMrxh/edZzPQcLzB5QNsgJXcS9\niIoAVPzy+PHjAc9+qnkvm7PYTSODrDHFBOnu8hGIBeORm+ojjQiK1dfnJHlQxeiKeVm+fDng71/c\nTh5nan1v1113Bfxop1VagdkDArKEnDC73nCtQDn44IMBWLJkCQArVqwAPFMceOCBgI+B0eodWWWS\naE9vCHEmbmh9pTymGvUUTyLZxZpJysyl0eqkk04CfJSqmD3uT9G8RD6GCy+8EIAddtgB8FYbXScw\ne0BAlpDTKb7edFkU4lYXWWdGjhwJ+HwjWq0jfbcYcqpIJuncYilZWQT1wQ9+8AMArrvuOsCvztL6\nTXlSk+BbEKNvskkqvaJi7Xv27An4FWIff/wx4O/jHnvsAfiRe/vttwe8ji4rW01NqpZvrkexwOwB\nZYOsMntcV4vn/Ntuu+0A/8ZfcsklgI950YqWV155Je16xaSzX3TRRYC3IUtPldVFfaDVWoofufji\niwEfV5KkvDFiXM25nn32WQD++Mc/AnDvvfcCfh6i8zfYYAPAy/Lhhx8C8Pe//x3I//rawOwBZYOs\nMHtcX1X0onQ85X9RnEjXrl0Br7Np5VFcT00CqzUV0mtnzJgB+Nj7O++8E/CsJ2aXnqoVSklk9DjE\n2M8//zzg769inqSr63nQutrnnnsOgNtvvx2AadOmAfmfjwRmDygbtKgOamVlpfvud7/7rY6uGG55\nSMUEeoPfeustAEaPHg3ASy+9BKzrRcwWampqpjrnds3GtSorK936ihGoH/VXEZ2K7JMlQiuWrrrq\nKsCvumppjH51dXVWZa2vGIHaplFKbVZeGHmFBd1X6fqK/GzpfV65cmVGsgZmDygbZIXZpa9qFbnW\nkmq/Iv2kq8WzAeTKvppPZhfUn3FvYhxiyWxFM+aD2eNoSNb46CQ/SbbmIYHZAwIaQVZoJZ4XJh5/\nruNisXzl4y4Ess3YSUaxyRqYPaBs0CKd3cwWAyrT1wHoCLyfhXZlC12yWBs0yJocZCRrix72tAuZ\nDQFOdc7tlZULthBm1p7UDXo/220KshYOLZG1lNWY64CZhW5EnhBkbQIyetjNrLOZjTOzxWa21Mxu\nruecUWY2z8yqzWyqmQ2sc6y/mU2Jji0ys5HR/tZmdm90zSozm2xmm8Wv3YT27QHsCNyRiXyxawVZ\n088pWlmb/bCbWSXwBCmdriuwBVDfItHJQB+gPXA/8JCZtY6OjQJGOefaAd2AB6P9pwAbAZ1J6Ypn\nAKuj3x0ddVR927RY+24GzgFapKMFWUtMVudcszZgALAYaBXbPwSYuJ7vLQd6R/+/DFwOdIydMxSY\nBOzU3HbVucavgVua0qYga3nJmoka0xmY65z7Zn0nmdkFZjbTzFaYWRWpN7tjdHgY0AOYFQ1ph0b7\n7wHGA2PMbIGZjTCzJgdSmNnmwLnA75spU0MIsq77u8Ura4YM8AXrYQBgYHROL6CiDgMMin2nAjga\nWAO0iR3rCrwHDIs+3wqsamCbEZ1zRHStz6NtBfDf6P/KIGt5y5oJs78BLASuNbM20eRjz9g5GwLf\nEA2LZjYcaKeDZjbYzDZxztUCVdHuWjPb18x6RfpZNfA1UAvgnDvDOde2gW2H6BpPR53ZJ9qGA28B\nfZxza4Os5S1rsx/26OI/AboDnwLzgeNip40HngFmk5rwrAHm1Tl+EDDDzFaRmtQc75xbDXQCxpLq\nkJnAS6SGwKa27Svn3OfaSDHA19H/zUaQtbRkzZpTKSAg6Shlp1JAQBrCwx5QNggPe0DZIDzsAWWD\n8LAHlA1atMSkoqLCJSnLbBxr165d4rIU411OsppZomWtra3NSNaWPuw0ZRFyobBixYq5jZ/VNFRU\nVHxb1DaJWLZsWVZlVcKrJKKmpiYjWRO1eFDZCCxWm6eUIBmLqc5pqSC5Y1VAQJaRKFpR1TRlnFJ+\nmVJgeHmqldlWsqrSRpJ15FJB6OGAskGimF352qdPnw7ArbfeCng2LGYoa5ZyPiofu+pNlYKMcWg0\na2hk1rxF9VJ1nnJGZntED8weUDbIK7PrTY1HWkpHV5ZX1cv8v//7v7Tzi1l3V9s1D1E231KC7pOy\nOgvxGrA63rFjaoGTKqAro9yIESOA7NdaCsweUDbIKbPLhnziiScCvvqdKmzouBhBucqvuOIKADbf\nfHMAvvjiC6A4quY1BDH5wIGpzBOlpKPHR94+ffoAsPPOOwMwdepUAN59913A38fDDjsM8DVgVbVE\nDi0xfbYQmD2gbJATZtebrmy9Rx11FODrZcraImbX39dffx3ws/O9994b8FmBi5HZZXEQk++zzz6A\nt68X8zxEsmnONXjwYMDXiVIlQFXVmzJlCuBrap166qmAt76omt7ChQvT9mcLgdkDygZ50dmle6u2\nTnx2rc/SzefMmQN4Fhwzpr7EVMUBsV/79u3T/v7pT38CklHBurnQyK37qxFYfpJOnToBvgriP/7x\nj7TzjzzySMAzvKwzYvRcrYsOzB5QNsipzq6Q2Pnz5wPw3nvvpX40FuknvVUs+NBDDwHw29/+Nu06\nssMXk54rK4x8B6odqr6I+x6SLFvcjn7eeecBcNZZZwH+vv7hD38AfN1TWVXkPdb5moPNmjULgJtv\nTuVR1WiX7T4JzB5QNsgJs4uhu3TpkvZ55cqVALRu3Tptf5zZJ0+eDPj6mttuuy3gZ/PFEAMeryB4\n3HGpfEOqBSuLk5i+uroa8JaNJEZBKr5HOvfpp58O+BH3r3/9KwB33XUX4GXZeOONATj77LMB7znV\n9XT/43VSdTxbSF6PBgTkCDll9p49ewJe595uu+0A2GqrrQBf675du1S6wKqqqrTjerNlwdB1iwFq\ne+/evQEYMGAA4Ctea3RTHMjVV18NwOzZs4FkrdbSvGObbbYB/FyqQ4cOgK9ze+655wJwwgknAN6f\novu3xx57AOuO6KqMfuONN6Zd97bbbgPgueeeA1o+ogdmDygb5ITZ9cbKrt6vXz8A7r77bsDPwpcv\nXw7AO++8A8Dnn3+etl+xMnvumUom+/jjjwPFVUdVjK6+EAuKpaS/jho1CvA+hfvvvx9YlwULAc0f\ndF8mTpwI+DmV/sp6onmIrC8a5eLrb+VR1yinUU/WnpdffhmAZ599NjtyZOUqAQFFgJwwu2bTTz75\nJOCZec2aNYC3uy5btgyAFStWAOvacXfbbTcAunXrBiTTQtEYpk1LlQVSHIj0WjG+7O/yMorFkpRd\nWSOx7tfw4cMB7w/RfZLl6f33UyVTTzvtNMBHP0r31/effvppwEc5akTQeZ988gmQPetb8T09AQEZ\nIqcGa9mS3377bWBdC4P+xmO79WaLUWSvl71W9tgkM71kk6dUq25kqZC3cOnSpQBMmDAB8Dq99Ngk\nWGME3Q+N0JpT6a90c8W8yK6u+yT7u+ZucTv7xx9/XO/vhZVKAQHNRE6ZXW9sSyP7ZH8vpjwyYiVZ\nYU455ZS042+88QYAI0eOBLxeWgwx+2Ja/ZV1RW0/88wzAb9iSbJqXiJGz3fEZ2D2gLJBooNMxOCy\n74rZi2H9ZjyaUbq7IF+BVuMkOZFoYxCzH3DAAYC3l+t+KUuEViJlewVSUxGYPaBskGhml76rWIli\niHaMQwzfkNf3pZdeAqBv375Asi1McWjU0qj0m9/8BvAxT2L0p556Cii857t4ejYgoIVIJFVqli5L\nhdiuGNdrNgSxnCL6lFulGKwxgphd90WMLqYXo8uaVui5VmD2gLJBIpldECMIxaTPNheanxQTs2s+\nIo/qLbfcAsD3v/99wFtfCq2rC6X79AQExJBoZi9lJhfEjsXE6EJ87bDWoOqz4nuSYkUr/acpICCC\ntSRu2swWAyrT1wHoCLyfhXZlC12yWBs0yJocZCarcy4rGzAEmJit67WgHXcC/wVW1dkqs/wbQdYi\nlLVU1ZgRzrm2dbbsJiBJFoKsTURGD7uZdTazcWa22MyWmtnN9ZwzyszmmVm1mU01s4F1jvU3synR\nsUVmNjLa39rM7o2uWWVmk81ss0zamC0EWdc5p3hlzWA4qQTeAW4E2gCtgb2IDXfAYFL6XivgfOBz\noHV07DXgpOj/tsDu0f+nA48DG0S/0xdoFx0bDVQ1sE2LDXfLom0q8NMWDJ1B1hKSNZNOGQAsBlrF\n9qd1Sj3fWw70jv5/Gbgc6Bg7ZygwCdipBTdtlzo342BgJbBnhtcKspaQrJmoMZ2Buc65b9Z3kpld\nYGYzzWyFmVUBG5Ga1QMMA3oAs6Ih7dBo/z3AeGCMmS0wsxFm1qyAGOfcm865pc65b5xzTwH3AUc1\n5xp1EGSNoahlzZABvmA9DAAMjM7pBVTUYYBBse9UAEcDa4A2sWNdgfeAYdHnW0mfidfdZqynvbcA\nI1vAdkHWEpE1E2Z/A1gIXGtmbaLJx56xczYEviEaFs1sONBOB81ssJlt4pyrJaWbAdSa2b5m1svM\nKoFq4GugFsA5d4ZLn4nX3Xaoc+2jzaytmVWY2QGkdMzHMpAzyFpisjb7YXcpc89PgO7Ap8B84LjY\naeOBZ4DZpJwTa4B5dY4fBMwws1XAKOB459xqoBMwllSHzAReIjUENgfnAZ+R6uw/Aac55yY08xpA\nkJUSk7VFHtSAgGJCqTqVAgLWQXjYA8oG4WEPKBuEhz2gbNCiqHozc0leYFFbW7vEZSnstaKiItGy\nrl27NsjaCFr0sFdUVNC2bduWXCKnqK6untv4WU1DRUXFt/nHk4iqqqqsyprk+7pixYqMZE3u6xsQ\nkGWEhz2gbJDTlbBaeKuyMXHEHVoWS0UdP66UDPHzAgKagsDsAWWDnBb9VbFXFQCLQ+nQVI4mPgLE\nmVyl0IupKIGgUaqxNuu8+KgW3x8vCBDQOEJPBZQNssrscfb65S9/CcBxx8WD51JQEh2VHYkzu1hr\nwYIFAAwbNgyAOXPmAMlOdBpnZrX1v//9b9pfnadRTAmFNCrqPI2O7dqlImpV9l19U4yQ7Lr/gjQD\nHVcCqVDOPSCgicgqs6ukowpHHXVUatWUyrTPmDED8Myv1Mbf+9736r3em2++mfY9XScp6dTqg0Yn\nsdMuu+wCwIUXXgjAFVdcAfgCt5JF5+2www5pn3WeGF7Mfv311wNw2WWX5UiSzCFG1lxMfSFYrECD\nStprpN9oo40AL6vuu4oAZ4rA7AFlg5zo7DvuuCPg3+wLLrgAgEmTJqV+tAFmtljRrfh+6b1JtMJI\n1q233hqA888/H4CePXsCvtT5j3/8YwB+97vfAX50k94q2VU0TWVo3n333bTrTJw4EfDsqFG1kBCD\ni5GPOeYYADbddNO081RATMWB9VfWOTG7+kJFKTRXyxSB2QPKBllhdr2BYqkBAwYA8MILLwDw8ssv\np34sYvT47FuIM3aSmVyQLCoAdtJJJwGw//77A/Dwww8DvuDtHnvsAXg99NNPPwXg1VdfBeC1114D\nYP78+QAsWrQI8An/4xaMJFmkNLpst912AFxyySUAbLnlloAvuKDS9qtWrQJ8Hzz66KMAvP322wB8\n8cUXACxevBjwI4JG0eYiMHtA2SArzB73mMoa8/HHHwNef5W9NM7U+r7Yrrq6GvAe0y+//BLw7JYk\nr6HY7OSTTwbgsMMOA+Ccc84B4LHHUtkeNKpJF1eJ87lzU9GqsrYI6iPJGu+zJBYv0Kij+7hixQrA\ny/bzn/8c8M+F9uv+xmWM90FL73tynpqAgBwjK8yuN06M/OCDDwIwePBgwFsg9KbGvYht2rQBfMyL\n8NFHH6Vd97777gPgn//8J5AMnV6yaxSSrNtvvz3gbcPvv5/K5a+iaNJDZVtOku7dVMTjdTQfufLK\nKwHo3bs34PtAfbVs2TLA6+D5KjAWmD2gbNCiJEmVlZWu7vKt+LVkE954443T9sdjYcRu0vUHDRoE\nwO677w5Aly5dAM/8Z511FgBPP/004HX+OMNXV1dPdc7tmpl06WjVqpWrb1meZBGTy6N5wAEHAPDZ\nZ58B8MADDwDeuqIiv7KfS7ZMGb6qqiqrsq5vWV6c0TVXe+aZZwBfGnLcuHGAvy+bb745AEOHDgW8\nTt/c+ceKFSsykjUwe0DZIKvMHoeu3RDzxiGWlIVj2223BTxLKr5EdtYTTjgBgHfeeQdYlxXzweyC\nRqkf/OAHgLc8HHvssQB06tQJ8H2wdOlSACZMmAB4m7SiGJvLdvlkduneuk/q98MPPxzwsj377LOA\nj33SXO4vf/kL4Mu9N3c0C8weENAIcho+GI91idtN49B+vemyxqiYrJj/mmuuAeBXv/oVAOeddx7g\nPXOFsMPrNxVnfumllwJw2223AX7+oVGqV69eABx55JGAn7fIA5tEyHqmdQqvv/46AP/5z38AuPPO\nO9POl29hs81S5ZPE8IofyjcCsweUDXLK7NI7d955Z2BdW3NjdnIdF2uOHTsW8BYPxUEXMr5d85Gt\nttoK8BF8igf68MMPAT9Kif0OPPBAAP7+978Dnuklk+JCkuQtFuQjuOiiiwAfWy+LkkYpxQv99Kc/\nBXxUo+J/8u0fSV5PBgTkCDmlRM3Wd9ttN8DrsZqNKypSumBDMTNiN9lp9Vkx3bLXFsKTqvmIVhQp\n9lptmzx5MuBHM0UE7rPPPmnfe/HFFwHPmklkdI2girE/5JBDAD8v0ZxK54nhH3nkEQDuuusuwM9r\n8h3fk7weDQjIEXJqZxezK2ZCMd2anT///POA9y5Onz4d8GwpPbh///4ADBkyBPBeSHniGso2kA87\nezyPyx/+8AcATjvtNMCzm+zwOk86ueJGtJpL622bOw/Jp51d91UrkAYOTBW5VqyLYvFnzZoFeLu7\n+qClMU3Bzh4Q0AhyyuyCmKBfv36AX5t40EEHAdC9e3fAW2uk+22zzTaAn+VrFn/DDTcA3r7bEEPk\n04OqflR/yJ6+1157AV4mMbdGMbHfypUrgcx19Xwyu6A5VXz9bDz+XPcnW3OqwOwBAY0gL8wuSGeT\nPipPmqLkxIa77pp6aRURqNm8WFGrfRqLg84nswvxzMXS2YX4qpy4LyFTFILZC4XA7AEBjSCvrkcx\nukaTDz74APB2V9lv44jH1ORrZUsmiLcxnkkhzvQB+UNg9oCyQUGCSsTU8qAlcaV8tpDknDflhsDs\nAWWDFlljzGwxkLWShDlAl2zVBg2yJgoZydqihz3tQmZDgFOdc3tl5YIJRpC1OFGSaoyZDTKzN82s\nxszmm9mxhW5TrhBkbTqSm9U/Q5jZ9sD9wCnAc8BGQP3VDoocQdbmyZoRs5tZZzMbZ2aLzWypmd1c\nzzmjzGyemVWb2VQzG1jnWH8zmxIdW2RmI6P9rc3s3uiaVWY22cw2a2bzLgFuc8497Zz7xjm31Dn3\nYSZyBllLS9ZmP+xmVgk8QWoC0xXYAhhTz6mTgT5Ae1Jv5ENmJo/KKGCUc64d0A14MNp/Cqk3tjPQ\nATgDWB397uioo+rbptX53d2j8981s4VRJ7dvrpxB1hKU1TnXrA0YACwGWsX2DwEmrud7y4He0f8v\nA5cDHWPnDAUmATs1t111rvFf4BOgB9AWeBi4L8NrBVlLSNZM1JjOwFznXP012iOY2QVmNtPMVphZ\nFak3u2N0eFjU6FnRkHZotP8eYDwwxswWmNkIM2tuPrjVwB3OudnOuVXA1cDBzbyGEGSNoahlzZAB\nvmA9DAAMjM7pBVTUYYBBse9UAEcDa4A2sWNdgfeAYdHnW4FVDWwz6nzvFWB4nc+7AMtbwHZB1hKR\nNRNmfwNYCFxrZm2iyceesXM2BL4hGhbNbDjQTgfNbLCZbeKcqwWqot21ZravmfWK9Mdq4GugFsA5\nd4Zzrm0D2w51fvsO4Odm9gMz2wD4LSldNBMEWUtI1mY/7M65tcBPgO7Ap8B8IF7CejzwDDCb1IRn\nDTCvzvGDgBlmtorUpOZ459xqoBMwllSHzAReIjUENqd9twN3A69Hv/0VcG5zrlHnWkHWdBS1rFnz\noAYEJB0l6UENCKgP4WEPKBuEhz2gbBAe9oCyQYsCwczMJTEnoVBbW7vEZS/G21mCVx0557Ima0VF\nRU7uq4wh+pvpb6xduzYjWVv0sFdUVHxbwj2JqKmpydoCBDP7Nr1bErFmzZqsyVpRUUFT0oY0F1p8\nrjJBenaaSyJVVVUZyZrIEF91SkPVnUsJYrl4Vi1B63OTPII2Bj3c3bp1A+CII44AfG56ZXzL9X0u\n3h4MCGgmEsHscTZTdlhVtl6zZg2wbt6ZYkY8l7mq6anatzKLLVmyBPC5IIthlNP9Ud59Mfrw4cMB\nn1NHefrzJVNg9oCyQaKY/Wc/+xkA55xzDuCrsamaxaeffgoUp/4qGcXYYrfDDjsM8HWilP9SuSLP\nOOMMAO6999607yUR8Xq3yt155ZVXAj7r8nXXXQesW0Ux1yi+pyYgIEMUlNmVt12V4v74xz8CvjJH\nz549AV+fSJXnkmwCjCPOXqqALSYfMGAA4GWW5ULMni9LRUugtkpWVff+/e9/D8Arr7wCwFVXXQX4\nqiP5HqUCsweUDQrC7GKCXXbZBfBvfPv2qfWzYghl9503LxUyXYy6ukYv6eKqGKc6Uar0d9NNNwHe\nAqW/Tz31FFDYWq8NQfdRlavPOusswNeTkqwjRowAvFWtUPOO4nt6AgIyRF7pQowtr6Cq3+23336A\nt8vKCnPxxRenfY5Xw0syZEdXm1XlWXWixOiqD/WnP/0J8GxpsUzHSdLZJZsqVst+PnjwYMDPR+Qh\n1egWz9Ycr8kk5Er2wOwBZYO8Mrve5BNOOAGA448/HvB1hsQU0lNViUP7iwEavcTo0mM1Sin4SXqs\nbM4Wq7GUZIiJ999/f8Dfx2uvvRaA0aNHA36OFWd0jeC6r9tvvz3g695+9tlnALz11ltA9uzxgdkD\nygZ5YXYx+hZbbAHAeeedB/g3W7ZlvfHyKk6blsp+9uyzz6Yam0CLRBySRbbmSy65BPAWiPHjxwNw\n2223AQ3XEI3LmgSdXfdx4403BuDqq68GYOrUqYCvYC5IFskm3b1Pnz4AnHtuKjmAPK01NTVp5518\n8skAvP3220DL739g9oCyQV6ZXYzQoUOHtP2CmF11UK+//nrAx8pIh0+iXis22mmnnQD45S9/CXhv\nr+zmn3zyCeDnLeqTZcuWAb7i9YQJEwBvnVFfFdLXENfVJdt9990HQFVVKi+S7o/O16i09957A36+\nsnDhQgBOP/10wM/dFEuj+rjS3VuKwOwBZYOC2NkbikfXrH3VqlWA9zrutttuALzwwgtAspg97jtQ\n5OaOO+4IeB1eFaOly+t81UWNz1umT58OwOTJkwEfN6S+KQTDa/Q69NBUvlKNQo8++ijgZdJ5Yv5f\n//rXgI/gvOeeVDIw2eM16snvIk+6KpoHO3tAQDORV2Zv6A0VI0jHk+ft2GNTJXN69+4NeHaMexmT\nAI02YnTJsmDBAmDd+J5///vfAHTp0gWAnXfeGYDu3bsD0LdvX8Dbnh9//HHAj275rJSt0UsWJf2N\nr5/VfenRowfgGX3gwFRxDllvFDOj82WNOfPMMwE/B5BOn606uYHZA8oGeWF2MXA87vnbRkT2U9lT\npdPJ3i6223zzzQH4+OOP076XBMhaopgXsZEiAu+8807Aj1qyXIglxdSKE1LMTDz9RCFHM/32u+++\nC8AhhxwCeKuJRt6bb06VYtpkk1RqF1lbnnvuOcBboBQvdOmllwLw/PPPA96rrD4NzB4Q0EzkhRql\np2oNqXQxMbXeYOm3ip7r2rUrAPPnzwc8c8Tt84WE2E4WCMWHdO7cGfBx67K7y/Jwyy23AN62LK/i\nQQcdlHZ9jXJvvPEGUJhY8Pho8s9//hOAoUOHAj5iU6ut5CnXft3/X/ziFwAcfHCqOoxiYp5++mnA\n6/TZZnQhMHtA2aBFxQgqKytdc9Lfif0OP/xwwOt2G220EeD1XUG63QMPPADAhRdemHZeY7bmmpqa\nqc65XZvcwPWgoqLCNWXta3xd7a233gp4a4vywGi/RqsjjzwS8PHuH3zwAeB1+M8//xxomO3WrFmT\nNVlbtWrl1pf+Tr4AMbXmF7ofixcvTmuz5i2LFi0CvGXpxRdfBLw9PZ4JriFUVVVlJGtg9oCyQV7N\nGbJFP/LII4BfaX/FFVcAngHEXsuXLwe83qr4kSR5UONQ22RZOv/88wEvs2S86KKLAC+rRo05c+ak\nHXEQcHoAABy0SURBVFdsd5LyxYh5x40bB/jRSKPBe++9B3irmeZoWlMsL3Dc+5xrr3Bg9oCyQUEM\n1bKPK8uV2E72WNmg7777bgBee+01INmMHoeY+s033wS8pUHx7e3apSoqiuUeeughwMe5K0Y8iTLr\n/mnupNErHpPf0N9CrasNzB5QNsirNebbH415VMVess/KoqHMUfG46KaiENaYOGQzFhvKtyAdXH0g\ny4Xs8PqtpsqcT2tMoRGsMQEBjaAgOnt8Fi7m1uxdbFYKlTckQ1zGOOLx7QHZR2D2gLJBIsIGxdxJ\nimLMNoopL0ypIjB7QNmgRdYYM1sMZK0kYQ7QJYt1UIOsyUFGsrboYU+7kNkQ4FTn3F5ZuWCCEWQt\nTpScGmNmM8xsVZ3tGzN7vNDtygWCrM2TteRmhM65HfS/pWaFHwEPFa5FuUOQtXmyZsTsZtbZzMaZ\n2WIzW2pmN9dzzigzm2dm1WY21cwG1jnW38ymRMcWmdnIaH9rM7s3umaVmU02s80yaWOEHwIdgYcz\nvUCQdZ1zilbWZj/sZlYJPEFqAtMV2AIYU8+pk4E+QHvgfuAhM5PHZBQwyjnXDugGPBjtPwXYCOgM\ndADOAFZHvzs66qj6tmkNNPcU4GHnXE1z5QyylqCszrlmbcAAYDHQKrZ/CDBxPd9bDvSO/n8ZuBzo\nGDtnKDAJ2Km57arn9zYAqoF9WnCNIGsJyZqJGtMZmOuc+2Z9J5nZBWY208xWmFkVqTe7Y3R4GNAD\nmBUNaYdG++8BxgNjzGyBmY0ws0y9MEcBy4CXMvw+BFnXQVHLmiEDfMF6GAAYGJ3TC6iowwCDYt+p\nAI4G1gBtYse6Au8Bw6LPtwKrGthm1NPO54A/tpBFgqwlJGsmzP4GsBC41szaRJOPPWPnbAh8QzQs\nmtlwoJ0OmtlgM9vEOVcLVEW7a81sXzPrFemP1cDXQC2Ac+4M51zbBrYd6v64mW0J7AvclYF8QdYS\nlbXZD7tzbi3wE6A78CkwHzgudtp44BlgNqkJzxpgXp3jBwEzzGwVqUnN8c651UAnYCypDplJaqi6\np7ltBE4CXnPOfZjBd79FkLW0ZM2aBzUgIOkoOQ9qQEBDCA97QNkgPOwBZYPwsAeUDVoUCGZmrpDV\n2xpDbW3tEpe9GG9nCV4L65zLmqwVFRWJvq9r167NSNYWPewVFRWJLrW+atWqrC1AMLNEL4ZevXp1\n1mStqKj4NuFqErFixYqMZE3u6xsQkGWEhz2gbBAe9oCyQXjYA8oGiV6Wp1AGVXpQXkRZCpQPMcmW\ng6ZCsibZ4iM0FmLS2HHJmG9Zi/8pCQhoIhLF7GIEZfFVpltVSlZ9oU8++QSA66+/HvDVKbJdXS0X\nkIzK7qsckPHjyo6WRJnitY/ilTMaqxKi+6varvmSMTB7QNkgEcwuphCbqdbSgQceCMCvfvUrwFdS\nnj59OuBrieaqbmY2EJ93CO3btwdghx1S6xNU81XnT5w4EfC1YwuZIzKug++///4ADByYSiwQr3Dd\nr1+/tO/pr+6PquOde+65AMydOzfteK4QmD2gbFAQZpdVRYzcoUMHAE488UQATj75ZMBXX7vzzjsB\n2GuvVAY21WBK0sITtUWjlGRTiIGqOvfu3Rvwla/FgqoOre899dRTAJx33nmArziXBMvTOeecA8CP\nfvQjANasWQP4+6o6UWqr+kajk2Q/5JBDALjxxhsBch56UvieCwjIE/LC7HG9ddCgQYCvXa+K16oK\nvWDBAgDOOussAF5//XXAV4lWRWwxSSFs05JJFgXZ/FWVW/ONwYMHA7DLLrsAfhQTg8+fPx+Ajz76\nCIBOnToBsPfeewPQsWMqS4UsToVgdovVwLrvvvsAX8laOvj777+f9lnfUx+pL2Rdy/ccKzB7QNkg\np8wuJhDrHXnkkQBceeWVgNdTVSHugQceAOAvf/kLADNnzgTguONSi9yl30qHz9csvi7iloV99tkH\ngAMOOADwvgCNWm3atAF8zVBVup40aRIATz75JOAZ/NFHH037vSTo6IKsZQ8+mMpqp/vVGMTsS5cu\nTdufb9mS05MBATlGTphdHjLZX0855RQAzj77bMBbWaTzSQfUrHz58uWAt2TIniuGGDMmlW8zbp/P\nBzRaXXXVVQD87Gc/A2DTTTcF/LxEdU3vuSeVHkXWFdnPVcXbYvWkZMmQHb5v374ATJkyJe28QqKx\nNmg+EvcGb7PNNmmfxfT5mnMFZg8oG2SVJvRGy2oi3Vx2VTH1rFmzALjhhhsAGDt2bNr39XfAgAGA\nt8eK0TXbL6THdMstt0xrgxj7xRdfBOCOO+4AvH1co5DOV2VwjYJi8k022STtfI2CSfIpNIT4fEZ2\ndckqS9Xq1asBmDcvlUwsX7p7YPaAskFWmF0sJM/mNddcA3iPp3Tw22+/HfDWlNmzZwNeZxOj7777\n7gCMGDEC8IzwzDPPAOtaefIJsdaFF14IQM+ePQEfryMfgVgtHhkYh2SWXV3ny0Il+7viTjQSJCnu\nXTLIAyoPqyxSaqv8KIqKlKdcVrcvvvgCyN2IHZg9oGyQFWaXrtalSxfAx39INxej/+1vfwO8xULM\nrM9bbbUVANdeey3gvY6y0siT2li8dC4h5pW1RXqnmLyp6TZkWdpjjz0AOPXUU9Ou/9hjjwHwwgsv\nAN6CIWZPEuKxL3369AG8zyEeu69YGj0vinLV/CYwe0BAC5EVZhcbyXZ8882pImvSsSdPngx4Rhaj\nixGk6yvCT8ygkUB/xYaFjO0WxD7NZaF4NOTpp58O+FHt448/BuDuu+8GvOdVfZckXV1QH6it559/\nPuCtchrppctrXvOTn/wE8KNVru9rYPaAskFWmV2xKpdeeimwLpPHId3t2GOPBWDIkCEAzJkzB/Be\nSulyhbC+ZBuan+y0006Aj/cR5EPQfEe6ehIZPQ49B2JuWdsWLVoEeGaXN1je4nyNWoHZA8oGWfWg\nSndTpF9D0Ky8c+fOAAwdOjTVmIjFJkyYAHgdMAk6ekshXV3zE7Fct27dAB8n8vDDqaLNK1euBJIV\n9dhU6DnQvETrD8Tc+Y6JEYqvJwMCMkReQ+ji8R1nnnkm4PXW5557DvDWF+n0pcDsGs3kHZaF4t13\n3wXg+eefB+D+++8HPKMXg64eRzxGZrvttgP8fSxUTFNg9oCyQV6ZXTEtWluqv6+88grg7eyavZcC\nowuaj8gScdRRR6Udj6/IL0ZGj0MyaA6nz5qL5RuB2QPKBnlldrGWZulaxSMLhFbQJ2E1TrYhVlM0\no6wzgvqmGK0vDUG6u+LXJbM86vlG6fRsQEAjyCuFahZ+3XXXAf5Nl2e0lHT0hlCKDB6HRjHJqJH7\nrbfeAryXON8jeOn2eEBADNaStY1mthjIWknCHKBLFuugBlmTg4xkbdHDnnYhsyHAqc65vbJywQQj\nyFqcKDk1xszam9kDZrbUzJaY2X1m1q7Q7coFgqzNk7XkHnbgSmBjYGugG7AZcFkhG5RDBFmbgYwe\ndjPrbGbjzGxx9KbdXM85o8xsnplVm9lUMxtY51h/M5sSHVtkZiOj/a3N7N7omlVmNtnMNmtm87YG\n/uWcq3bOrQAeAXbIRM4ga4nJ6pxr1gZUAu8ANwJtgNbAXsAQYGKd8wYDHUiZN88HPgdaR8deA06K\n/m8L7B79fzrwOLBB9Dt9gXbRsdFAVQPbtDq/eyjwFCkW2Bh4EfhVc+UMspaerJl0ygBgMdAqtj+t\nU+r53nKgd/T/y8DlQMfYOUOBScBOmdyw6BqbA88DtdH2/9s78xirqmwPf6vqpaWjgMoYCQJPBI2i\n+CRODSrY+gBFRTDghExRFFHQVhSUICGKBJlEQGOIAWwRgYhxADUKBicQAx0mER+K0CjQKEWVQ9qu\n8/449+emDuMd69w6+0sqxb331OWsc/b57bXXXnuvd4E/Zfhd3tYaZGsmbkxT4NsgCH4/0kFm9jcz\n22hm+8zsJ6AuUD/18QCgFbAp1aVdk3p/DrAUmGdm/zSz8WaW7kzTfGAzUBuoA3wNzE3zO4S3NUJR\n25qhAuziCAoAdEgd0wYoOUAB/hr5mxKgJ/ArcHzks+bABmBA6vVMoPwwP+sP+LtyUkqTet0WKM9C\n7bytNcTWTJR9JbATGGdmx6cGH3+JHFMb+J1Ut2hmowifRgDM7FYzaxAEQSWhbwZQaWYdzayNmZUC\nZcC/CbssgiAYFATBCYf5OXCgsgoYaGZ/NrM/A3cA/8jATm9rDbM17cYeBMF/gG5AS2AbsB3oFTls\nKbCEsNv5lvAJ/+6AzzsD682sHJgC9A6C4BegMbCA8IJsBJYTdoHp0J9QPbYDO4D/Bm5P8zsAbys1\nzNaczaB6PHGnJk4qeTyHxDd2T2Lwjd2TGHxj9ySGrJaKmFkQ5xU3lZWVe4Lc5Xh7W2NCprZm1dhL\nSkr+KA4VRyoqKnK2AKGkpOSPMipxpLy8PKe2qqxNHCkrK8vI1pq3jN8TO6KVNyxS+7VQxLev8nhy\nTKyUXRNc2jksWilZuxBYjHfL0rlJxWSLdk6Ivi/b9Ft/L1u1I0MxTv5pL3q5f6p4rmugfdwLhVd2\nT2KIhbLrSVcFBu362rBhQ8Cp3Icffgi42kpxVHj1Rjr35s2bA263XlXGbtGiBeD2QdQuaaovpCp5\nqiEb5+hIFN0f7c48ePBgwO1v+c033wDQpUsXwO3tme/dfYvnCno8WVItyi7/UypWp06YJaqdwi69\n9FLAVXXWEz9o0CAA3nrrLSCee0JG60RJ1VSpWopfr1494OAqFXo9Y8YMAEaPHg0Uh8+u+3n66acD\nMHHiRMAp9/333w+4a9KuXTsAFi9eDHhl93hyRrVU3pD/ed111wHwyCOPANCyZUsA1q1bB7gqz1dd\ndRXgKsy9/fbbBTrjzNE4pEGDcKJPqrdy5UrA+fCqinfyyScDcOONNwKuztQrr7xS5fjqqlpxJDRO\nUe2kxx57DIA1a9YA8OCDD1Y5rmfPngDUrl0bKFyv5ZXdkxgKquxSu44dOwLw7LPPAm5318mTJwPw\nzjvvAM7X69UrXDAjBZBCxFHldE7y0eXDy8YhQ4YArmKcejlFZdSLqXdT1fA4Rp6E7qsiTxdccAHg\nFFz7s6tSoHqxQt9Hr+yexFAtPrtG65phUxTmgw8+CE8qFWVRlEa+3+WXXw64+OySJUuqHB8HNFMq\n3/yTTz4B4OqrrwZg+PDhAEyaNAmAZs2aAa5yYP364a4Uqi+lquGag4hzVEbjEtWH0rVQryQl1/3S\nfe/UqRMA9957L+Bml3Pdm3ll9ySGgkqinlRVjKuoqADcE71161bAzTL26dMHcIouv3bChAnAwXWJ\n4oB8cPnkUugpU6YATsEvvvhiwPmxX3zxBQADBgwAYOfOnUBxVCORzTrn3bt3A26W+NNPPwWgfftw\n12vNNWg8ozi7evp8zRZ7ZfckhmqpqRSNTIwfPx5w6qd4u35v3rwZgDFjxgCwaNEiwPmxcUI+tRRZ\nURn1RmedFe77I5XbuHEjAI8//jjgMgGVDxTNiowjUuL9+/cD7n4pN+bjjz8GYOjQoYBrB8OGDQPg\nzTffBJwvn6/Ik1d2T2KoFp9do20pvHw9RVlUK1QRi+nTpwOFy47Lhmg+u3K4+/XrB7jcbs0CKyat\nXm7EiBHAwfntcVZ2naN87gULFgAwbVq4vXvXrl0BaNKkCQCjRo0CDs5xyvdcgld2T2Ko1qzHK6+8\nEoBzzz0XcNEV+bFShl27dgHFEZmI+tjK8OvRowfgMgGnTp0KOBsVeZJ/q7ygYrJZ90/nrExOzSWs\nWLECgJdffrnK3xdqdtgruycxFFTZ5aufffbZAIwdOxZwSiBVU6ZgMVaDlq/eunVrAHr37g24CNIz\nzzwDuBVITz/9NAAXXngh4LIe3333XcDNSsY5N0aKfsUVVwDuvipnX2Myjb1kuyJOhaJ4WpHHkyXV\nkhvTvXt3AE499VQA5s+fD7hsx4cffhhweSKKwsSZaERCmZ3qtWbNmgW4LEb5s2vXrgXgpZdeAqBb\nt26Am0XesmULEK/8H6Fe57zzzgPg+eefB2D79u2AG6+ot1IcvroiS17ZPYmhWuRC/qnUcNmyZQC8\n8cYbANx+e1hQQbONyu2OM/Jblamp3BdFlhSJ0ByBxi9SR+WJnHbaaQA0btwYgC+//BKIp7LLBs0V\nnHLKKYCbQ9DMaNu2bQE3a6zs1uhcQr7xyu5JDNUiF1I57SKgeLtUUfnuUrtiQMquHRFOOukkwK3K\nUW/2/fffV/k75Y/07dsXcPnvyqmJo6IL9VJfffUVAEuXLgVc7v61114LuN5rw4YNgLvPmin3K5U8\nnhxTUNlQvFzRF43SFZ3R688++wxwuTPFEGeXOimPXbsB3HPPPQAsXLgQcIotm9q0aQO4KI3i7jt2\n7ADiPYOqrNP3338fgFWrVgGuF7v55psBuOyyywB3X7UmtdBzB/FvRR5PjqiWfHapm/YTGTlyJMAf\nhQ20NjG6/jLORNdZzp49G3CRCM0Ky1+VsisyoZlV7WcZXb8Z56xHnevPP/8MON9dK5Q0blGvp1nm\nQvfYXtk9iSGror+lpaVBNmVmonuUCz3x2Y7SKyoqVgdB0C6rL0lRWloaHEuZmWg+e7rrZKO53cd6\nf8rLy3Nqay7KzMh2/c5VrlNZWVlGtnpl9ySGag3ixjmGnCnR/SwzVbE4++jHStyyVuNxFh5PAcjK\nZzez3UDOShLmgWY5rA3qbY0PGdmaVWOv8kVmfYGBQRC0z8kXxhhva3FS49wYM2tiZovNbK+ZbTez\nQdV9TvnC25oeNa6xA3OBrUAj4GrgCTPrWL2nlDe8rekQBEHaP0BTYBGwG/gXMA3oC6w44JgpwHdA\nGbAa6HDAZxcAn6c++wGYmHq/VsqofwE/AauARmmc1wlAADQ44L3ngTmZ2OltrVm2pq3sZlYKvEE4\ngGkONAHmHeLQVUBb4GTg78CrZlYr9dkUYEoQBHWA04D5qfdvB+qmLno9YBDwS+r/nW5mPx3m5x86\nvchv/fvsdO30ttZAWzN4+i8mfPL/K/J+Xw5QgEP83Y/Aual/fwg8DtSPHNMf+Bg4Jwt1WgE8Q6gm\n/wPsBb7M8Lu8rTXI1kx89qbAt0EQ/H6kg8zsb2a20cz2mdlPhE92/dTHA4BWwCYzW2Vm16TenwMs\nBeaZ2T/NbLyZpZvjegvQgrCrnUHYfW5P8zuEtzVCUduaoQLs4ggKAHRIHdMGKDlAAf4a+ZsSoCfw\nK3B85LPmwAZgQOr1TKD8MD/rj3C+fweezELtvK01xNZMlH0lsBMYZ2bHm1ktM/tL5JjawO+kukUz\nGwXU0YdmdquZNQiCoJJwwAJQaWYdzaxNyn8sA/4NVAIEQTAoCIITDvNz1gHffaaZ1TazP5nZrcBV\nwMQM7PS21jBb027sQRD8B+gGtAS2EXYlvSKHLQWWAJsJBzy/EnY/ojOw3szKCQc1vYMg+AVoDCwg\nvCAbgeWEXWA6/C/wf4SKMwjoHATB7jS/A/C2UsNszdkMqscTd2ripJLHc0h8Y/ckBt/YPYnBN3ZP\nYshqqZCZBXFZhXIoKisr9wS5y/H2tsaETG3NqrGXlJT8UYg3juzfvz9nCxCSZmsuFlzni7Kysoxs\nje/j6/HkGN/YPYnBN3ZPYvCN3ZMYYrFxi1IWtHOU9vMWltodSzuEFeN+M7JNNZeE9rGMc/TjcOi+\n6Xd0FzSL7Gqmz/W+7qPuq963PO3uW3xX2OPJkGqVSKmcVE0VNwYMGACA9lZU9bXPP/8cgOXLlwPx\nrg0qtJ9lo0aNABg6dCjgbHviiScA2LNnD1AcNkWTB1X/SXvN6z5K4evWrQtAu3bh9oz79u0D4LXX\nXgNc/du9e/dW+f5c93Ze2T2JoVqUXWrXsmVLAPr16wdAr15h+rTqZX73XZgqfcYZZwCuooPqDqln\niKMaSp20d/l9990HwG233Qa4ShuqqdSjRw/A1VwqVJ2hTNB179+/PwCPPvoo4Pag1z7sqoklhdb4\nRNdGtZek6FOnTgVcPVX5+Lm6Fl7ZPYmhoMouRTj//PMBeO655wBX7/TJJ58EYM6ccBHLpk2bAGje\nvDngaoSqZ4ijogtFlHr27AnA3XffDcCMGTMAZ+Mtt9wCuIrWO3fuLOh5poOuu6ocDh8+HHCVNSZM\nmAC4MZWUPRqVkdKrwvk114Trsh966CEA6tWrB8DkyZMBV9EjWx/eK7snMRRE2eV7SZlVP0hVOzp3\n7gy4ekI6XslIqhGqKM31118PuOrRcfJvde4aZ4wZMwaAtWvXAjBt2jQAtmzZAkDDhg0BGDt2LADD\nhg0D4mVbVJHVG6mitXz2SZMmAU7Jj6bEqielqoiqjK0eQ1X4Pvroo2P6vqPhld2TGAqi7Kp7edNN\nNwHOB+/WrRvgqqpJxZRKO27cOMDV01ywYAHgojRxnHWUX6sqebJFCv/1118DLu5+1113AS5W/cMP\nPwDxGo9I2WvVCne5a926NQDbtm0DXHU83b+jVTdU/F02q2qiojmKw+c62ha/1uLx5Im8KrueYI2u\nFVOWAsi/7dChAwAtWrSo8rpPnz4AvPjiiwA88MADgFPPOPizUXRu8tkVlVm9ejXgfHT5pR07hrsu\nb9iwAYDdu8OtUOJc+zVaBU/KfzQF1vHl5eWAq2jeu3dvwEVdouMcH2f3eNIkr8quJ15+q3JcNHv4\n3nvvAe6JVxacojTy9TUqr6ioAFxeSRzRjKnyeO644w4AFi5cCLjYs2LMOn7lypVAvHx1oXPSuatC\nuXxu2XC4mq+//fYb4KJrmi2Wry7lnj59OuDmX/R+rsZmXtk9iSGvyq4nUxl9iiEPHDgQgKeeego4\nOP95x44dgHvSlyxZAjjFjzNSOUWYRo8eDUCrVq0A+PHHHwFnq7IgdY3iiM5VCq2IkWZ9b7jhBsDN\nHagH0LW45JJLABdV06zyiSeeCLj7K0XPV/1Ur+yexFCQOLueUCn4vHlh9ZJVq1YBMGLECAC6dOkC\nuCd84sRwR2L58nGMq0eRCmpc8sILLwCul1N0Rrky6q1WrFhR0PPMBCn1q6++CkDXrl0Bl9PSvXt3\nwI21dN+aNm0KuLGbrpHygHSNlPGZr0hU/FuPx5MjCpr1KGWW3yoFUM7M66+/DsDcuXOrHF8Mih4l\neu7R/JLatWtXeX24SEacUO+k+LciTXfeeScA55xzDuBWJun+KjJ10UUXAU65leW6ePFiIP9RtuJr\nRR5PhhRU2aVeircqUiHfbuTIkYDz5eI8i5gu0ZXz0VlHzSYXA7ova9asAWDw4MGAi84oh0YzolL8\n9u3DivDLli0DnO+v4/ONV3ZPYqiWlUpDhgwB3GheuRFamVQM8fRskaIrT0TrMOM4g3o4ouORrVu3\nAq4HV/RG6xfUe+m1xm7HHXdcYc63IP+LxxMDCqLsUnTNpCmHe+bMmYBbs1iTfPSjoXHKunXrAJfD\nXUzKLnTOUnIpuHJgOnXqBLhZZfn6hd7ZzSu7JzHk9dHSbOGZZ54JwKxZswBYv349AOPHjwegrKwM\ncMqQBNSLKXZdDHH2o6FcfkVllLMvtAOYerFCz594Zfckhrwqu3w5KfyiRYsAN0Oq7LkkKbquifxX\n7Z4V3feyGNG5K+tRcXRFmmbPnl3l+EKPT4r3yno8aZJVOffS0tLgWIpqaXSuJ14zZvleQ7p///7V\nQRC0y8V3Hautx4qUXNdG1yRTtcu1rdkUEFObUm6MbMrWRlFWVpaRrV7ZPYkhK2U3s91AzkoS5oFm\nOawN6m2NDxnZmlVj93iKCe/GeBKDb+yexOAbuycx+MbuSQy+sXsSg2/snsTgG7snMfjG7kkMvrF7\nEsP/A/F8lzfE80ZUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x117235320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot three samples from each class\n",
    "n_rows = 10\n",
    "n_cols = 3\n",
    "\n",
    "selected = np.linspace(0, X.shape[0] - 1, n_rows * n_cols, dtype='int16')\n",
    "\n",
    "plt.figure(figsize=(n_cols, n_rows))\n",
    "plt.subplots_adjust(hspace=.4)\n",
    "\n",
    "for i, idx in enumerate(selected):\n",
    "    plt.subplot(n_rows, n_cols, i + 1)\n",
    "    plt.imshow(X[idx].reshape((20,20), order='F'), cmap=plt.cm.gray)\n",
    "    plt.title(f'class={y[idx]}')\n",
    "    plt.xticks(())\n",
    "    plt.yticks(())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='newton-cg', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train a logistic regression classifier with C=1.0 (inverse regularization strength)\n",
    "clf = LogisticRegression(C=1.0, solver='newton-cg')\n",
    "clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9446"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Classification accuracy on training set\n",
    "clf.score(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.1, 1.0, 10.0], class_weight=None, cv=None,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l2',\n",
       "           random_state=None, refit=True, scoring=None, solver='newton-cg',\n",
       "           tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train a logistic regression classifier, running \n",
    "# built-in cross validation for selecting the best \n",
    "# C value \n",
    "clf_cv = LogisticRegressionCV(Cs=[1e-1, 1e0, 1e1], solver='newton-cg')\n",
    "clf_cv.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.94079999999999997"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Classification accuracy on training set \n",
    "# using the model that was selected during \n",
    "# cross-validation\n",
    "clf_cv.score(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Neural networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((25, 401), (10, 26))"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the parameters of a pre-trained neural network\n",
    "weights = sio.loadmat('data/ml-ex3/ex3weights.mat')\n",
    "\n",
    "Theta1 = weights['Theta1']\n",
    "Theta2 = weights['Theta2']\n",
    "\n",
    "Theta1.shape, Theta2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Number of examples (first dimension of design matrix X)\n",
    "n_examples = X.shape[0]\n",
    "\n",
    "# sigmoid function\n",
    "sigmoid = sp.special.expit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Input layer data\n",
    "A1 = np.c_[np.ones(n_examples), X]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Hidden layer activations\n",
    "Z2 = A1.dot(Theta1.T)\n",
    "A2 = np.c_[np.ones(n_examples), sigmoid(Z2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Output layer activations\n",
    "Z3 = A2.dot(Theta2.T)\n",
    "A3 = sigmoid(Z3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Predicted class is index of highest value per row + 1\n",
    "y_pred = np.argmax(A3, axis=1) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          1     0.9684    0.9820    0.9752       500\n",
      "          2     0.9818    0.9700    0.9759       500\n",
      "          3     0.9776    0.9600    0.9687       500\n",
      "          4     0.9699    0.9680    0.9690       500\n",
      "          5     0.9723    0.9840    0.9781       500\n",
      "          6     0.9782    0.9860    0.9821       500\n",
      "          7     0.9778    0.9700    0.9739       500\n",
      "          8     0.9781    0.9820    0.9800       500\n",
      "          9     0.9657    0.9580    0.9618       500\n",
      "         10     0.9822    0.9920    0.9871       500\n",
      "\n",
      "avg / total     0.9752    0.9752    0.9752      5000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print classification report showing the main classification metrics\n",
    "print(classification_report(y, y_pred, target_names=['1','2','3','4','5','6','7','8','9','10'], digits=4))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}