ANN_pre_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math,struct,pickle\n",
    "from pathlib import Path\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm_notebook\n",
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bypass(x):\n",
    "    return x\n",
    "def tanh(x):\n",
    "    return np.tanh(x)\n",
    "def softmax(x):\n",
    "    exp=np.exp(x-x.max())\n",
    "    return exp/exp.sum()\n",
    "\n",
    "def d_softmax(data):\n",
    "    sm=softmax(data)\n",
    "    return np.diag(sm)-np.outer(sm,sm)\n",
    "\n",
    "# def d_tanh(data):\n",
    "#     return np.diag(1/(np.cosh(data))**2)\n",
    "def d_tanh(data):\n",
    "    return 1/(np.cosh(data))**2\n",
    "def d_bypass(x):\n",
    "    return 1\n",
    "\n",
    "differential={softmax:d_softmax,tanh:d_tanh,bypass:d_bypass}\n",
    "d_type={bypass:'times',softmax:'dot',tanh:'times'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "dimensions=[28*28,100,10]\n",
    "activation=[bypass,tanh,softmax]\n",
    "distribution=[\n",
    "    {}, # leave it empty!!\n",
    "    {'b':[0,0],'w':[-math.sqrt(6/(dimensions[0]+dimensions[1])),math.sqrt(6/(dimensions[0]+dimensions[1]))]},\n",
    "    {'b':[0,0],'w':[-math.sqrt(6/(dimensions[1]+dimensions[2])),math.sqrt(6/(dimensions[1]+dimensions[2]))]},\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_parameters_b(layer):\n",
    "    dist=distribution[layer]['b']\n",
    "    return np.random.rand(dimensions[layer])*(dist[1]-dist[0])+dist[0]\n",
    "def init_parameters_w(layer):\n",
    "    dist=distribution[layer]['w']\n",
    "    return np.random.rand(dimensions[layer-1],dimensions[layer])*(dist[1]-dist[0])+dist[0]\n",
    "def init_parameters():\n",
    "    parameter=[]\n",
    "    for i in range(len(distribution)):\n",
    "        layer_parameter={}\n",
    "        for j in distribution[i].keys():\n",
    "            if j=='b':\n",
    "                layer_parameter['b']=init_parameters_b(i)\n",
    "                continue\n",
    "            if j=='w':\n",
    "                layer_parameter['w']=init_parameters_w(i)\n",
    "                continue\n",
    "        parameter.append(layer_parameter)\n",
    "    return parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(img,parameters):\n",
    "    l_in=img\n",
    "    l_out=activation[0](l_in)\n",
    "    for layer in range(1,len(dimensions)):\n",
    "        l_in=np.dot(l_out,parameters[layer]['w'])+parameters[layer]['b']\n",
    "        l_out=activation[layer](l_in)\n",
    "    return l_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_path=Path('./MNIST')\n",
    "train_img_path=dataset_path/'train-images.idx3-ubyte'\n",
    "train_lab_path=dataset_path/'train-labels.idx1-ubyte'\n",
    "test_img_path=dataset_path/'t10k-images.idx3-ubyte'\n",
    "test_lab_path=dataset_path/'t10k-labels.idx1-ubyte'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_num=50000\n",
    "valid_num=10000\n",
    "test_num=10000\n",
    "\n",
    "with open(train_img_path,'rb') as f:\n",
    "    struct.unpack('>4i',f.read(16))\n",
    "    tmp_img=np.fromfile(f,dtype=np.uint8).reshape(-1,28*28)/255\n",
    "    train_img=tmp_img[:train_num]\n",
    "    valid_img=tmp_img[train_num:]\n",
    "    \n",
    "with open(test_img_path,'rb') as f:\n",
    "    struct.unpack('>4i',f.read(16))\n",
    "    test_img=np.fromfile(f,dtype=np.uint8).reshape(-1,28*28)/255\n",
    "\n",
    "with open(train_lab_path,'rb') as f:\n",
    "    struct.unpack('>2i',f.read(8))\n",
    "    tmp_lab=np.fromfile(f,dtype=np.uint8)\n",
    "    train_lab=tmp_lab[:train_num]\n",
    "    valid_lab=tmp_lab[train_num:]\n",
    "    \n",
    "with open(test_lab_path,'rb') as f:\n",
    "    struct.unpack('>2i',f.read(8))\n",
    "    test_lab=np.fromfile(f,dtype=np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_train(index):\n",
    "    plt.imshow(train_img[index].reshape(28,28),cmap='gray')\n",
    "    print('label : {}'.format(train_lab[index]))\n",
    "def show_valid(index):\n",
    "    plt.imshow(valid_img[index].reshape(28,28),cmap='gray')\n",
    "    print('label : {}'.format(valid_lab[index]))\n",
    "def show_test(index):\n",
    "    plt.imshow(test_img[index].reshape(28,28),cmap='gray')\n",
    "    print('label : {}'.format(test_lab[index]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-4.16213710e-06  1.48970149e-06  1.67382823e-06  9.98606271e-07]\n",
      "[ 1.67592913e-06 -4.48206604e-06  1.04555234e-06  1.76058512e-06]\n",
      "[ 1.01810631e-06  2.12428173e-06 -4.03150025e-06  8.89112203e-07]\n",
      "[ 2.11619599e-06  1.46305923e-06  1.01222873e-06 -4.59148451e-06]\n"
     ]
    }
   ],
   "source": [
    "h=0.0001\n",
    "func=softmax\n",
    "input_len=4\n",
    "for i in range(input_len):\n",
    "    test_input=np.random.rand(input_len)\n",
    "    derivative=differential[func](test_input)\n",
    "    value1=func(test_input)\n",
    "    test_input[i]+=h\n",
    "    value2=func(test_input)\n",
    "#     print((value2-value1)/h)\n",
    "#     print(derivative[i])\n",
    "    print(derivative[i]-(value2-value1)/h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.807685993173493e-07\n",
      "3.396431955127177e-07\n",
      "3.718994151524768e-07\n",
      "9.35760171572042e-08\n"
     ]
    }
   ],
   "source": [
    "h=0.000001\n",
    "func=tanh\n",
    "input_len=4\n",
    "for i in range(input_len):\n",
    "    test_input=np.random.rand(input_len)\n",
    "    derivative=differential[func](test_input)\n",
    "    value1=func(test_input)\n",
    "    test_input[i]+=h\n",
    "    value2=func(test_input)\n",
    "#     print((value2-value1)/h)\n",
    "#     print(derivative[i])\n",
    "    print(derivative[i]-((value2-value1)/h)[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "onehot=np.identity(dimensions[-1])\n",
    "\n",
    "def sqr_loss(img,lab,parameters):\n",
    "    y_pred=predict(img,parameters)\n",
    "    y=onehot[lab]\n",
    "    diff=y-y_pred\n",
    "    return np.dot(diff,diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grad_parameters(img,lab,parameters):\n",
    "    l_in_list=[img]\n",
    "    l_out_list=[activation[0](l_in_list[0])]\n",
    "    for layer in range(1,len(dimensions)):\n",
    "        l_in=np.dot(l_out_list[layer-1],parameters[layer]['w'])+parameters[layer]['b']\n",
    "        l_out=activation[layer](l_in)\n",
    "        l_in_list.append(l_in)\n",
    "        l_out_list.append(l_out)\n",
    "    \n",
    "    d_layer=-2*(onehot[lab]-l_out_list[-1])\n",
    "    \n",
    "    grad_result=[None]*len(dimensions)\n",
    "    for layer in range(len(dimensions)-1,0,-1):\n",
    "        if d_type[activation[layer]]=='times':\n",
    "            d_layer=differential[activation[layer]](l_in_list[layer])*d_layer\n",
    "        if d_type[activation[layer]]=='dot':\n",
    "            d_layer=np.dot(differential[activation[layer]](l_in_list[layer]),d_layer)\n",
    "        grad_result[layer]={}\n",
    "        grad_result[layer]['b']=d_layer\n",
    "        grad_result[layer]['w']=np.outer(l_out_list[layer-1],d_layer)\n",
    "        d_layer=np.dot(parameters[layer]['w'],d_layer)\n",
    "    \n",
    "    return grad_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters=init_parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8.633561339776996e-05"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h=0.001\n",
    "layer=2\n",
    "pname='b'\n",
    "grad_list=[]\n",
    "for i in range(len(parameters[layer][pname])):\n",
    "    img_i=np.random.randint(train_num)\n",
    "    test_parameters=init_parameters()\n",
    "    derivative=grad_parameters(train_img[img_i],train_lab[img_i],test_parameters)[layer][pname]\n",
    "    value1=sqr_loss(train_img[img_i],train_lab[img_i],test_parameters)\n",
    "    test_parameters[layer][pname][i]+=h\n",
    "    value2=sqr_loss(train_img[img_i],train_lab[img_i],test_parameters)\n",
    "    grad_list.append(derivative[i]-(value2-value1)/h)\n",
    "np.abs(grad_list).max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.7486410737881384e-05"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h=0.001\n",
    "layer=1\n",
    "pname='b'\n",
    "grad_list=[]\n",
    "for i in range(len(parameters[layer][pname])):\n",
    "    img_i=np.random.randint(train_num)\n",
    "    test_parameters=init_parameters()\n",
    "    derivative=grad_parameters(train_img[img_i],train_lab[img_i],test_parameters)[layer][pname]\n",
    "    value1=sqr_loss(train_img[img_i],train_lab[img_i],test_parameters)\n",
    "    test_parameters[layer][pname][i]+=h\n",
    "    value2=sqr_loss(train_img[img_i],train_lab[img_i],test_parameters)\n",
    "    grad_list.append(derivative[i]-(value2-value1)/h)\n",
    "np.abs(grad_list).max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.455954227267078e-07"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h=0.00001\n",
    "layer=2\n",
    "pname='w'\n",
    "grad_list=[]\n",
    "for i in range(len(parameters[layer][pname])):\n",
    "    for j in range(len(parameters[layer][pname][0])):\n",
    "        img_i=np.random.randint(train_num)\n",
    "        test_parameters=init_parameters()\n",
    "        derivative=grad_parameters(train_img[img_i],train_lab[img_i],test_parameters)[layer][pname]\n",
    "        value1=sqr_loss(train_img[img_i],train_lab[img_i],test_parameters)\n",
    "        test_parameters[layer][pname][i][j]+=h\n",
    "        value2=sqr_loss(train_img[img_i],train_lab[img_i],test_parameters)\n",
    "        grad_list.append(derivative[i][j]-(value2-value1)/h)\n",
    "np.abs(grad_list).max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cb2446d69114498eb9a97fc4cbcea861",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=784), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3.683723256361904e-07"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h=0.00001\n",
    "layer=1\n",
    "pname='w'\n",
    "grad_list=[]\n",
    "for i in tqdm_notebook(range(len(parameters[layer][pname]))):\n",
    "    for j in range(len(parameters[layer][pname][0])):\n",
    "        img_i=np.random.randint(train_num)\n",
    "        test_parameters=init_parameters()\n",
    "        derivative=grad_parameters(train_img[img_i],train_lab[img_i],test_parameters)[layer][pname]\n",
    "        value1=sqr_loss(train_img[img_i],train_lab[img_i],test_parameters)\n",
    "        test_parameters[layer][pname][i][j]+=h\n",
    "        value2=sqr_loss(train_img[img_i],train_lab[img_i],test_parameters)\n",
    "        grad_list.append(derivative[i][j]-(value2-value1)/h)\n",
    "np.abs(grad_list).max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "def valid_loss(parameters):\n",
    "    loss_accu=0\n",
    "    for img_i in range(valid_num):\n",
    "        loss_accu+=sqr_loss(valid_img[img_i],valid_lab[img_i],parameters)\n",
    "    return loss_accu/(valid_num/10000)\n",
    "def valid_accuracy(parameters):\n",
    "    correct=[predict(valid_img[img_i],parameters).argmax()==valid_lab[img_i] for img_i in range(valid_num)]\n",
    "    return correct.count(True)/len(correct)\n",
    "def train_loss(parameters):\n",
    "    loss_accu=0\n",
    "    for img_i in range(train_num):\n",
    "        loss_accu+=sqr_loss(train_img[img_i],train_lab[img_i],parameters)\n",
    "    return loss_accu/(train_num/10000)\n",
    "def train_accuracy(parameters):\n",
    "    correct=[predict(train_img[img_i],parameters).argmax()==train_lab[img_i] for img_i in range(train_num)]\n",
    "    return correct.count(True)/len(correct)\n",
    "def test_accuracy(parameters):\n",
    "    correct=[predict(test_img[img_i],parameters).argmax()==test_lab[img_i] for img_i in range(test_num)]\n",
    "    return correct.count(True)/len(correct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grad_add(grad1,grad2):\n",
    "    for layer in range(1,len(grad1)):\n",
    "        for pname in grad1[layer].keys():\n",
    "            grad1[layer][pname]+=grad2[layer][pname]\n",
    "    return grad1\n",
    "def grad_divide(grad,denominator):\n",
    "    for layer in range(1,len(grad)):\n",
    "        for pname in grad[layer].keys():\n",
    "            grad[layer][pname]/=denominator\n",
    "    return grad\n",
    "\n",
    "def combine_parameters(parameters,grad,learn_rate):\n",
    "    parameter_tmp=copy.deepcopy(parameters)\n",
    "    for layer in range(1,len(parameter_tmp)):\n",
    "        for pname in parameter_tmp[layer].keys():\n",
    "            parameter_tmp[layer][pname]-=learn_rate*grad[layer][pname]\n",
    "    return parameter_tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size=100\n",
    "def train_batch(current_batch,parameters):\n",
    "    grad_accu=grad_parameters(train_img[current_batch*batch_size+0],train_lab[current_batch*batch_size+0],parameters)\n",
    "    for img_i in range(1,batch_size):\n",
    "        grad_tmp=grad_parameters(train_img[current_batch*batch_size+img_i],train_lab[current_batch*batch_size+img_i],parameters)\n",
    "        grad_add(grad_accu,grad_tmp)\n",
    "    grad_divide(grad_accu,batch_size)\n",
    "    return grad_accu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters=init_parameters()\n",
    "current_epoch=0\n",
    "train_loss_list=[]\n",
    "valid_loss_list=[]\n",
    "train_accu_list=[]\n",
    "valid_accu_list=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.093"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_accuracy(parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4f8b7ed36a943ad855529447cd6ad8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "learn_rate=10**-0\n",
    "epoch_num=1\n",
    "for epoch in tqdm_notebook(range(epoch_num)):\n",
    "    for i in range(train_num//batch_size):\n",
    "#         if i%100==99:\n",
    "#             print('running batch {}/{}'.format(i+1,train_num//batch_size))\n",
    "        grad_tmp=train_batch(i,parameters)\n",
    "        parameters=combine_parameters(parameters,grad_tmp,learn_rate)\n",
    "    current_epoch+=1\n",
    "    train_loss_list.append(train_loss(parameters))\n",
    "    train_accu_list.append(train_accuracy(parameters))\n",
    "    valid_loss_list.append(valid_loss(parameters))\n",
    "    valid_accu_list.append(valid_accuracy(parameters))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.99868"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_accuracy(parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9775"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_accuracy(parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.975"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_accuracy(parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl4VPW9x/H3NwsQQjBBQkiIEGQREBFoRNw3VEButXWp0sW2trTWpV7tU5d7r732Frtar0uLxWtdUFSsqFyvG0KxdYtG1iAgYccECGuAQNbv/ePMhCCRLCRMOHxez3Oemfmdc2a+ORw+v3N+c2bG3B0REQmvuFgXICIirUtBLyIScgp6EZGQU9CLiIScgl5EJOQU9CIiIaegFxEJOQW9iEjIKehFREIuIdYFAHTt2tVzcnJiXYaIyBHlk08+2ezu6Q0t1yaCPicnh/z8/FiXISJyRDGzNY1ZTkM3IiIhp6AXEQk5Bb2ISMgp6EVEQk5BLyIScgp6EZGQU9CLiIRcm7iOvrmKior46KOPKC0tZefOnZSWltKhQwdyc3MZPnw4ycnJ9a5XXV3Nhg0bcHeys7MPc9UiIofXER307733HldddVW98+Li4hg8eDDp6enU1NRQU1NDRUUFRUVFfP7551RVVQEwevRobrnlFi666CLMDICKigpWrVpFTk4O7du3P2x/j4hIa7C28OPgubm53pxPxm7dupW1a9fSuXNnOnfuTEpKCtu3b+fjjz/mo48+Ij8/n9LSUuLi4oiLiyMhIYHMzEyOO+44evbsyaZNm5g0aRIbNmxg4MCBDB48mMWLF/PZZ59RVVVFamoqV1xxBePHj+fss88mPj6+Ff56EZHmMbNP3D23weWO5KBvCRUVFUybNo2HH36YzZs3c+KJJzJ48GCOP/543nnnHV566SV27dpFZmYmV111Fd/4xjcYOXJk7dG/iEisKOhbSFlZGa+++irPPvssr7/+OuXl5fTs2ZNLLrmEQYMGccIJJzBgwAAyMzNJSDiiR8JE5AijoG8FO3bs4JVXXuH555/n3XffpbS0dL/5nTp1IjU1ldTUVLp27Vo79ejRg5NPPplhw4bRo0cPnQ2ISItQ0Lcyd2fjxo0sXbqUZcuWsWnTJrZv3862bdvYtm0bW7ZsYfPmzZSUlLBlyxai2zk9PZ2uXbtSVlZGWVkZe/bsISUlhW7dupGenk52djbnnXceF110Ed27d4/xXykibZmCvg3ZtWsXCxYsYN68ecybN4/S0lI6duxIx44d6dChA6WlpZSUlFBSUsKKFSsoKSkBYOjQoZx22mlkZmaSkZFBRkYGxx9/PP369aNDhw61z79582bmzZtHZWUlF198sd40FjlKKOiPUDU1NSxYsIA33niDN954g0WLFrFt27b9lomLi6N379707NmTzz77jM8//7x2Xv/+/bnrrrsYP348iYmJLVbXrl27ePHFF3niiSdYs2YNX/va1xg/fjzDhw9v8lDUqlWreP3113nnnXc49dRTufHGG2nXrl29y+7Zs4eVK1dSWFhI165dGTFiRL1/l7vXW0d1dTXFxcVkZGQ0enuUl5fz9ttv89JLL/HOO+8wcuRIvvOd73D++efv14nW1NRQWVlJYmIicXFx1NTUUFBQwJw5c5gzZw4rV67k1ltv5dvf/naTtlFVVRWlpaV06dKl0eu0hi/bpgB79+6lrKysWTXu3LmTuLi4L/2cizSegj5EKioq2LRpExs2bKCwsJAlS5awdOlS1qxZQ9++fRk2bBjDhg1jy5YtTJw4kQULFtC7d29uuukmrrzyymZ/KKympoY5c+bw1FNP8be//Y3du3fTt29f+vfvz8yZM6msrKRfv36MGTOGIUOGMGTIEAYMGMCqVav4+OOPyc/PZ/ny5bWXtiYkJPDZZ5+xbNkyALp168amTZvo06cPv//977nsssvYvXs3r7/+OtOnT+fdd99l/fr1+9WUkpLCeeedx9lnn01JSQkLFy5k0aJFfP7553Tr1o2srCyysrKorq5mxYoVrF69msrKStLT07nyyisZP348p512GmVlZRQUFFBQUMCqVav2G3L78MMP2blzJykpKZx55pl88MEHbN++naysLC688EI2btzIqlWrWL16NeXl5QC1l/BGP5/Ru3dvUlJSWLhwIeeddx6PPPII/fv3B2D9+vXk5eVRXl5Oly5dOPbYY2nfvj3vv/8+b731FrNmzaK0tJRzzz2Xa6+9lssvv5yUlJR6/43cnS1btpCamnrAxQCVlZVs2LCBrl27kpSUtN+8vXv3MnXqVPLz8xk+fDinn346AwYMYNeuXcyYMYNp06bx1ltvcd5553HfffcxaNCg2td76aWXuOWWW9i6dSv33nsvN9xwQ20HWF1dzdNPP81TTz3FFVdcwYQJE2rnuTvPP/88N954I0lJSUydOpWzzjqrWfumBBT0Ryl359VXX2XixInk5eUBcMYZZ3DFFVfQp08fUlJSat80zsrKomPHjvutv2PHDpYvX8706dN5+umnWbduHSkpKXzjG9/gu9/9LqeffjpmxrZt23jxxRd59tlnycvLY/fu3QfU0rlzZwYNGoSZUVVVRVVVFRkZGYwePZoxY8bQr18/3nzzTW677TY+/fRTBg0axIoVKygvL6dr165cdNFFDBgwgL59+9K3b1/Wrl3LzJkzmTlzJitXriQxMZGBAwdy0kkn0atXL0pKSigqKqKoqAgzo0+fPvTp04fs7Gz++c9/MmPGDPbs2UNqairbt2+vrTM+Pp60tDRSU1NJS0vj5JNP5utf/zrnn38+7du3Z+/evbz66qs89dRT5OXlkZ2dTe/evenduzdpaWlUVVVRWVlJVVUVAwcO5JxzzqFXr17U1NTw6KOPcvvtt7Nnzx4uvPBCFixYcEDnVVfPnj25+OKLycjI4LnnnqOwsJCOHTty5plnkp2dTVZWFhkZGaxevZq5c+cyd+5cduzYgZnVdnSJiYmsX7+e4uJi3J2kpCRGjRrFuHHjGDFiBC+88AKTJ09m8+bNJCUlsWfPHgDS0tIoKyujvLyc7OxsRo0aVXt58fXXX8+3vvUtfvGLX/Dmm29y8sknk5GRwVtvvcXIkSN57LHHKC4u5mc/+xnz58+v7cSHDBnCgw8+yMCBA7n++uuZPn06p5xyCtu2bWPlypXcc8893HnnncTHx7Ns2TKmTJnC3Llzeeihh+jTp89+22bt2rV89atfZejQoUyaNOmAzqugoIB58+bxrW99q94zkYKCAvr27bvfsOeRrrFBj7vHfPrKV77i0vKWLVvmv/rVr/zkk092oN6pS5cuPmTIEB86dKinpqbWtsfFxfmYMWP82Wef9d27dx/0daqrq72wsNCnT5/u9957r0+dOtU/++wzr66ublSdlZWV/uc//9lHjhzpN998s8+ZM8crKysPus6GDRu8oqKi0dvC3b20tNSnTJni1113nf/Xf/2Xv/zyy75ixYpG19lcxcXF/s1vftP79u3rV199tT/wwAOel5fnS5cu9ffee89nzJjhzzzzjC9btsxrampq16upqfH33nvPf/SjH/kpp5ziWVlZHhcX54C3b9/eR4wY4T/+8Y/9/vvv97vvvtt/8IMf+CWXXOKjRo3y733ve3733Xf7I4884jfddJPn5OTU/tuamV966aU+e/Zsr66u9mXLlvnjjz/uP/zhD/2WW27x9957r3abbNq0ya+//vra1+3cubM/+OCDXllZ6TU1NT5lyhQ/9thja+f36tXLp06d6tXV1f7CCy94r169HPDk5GRv166d//rXv/bKykrfsWOHjx8/3gE/66yz/NRTT63d75KTk7179+6+cOHC2m2xatUqz8nJ8eTkZDczHz58uK9Zs8bdg/1n4sSJnpiY6IB///vf32//qa6u9jvuuMMBHzJkiC9evHi/f5+dO3f6f/zHf/jNN9/sO3fuPODf7x//+IePHTvW33333Xr/fffs2dPg/tpagHxvRMbGPORdQX9YrFmzxj/++GOfPXu2v/LKK/7kk0/6xIkT/frrr/dx48b52LFj/Sc/+Yn/7ne/82nTpnlRUVGsS5Z6VFZWelFRUZM7uZqaGi8oKPBHH33UV6xY0eTXXbRokd93332+YcOGA+Zt3LjRb731Vv/DH/7ge/bs2W9eWVmZ//KXv/RLLrnECwoKDqjpscce85SUFB88eLD//ve/96KiIl+8eLFnZWV5Wlqaf/DBB15YWOg9e/b0tLQ0//jjj33GjBmekpLi6enp/vTTT/uIESMc8Kuuuqo20MeNG+e7d+/23bt3+9e//nUH/PLLL/f09HRPSkryyZMne3V1tU+ZMsWzsrJqO8ATTjjB58+f7+5BB3Hvvfd6XFycm5knJib6I488sl/9U6ZM8fT0dB8yZIivXbv2gG3z6aef+r333uslJSUHzKuoqPAHH3zQ8/Pzm/zvEdViQQ90AD4CFgCLgXsi7U8Aq4D5kWlopN2AB4FCYCEwvKHXUNCLHL1qamr2O5NxD47g+/Tp48nJyZ6VleVdunTxuXPn1s5fsmSJ9+/fv/as9Lnnnqud96c//cnNzE8//XTPzc11M/P77rvPa2pqvKioyEeNGuWAZ2dnO+CnnHKKv//++z579mzPzMz09u3b+3333ecXX3yxA37NNdf42rVrfcyYMQ74hAkTfNGiRX7BBRc44Lm5ud65c2fPzMysDe2amhp/+OGHvUOHDrU1PvLII15VVeXu7m+88YYPHDjQAb/tttuave1aMugN6BS5nwjkASMjQX9FPcuPBV6PrDcSyGvoNRT0IvJFxcXFPmTIEO/atWvtUXZd27dv9/vvv9+Li4sPmPfCCy94u3btPDk52WfMmLHfvOrqav/d737ngwYN8r/+9a/7Dd1t2rSpNtDbt2/vkydPru2Eqqqq/K677qodAjvmmGN80qRJXl1d7QUFBd6rVy/v2LGjP/744z527FgHfPTo0T579mw/55xzajuFSy65xAHv06ePv/zyywd0ck3RKkM3QEdgLnDqQYL+L8A1dR4vAzIP9rwKehGpT3l5ue/YsaNZ6y5cuNCXL1/e5PWqq6v9mWee8UWLFtU7/8UXX/QbbrjhgOHNDRs21A4jdejQwR966KHaEK+pqfFnnnnGu3fv7p06dfLf/va3vnfv3qb/UV/Q2KBv1FU3ZhYPfAL0Bf7k7reb2RPAaUA5MAu4w93LzexV4Dfu/m5k3VnA7e6e/4XnnABMAOjZs+dX1qxZ02AdIiJtWVlZGQ899BDjxo3jxBNPPGB+RUUF1dXVB1wx1FyNveqmUb8w5e7V7j4UyAZGmNlg4E5gAHAK0AW4Pfra9T1FPc852d1z3T03PT29MWWIiLRpHTt25Pbbb6835AHatWvXYiHfFE36KUF33w7MAUa7e3RgrBx4HBgRWWw9cFyd1bKBohaoVUREmqHBoDezdDNLjdxPAkYBS80sM9JmwGVAQWSVGcB3LDAS2OHuxa1SvYiINKgxX6CeCTwZGaePA6a5+6tmNtvM0gmGauYDP44s/xrBlTeFQBnwvZYvW0REGqvBoHf3hcCwetrP/5LlHbjh0EsTEZGW0KQxehEROfIo6EVEQk5BLyIScgp6EZGQU9CLiIScgl5EJOQU9CIiIaegFxEJOQW9iEjIKehFREJOQS8iEnIKehGRkFPQi4iEnIJeRCTkFPQiIiGnoBcRCTkFvYhIyCnoRURCTkEvIhJyDQa9mXUws4/MbIGZLTazeyLtvc0sz8yWm9nzZtYu0t4+8rgwMj+ndf8EERE5mMYc0ZcD57v7ycBQYLSZjQR+C9zv7v2AbcB1keWvA7a5e1/g/shyIiISIw0GvQd2RR4mRiYHzgf+Fml/Ergscv/SyGMi8y8wM2uxikVEpEkaNUZvZvFmNh/YBMwEVgDb3b0qssh6oEfkfg9gHUBk/g7g2Hqec4KZ5ZtZfklJyaH9FSIi8qUaFfTuXu3uQ4FsYAQwsL7FIrf1Hb37AQ3uk909191z09PTG1uviIg0UZOuunH37cAcYCSQamYJkVnZQFHk/nrgOIDI/GOArS1RrIiINF1jrrpJN7PUyP0kYBSwBPg7cEVksWuBVyL3Z0QeE5k/290POKIXEZHDI6HhRcgEnjSzeIKOYZq7v2pmnwLPmdmvgHnAY5HlHwOmmFkhwZH81a1Qt4iINFKDQe/uC4Fh9bSvJBiv/2L7XuDKFqlOREQOmT4ZKyIScgp6EZGQU9CLiIScgl5EJOQU9CIiIaegFxEJOQW9iEjIKehFREJOQS8iEnIKehGRkFPQi4iEnIJeRCTkFPQiIiGnoBcRCTkFvYhIyCnoRURCTkEvIhJyCnoRkZBT0IuIhFyDQW9mx5nZ381siZktNrOfRtr/08w+N7P5kWlsnXXuNLNCM1tmZhe35h8gIiIH1+CPgwNVwG3uPtfMUoBPzGxmZN797v6Hugub2SDgauBEIAt428z6u3t1SxYuIiKN0+ARvbsXu/vcyP2dwBKgx0FWuRR4zt3L3X0VUAiMaIliRUSk6Zo0Rm9mOcAwIC/SdKOZLTSzv5pZWqStB7CuzmrrqadjMLMJZpZvZvklJSVNLlxERBqn0UFvZp2AF4Fb3L0UmAT0AYYCxcB90UXrWd0PaHCf7O657p6bnp7e5MJFRKRxGhX0ZpZIEPLPuPt0AHff6O7V7l4DPMq+4Zn1wHF1Vs8GilquZBERaYrGXHVjwGPAEnf/Y532zDqLfQ0oiNyfAVxtZu3NrDfQD/io5UoWEZGmaMxVN2cA3wYWmdn8SNtdwDVmNpRgWGY18CMAd19sZtOATwmu2LlBV9yIiMROg0Hv7u9S/7j7awdZZyIw8RDqEhGRFqJPxoqIhJyCXkQk5BT0IiIhp6AXEQk5Bb2ISMgp6EVEQk5BLyIScgp6EZGQU9CLiIScgl5EJOQU9CIiIaegFxEJOQW9iEjIKehFREJOQS8iEnIKehGRkFPQi4iEnIJeRCTkGvPj4MeZ2d/NbImZLTazn0bau5jZTDNbHrlNi7SbmT1oZoVmttDMhrf2HyEiIl+uMUf0VcBt7j4QGAncYGaDgDuAWe7eD5gVeQwwBugXmSYAk1q8ahERabQGg97di919buT+TmAJ0AO4FHgystiTwGWR+5cCT3ngQyDVzDJbvHIREWmUJo3Rm1kOMAzIAzLcvRiCzgDoFlmsB7CuzmrrI20iIhIDjQ56M+sEvAjc4u6lB1u0njav5/kmmFm+meWXlJQ0tgwREWmiRgW9mSUShPwz7j490rwxOiQTud0UaV8PHFdn9Wyg6IvP6e6T3T3X3XPT09ObW7+IiDSgMVfdGPAYsMTd/1hn1gzg2sj9a4FX6rR/J3L1zUhgR3SIR0REDr+ERixzBvBtYJGZzY+03QX8BphmZtcBa4ErI/NeA8YChUAZ8L0WrVhERJqkwaB393epf9wd4IJ6lnfghkOsS0REWog+GSsiEnIKehGRkFPQi4iEnIJeRCTkFPQiIiGnoBcRCTkFvYhIyCnoRURCTkEvIhJyCnoRkZBT0IuIhJyCXkQk5BT0IiIhp6AXEQk5Bb2ISMgp6EVEQk5BLyIScgp6EZGQU9CLiIRcg0FvZn81s01mVlCn7T/N7HMzmx+ZxtaZd6eZFZrZMjO7uLUKFxGRxmnMEf0TwOh62u9396GR6TUAMxsEXA2cGFnnz2YW31LFiohI0zUY9O7+D2BrI5/vUuA5dy9391VAITDiEOoTEZFDdChj9Dea2cLI0E5apK0HsK7OMusjbSIiEiPNDfpJQB9gKFAM3Bdpt3qW9fqewMwmmFm+meWXlJQ0swwREWlIs4Le3Te6e7W71wCPsm94Zj1wXJ1Fs4GiL3mOye6e6+656enpzSlDREQaoVlBb2aZdR5+DYhekTMDuNrM2ptZb6Af8NGhlSgiIocioaEFzOxZ4Fygq5mtB34BnGtmQwmGZVYDPwJw98VmNg34FKgCbnD36tYpXUREGsPc6x1CP6xyc3M9Pz8/1mWIiBxRzOwTd89taDl9MlZEJOQU9CIiIaegFxEJOQW9iEjIKehFREJOQS8iEnIKehGRkFPQi4iEnIJeRCTkFPQiIiGnoBcRCTkFvYhIyCnoRURCTkEvIhJyCnoRkZBT0IuIhJyCXkQk5BT0IiIhp6AXEQm5BoPezP5qZpvMrKBOWxczm2lmyyO3aZF2M7MHzazQzBaa2fDWLF5ERBrWmCP6J4DRX2i7A5jl7v2AWZHHAGOAfpFpAjCpZcoUEZHmajDo3f0fwNYvNF8KPBm5/yRwWZ32pzzwIZBqZpktVayIiDRdc8foM9y9GCBy2y3S3gNYV2e59ZG21rF5M/zrv8Lu3a32EiIiR7qWfjPW6mnzehc0m2Bm+WaWX1JS0rxXe/tteOABOOUUWLy4ec8hIhJyzQ36jdEhmcjtpkj7euC4OstlA0X1PYG7T3b3XHfPTU9Pb14VV18dhP3WrUHYP/lkw+uIiBxlmhv0M4BrI/evBV6p0/6dyNU3I4Ed0SGeVnP++TB/PowcCd/9bjDt2NGqLykiciRpzOWVzwIfACeY2Xozuw74DXChmS0HLow8BngNWAkUAo8CP2mVqr+oe3eYORPuvhumTIGTTgqO9EVEBHOvdwj9sMrNzfX8/PyWebK8PLj2Wli2DH7yE/jtb6FTp5Z5bhGRNsTMPnH33IaWC98nY089FebNC67GmTQJBg2C6dOhDXRoIiKxEL6gB0hKgj/+Ef75T0hLg8svhzFjYPnyWFcmInLYhTPoo844Az75BP77v+H992HwYPjZz2DLllhXJiJy2IQ76AESEuCnPw3G7MePD470+/QJxu737Il1dSIirS78QR+VmQmPPw4LFsBZZ8Edd8CAAfDBB7GuTESkVR09QR910knwv/8Lc+ZAfDycfTbcf7/erBWR0Dr6gj7qnHNg7lwYNw5uvRWuuAK2b491VSIiLe7oDXqA1NTg0ss//AFeeSUY3hk7Fh5+GFatinV1IiIt4ugOegAzuO02+OgjmDABPvsMbroJjj8eLrwQ3nkn1hWKiBwSBX3U8OHBN2EWFgZhf++9sGgRnHtuMI4/c6bG8UXkiKSgr0+/fnDnncHwzYMPBrcXXRRMCxbEujoRkSZR0B9MUlIwjFNYGBztz50Lw4bB978PRfV++7KISJujoG+M9u3h5puDwL/1Vnj6aejbF/793/WVyCLS5inomyItLbhCZ+lSuPRSmDgx+JTtAw9AeXmsqxMRqZeCvjmOPx6efRby82HoULjlFsjJgV//Ovi1KxGRNkRBfyi+8pXgapy334YhQ+Cuu+C444Jx/aVLY12diAigoD90ZnDBBfDmm8EVOVdeCX/5CwwcGHz6dupUDeuISEwp6FvSkCHwxBOwfn3w7Ziffw7f/CZkZ8O//VvwWETkMFPQt4Zu3eDnPw8+eDVzJpx5ZjB+n5MD11wDb70FZWWxrlJEjhKHFPRmttrMFpnZfDPLj7R1MbOZZrY8cpvWMqUegeLiYNQoeOklWLEiuETztdfg4ouD79k55xy45x748EOoro51tSISUof04+BmthrIdffNddp+B2x199+Y2R1AmrvffrDnadEfB2/rdu+Gd9+F2bOD6ZNPgq9W6NoVRo8OfvLw9NOhV69g/F9E5Es09sfBWyPolwHnunuxmWUCc9z9hIM9z1EV9F+0dWswlPPaa/D667A5sim7dYORI4PQv+CC4BO58fGxrVVE2pTDFfSrgG2AA39x98lmtt3dU+sss83dDzp8c1QHfV3V1cGVO3l5wXDOhx8G4/wQfFjr3HODYZ+xY4PLOEXkqHa4gj7L3YvMrBswE7gJmNGYoDezCcAEgJ49e35lzZo1za4j1DZsCIZ4Zs0KrtdfuzZoP+mkIPT79oWMDOjePRjuycyMbb0ictgclqD/wgv+J7AL+CEaumkd7rBkSTDM83//F4z1V1Xtv0xWFpx6ajCdeSaMGAGJibGpV0RaVasHvZklA3HuvjNyfybwS+ACYEudN2O7uPvPD/ZcCvpmqqyETZtg48bgyH/58uAHVPLygqt8ADp1Cr5P/4ILYPDg4OsbevaEdu1iW7uIHLLGBn3CIbxGBvCSBVeGJABT3f0NM/sYmGZm1wFrgSsP4TXkYBIToUePYPqikhL4xz+CIZ9Zs4KzgKi4uGCYJzd339H/iSfCMccE80QkVFps6OZQ6Ij+MCguDr5meeXKYFq6NDj6X7163zJxcdClCxx77P63aWlBpxIXF0zduwcf/OrWLWZ/jojEYIz+UCjoY2jjxiDwCwthy5Z909atwbRlC2zbFlwR5A41NcF39yQmwuWXw49+FAwN6UxA5LA7HEM3EgYZGfAv/9K0dZYsgcmTg+/1ee65oC0pCTp2hOTk4Lt9evfeNx1/fHCbna3PAojEgI7opfn27IHp04Nr/cvKgmnnTli3Lvid3XXrgjOAqMTE4PLPtLRgSKhLl+D9hbqdQu/ekJISu79J5AiiI3ppfUlJwbdzfpnKyuC6/1WrgmnlyuC9gm3bgmGhTz8Nvt5516791zv22OAL4Hr2DN4HiE45OcGVQz17aqhIpAkU9NJ6EhODn1rs0+fLl3EP3geIdgSrVwfTqlWwbFnwWYEtW/Y/M0hOhgEDoEOH4L2D6mpISAg+LZyTs++DY8ccE0ypqUHnccwx+v4gOSop6CW2zIIvdOvaFU45pf5lqquDsC8shMWLoaAg6ASqqoIj+/h4qKiAuXPh5ZeD+/VJSAheJy0t6GCqqoKzjoSE4IwhPT247dw5+EH4Dh2CqXPnYJ3U1GBYKXo2YRbMT00N5qekqCORNklBL21ffPy+4ZvTTz/4sjU1wZVEGzfCjh2wfXswbdkSfGFcSUkwdBQfHwR8QkIQ9iUlwZlEXl4wlFRefuCnjhsSFxecxcTHB/cTEoI3qDt1Cqb4+ODbS6NTQkLQnpISnKVE142PD+5HO5qkpOBspHv34M3zjIxgvfbtg6ldu2BKTAxuo294R99/S0gInicxUR3RUUpBL+ESFxcM27TEd/5UVwdvOJeWBp3Ftm3Bm81R7rB3b9C+bVuwTEVF0NnU1AQdSFlZ0HHs3h10HD17BqHsQcnxAAAFeUlEQVSenBw83rUreM5du4Ll9+4NXjd6f+/eoIbt24P7h8Is6Bji44P70bOhdu32dRqJicEU7QTrdlRJScHfVVUVTGZBe3JycJuauu/zF6mp+7Zh9LcWos/TqVPQ8SQk7HutunVJi1PQi3yZ+Ph9wZSVFdta3IMOYcOG4Gxl9+7grCM6VVYGU0XFvhCOhmZVVdBJlJcHt9GOKDp8VVGx//NEgzzaUa1bF3REZWX7OoCEhCDA656h1H0fpTmiHVG7dvteI9oRRTvH5OTgcbSTis6v21l17Ljvct+kpH1nRR06BK8T7Xzcg/U6dNjXyUWfNy5uX2cYvV/3b4++dvTMsH37oLaOHdvkJcQKepEjgVnwXkHnztC/f6yrOVC0I4p+yG779qA9GoQ1NfvObnbu3L9TqagI7tftuKJhXFUVPK7boUTPeqJnFxUV+zqr6BnQnj2H3vE0V4cOB4Z93aG1uh1FXBz88Idw662tWpKCXkQOXd2OKCcn1tUEHU9FRRD40fDfu3f/ISuz/TuIysp9ZzvRI/7op8GjbdGznaqq+juj6JlP3U7Gfd/ZVvSMK/p8NTXBey+tTEEvIuETHQZq3z7WlbQJ+tSJiEjIKehFREJOQS8iEnIKehGRkFPQi4iEnIJeRCTkFPQiIiGnoBcRCbk28QtTZlYCrGnm6l2BzS1YTphpWzWOtlPjaDs1Tmtup17unt7QQm0i6A+FmeU35qe0RNuqsbSdGkfbqXHawnbS0I2ISMgp6EVEQi4MQT851gUcQbStGkfbqXG0nRon5tvpiB+jFxGRgwvDEb2IiBzEER30ZjbazJaZWaGZ3RHretoKMzvOzP5uZkvMbLGZ/TTS3sXMZprZ8shtWqxrbQvMLN7M5pnZq5HHvc0sL7KdnjezdrGusS0ws1Qz+5uZLY3sW6dpnzqQmf1r5P9dgZk9a2YdYr1PHbFBb2bxwJ+AMcAg4BozGxTbqtqMKuA2dx8IjARuiGybO4BZ7t4PmBV5LPBTYEmdx78F7o9sp23AdTGpqu15AHjD3QcAJxNsM+1TdZhZD+BmINfdBwPxwNXEeJ86YoMeGAEUuvtKd68AngMujXFNbYK7F7v73Mj9nQT/IXsQbJ8nI4s9CVwWmwrbDjPLBi4B/ify2IDzgb9FFtF2AsysM3A28BiAu1e4+3a0T9UnAUgyswSgI1BMjPepIznoewDr6jxeH2mTOswsBxgG5AEZ7l4MQWcAdItdZW3GfwM/B6I/8nkssN3dqyKPtV8FjgdKgMcjw1z/Y2bJaJ/aj7t/DvwBWEsQ8DuAT4jxPnUkB73V06ZLiOows07Ai8At7l4a63raGjMbB2xy90/qNtezqPar4Ch1ODDJ3YcBuznKh2nqE3mP4lKgN5AFJBMML3/RYd2njuSgXw8cV+dxNlAUo1raHDNLJAj5Z9x9eqR5o5llRuZnAptiVV8bcQbwVTNbTTD0dz7BEX5q5LQbtF9FrQfWu3te5PHfCIJf+9T+RgGr3L3E3SuB6cDpxHifOpKD/mOgX+Td7HYEb3jMiHFNbUJknPkxYIm7/7HOrBnAtZH71wKvHO7a2hJ3v9Pds909h2D/me3u3wT+DlwRWeyo304A7r4BWGdmJ0SaLgA+RfvUF60FRppZx8j/w+h2iuk+dUR/YMrMxhIcgcUDf3X3iTEuqU0wszOBfwKL2Df2fBfBOP00oCfBDnmlu2+NSZFtjJmdC/zM3ceZ2fEER/hdgHnAt9y9PJb1tQVmNpTgTet2wErgewQHi9qn6jCze4BvEFz9Ng/4AcGYfMz2qSM66EVEpGFH8tCNiIg0goJeRCTkFPQiIiGnoBcRCTkFvYhIyCnoRURCTkEvIhJyCnoRkZD7f26VX/dg478UAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "lower=20\n",
    "plt.plot(valid_loss_list[lower:], color='black', label='validation loss')\n",
    "plt.plot(train_loss_list[lower:], color='red', label='train loss')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4FWX+NvD7G0JCCwRCQAJRUJogkRLqIm0XBLEA6/ti0J8FBRGxIHbQVRaFKGJZd12RRXRti4DKrvhagFX5IUgQKaFIUwiEECCUECAk537/mDmHk0ZOCiQh9+e65krOzDNznplMnnvmmTlnjCRERESCyroCIiJSPigQREQEgAJBRERcCgQREQGgQBAREZcCQUREACgQRETEpUAQEREACgQREXEFl3UFiqJ+/fps2rRpWVdDRKRCWb169QGSkYWVq1CB0LRpUyQkJJR1NUREKhQz+y2QcuoyEhERAAoEERFxKRBERASAAkFERFwKBBERAaBAEBERlwJBREQAVLDPIYiIVFjZ2UBqKpCS4gxpaQWXJYETJ4D0dOD4cefn+PFARMQ5raICQURKxuNxGi1vw+X96f3d48lb1jvt1KnA3ycrK+f7nD5dcNnsbCAjI7D38Xicst7lFqVO50tQEDBihAJBpFzLzgYOHAAOHszZGGZlFW95JHDy5JllZWQ44wKty8GDztHnvn3OEWig8+Zu1IvSKAb6HiVlBtSsCdSq5fwMCQmsbJMmTlmzgsvWqHGmfLVqBZctiaAgoH59oGFDZ6hb1xlXkOrVnfrUqgWEhp6bOuWiQJALD+k00tu3A3v3njla9W+wvY2f/9FrbpmZOctnZ5+ZlpXlnP4fOHD2ZZxvdeo4jc1FFwGXXXb2Bsdf7sa2KA1QUNCZ+fx/1qrlNLTBfs1MSd7HO7+cMwoEKf9I4PBh56g3JQU4dChnt8SBA2f6ZZOTgR07gGPHCl6et0GqUQOoUqXgcsHBZxq2Ro1yNmxBQUCPHk7D27Chc+TnLVuzJlC1avHXt1q1M8uqXr1ojfrZ1kekEAoEKRtZWcDu3c5R/PbtTqPuPRI/duzMxbd9+4D9+52j9YKEhJw5DY+OBnr1Ai691DlCjo4GwsLONNQ1agTewIpUMgoEKT2k02/tbci9R+3ewX/cvn15+9mrVj3TcEdGOg38FVecaey9Q716ORv5WrXUlSBSChQIEpiMDOCnn4AtW84c1e/e7RzNe7tvDh/O/86P4GCgQYMzDXq7dk4XjPco/rLLnK6Xs10kFJFzToFQmXkvviYn539vNOn0xy9fDqxZc+aIPjgYuOQSZ7joojNH6uHhOY/kvf3rhd1NISLlggLhQpSSAmzd6jTm3v55L9Lprtm+3Zmenn72ZVWvDnTpAjzyCNC9O9C2LXDxxTkvsIrIBUH/1RVdVpbTVfO//wt89ZUzbNt2ZnpQkHOE7t/HHhnpdNP07et020RFFXxvdO3aJbtjRkQqDAVCRbJ1K/DJJ8Cnnzp9+bk/gVmzJtCnD3DPPcDllzuNftOm6psXkYAoEMqzpCSn/375cuCbb4DERGd8x47A8OFn7rSpVcsZ1727Gn8RKTYFQnng8QCLFwOrV5+5g+eXX4A9e5zp1asD3boBo0YBQ4Y4F3NFREqZAqEs7d8PzJ4NzJwJ7NzpjGvQ4Ez/fufOzqdhr7xS/fgics4pEM4nEti0ybnw+/XXznD6tNP4T5sGDBrkdAOJiJQBBcK5dPo0sG7dmesA339/phuoZUtg3Dhg9GigdeuyraeICBQI5wYJvPceMGGC8508gPOdOj17An/4A9C/v64DiEi5o0AobTt2AGPGON1B3boBf/mLcx0gOrqsayYiclYKhNLyyy/AO+8AL7/sfIr39dedzwPoKxtEpIJQIBRXWppze+iKFcA//wn8+KPT+A8dCrzyivOUJhGRCkSBUBRr1wL33w+sX5/zS+CuvBJ48UXnmadRUWVXPxGRElAgBGrOHKcLqG5d51PC3q9tbtMGaNWqrGsnIlJiCoTCnDwJ3HcfMGsW0K8f8OGHzofHREQuMLriWZDDh4HXXgNiYpwwePJJ5wNlCgMRuUDpDCG3LVuAF15wzgROnHCeBbBokfMpYhGRC5gCwevUKefrI55/3rlt9JZbnM8TdOxY1jUTETkvFAiA83CZUaOc7xmKi3NuG1XXkIhUMpX3GgIJfPstcN11zldKZGQ4XUMffKAwEJFKKaBAMLOBZrbFzLaZ2eP5TL/EzBab2Toz+6+ZNfGbFm9mG9xhuN/4OWa208x+dof2pbNKAfjsM6BrV+fpYitWAM88A2zYoOsEIlKpFRoIZlYFwF8BDALQBkCcmbXJVWw6gHdJxgCYDGCqO+9gAB0BtAfQFcAjZlbbb75HSLZ3h59LvDaBmDrVecjM4cPA3/8O7NoF/OlPzlPHREQqsUDOELoA2EZyB8lMAB8BuCFXmTYAFru/L/Wb3gbAtySzSB4HsBbAwJJXuxhIYOJE5/bRESOcx1HefbfzNDIREQkoEBoD2O33Oskd528tgD+6vw8FEGZmEe74QWZWw8zqA+gLwP9rP59zu5leNrPQ/N7czEabWYKZJaR6v0q6qEjgwQedO4hGjQLefVdPIBMRySWQQLB8xjHX64cB9DazNQB6A9gDIIvkVwAWAVgO4EMAPwDIcud5AkBrAJ0B1APwWH5vTnImyViSsZGRkQFUN88CnDOB114Dxo8H3nwTqFKl6MsREbnABRIISch5VN8EwF7/AiT3khxGsgOAie64I+7P59xrBP3hhMtWd3wyHacAvA2na6r0mQFt2wJPPQW89JLzWkRE8gjkcwirALQws2ZwjvxvAjDCv4DbHXSIpAfOkf9sd3wVAOEkD5pZDIAYAF+50xqRTDYzAzAEwIZSWqe8HnjgnC1aRORCUWggkMwys3EAvgRQBcBskolmNhlAAsmFAPoAmGpmBPAdgHvd2asC+N5p83EUwC0kvV1G75tZJJyzhp8BjCm91RIRkaIyMvflgPIrNjaWCQkJZV0NEZEKxcxWk4wtrFzl/aSyiIjkoEAQEREACgQREXEpEEREBIACQUREXAoEEREBoEAQERGXAkFERAAoEERExKVAEBERAAoEERFxKRBERASAAkFERFwKBBERAaBAEBERlwJBREQAKBBERMSlQBAREQAKBBERcSkQREQEgAJBRERcCgQREQGgQBAREZcCQUREACgQRETEpUAQEREACgQREXEpEEREBIACQUREXAoEEREBoEAQERGXAkFERAAoEERExBVQIJjZQDPbYmbbzOzxfKZfYmaLzWydmf3XzJr4TYs3sw3uMNxvfDMzW2lmW83sX2YWUjqrJCIixVFoIJhZFQB/BTAIQBsAcWbWJlex6QDeJRkDYDKAqe68gwF0BNAeQFcAj5hZbXeeeAAvk2wBIA3AnSVfHRERKa5AzhC6ANhGcgfJTAAfAbghV5k2ABa7vy/1m94GwLcks0geB7AWwEAzMwD9AMxzy70DYEjxV0NEREoqkEBoDGC33+skd5y/tQD+6P4+FECYmUW44weZWQ0zqw+gL4BoABEADpPMOssyAQBmNtrMEswsITU1NZB1EhGRYggkECyfccz1+mEAvc1sDYDeAPYAyCL5FYBFAJYD+BDADwCyAlymM5KcSTKWZGxkZGQA1RURkeIIJBCS4BzVezUBsNe/AMm9JIeR7ABgojvuiPvzOZLtSfaHEwRbARwAEG5mwQUtU0REzq9AAmEVgBbuXUEhAG4CsNC/gJnVNzPvsp4AMNsdX8XtOoKZxQCIAfAVScK51nCjO89tAD4r6cqIiEjxFRoIbj//OABfAtgEYC7JRDObbGbXu8X6ANhiZr8AaAjgOXd8VQDfm9lGADMB3OJ33eAxAA+Z2TY41xT+UUrrJCIixWDOwXrFEBsby4SEhLKuhohIhWJmq0nGFlZOn1QWEREACgQREXEpEEREBIACQUREXAoEEREBoEAQERGXAkFERAAoEERExKVAEBERAAoEERFxKRBERASAAkFERFwKBBERAaBAEBERlwJBREQAKBBERMSlQBAREQAKBBERcSkQREQEgAJBRERcCgQREQGgQBAREZcCQUREACgQRETEpUAQEREACgQREXEpEEREBIACQUREXAoEEREBoEAQERGXAkFERAAoEERExKVAEBERAAEGgpkNNLMtZrbNzB7PZ/olZrbYzNaZ2X/NrInftBfMLNHMNpnZa2Zm7vj/usv82R0alN5qiYhIURUaCGZWBcBfAQwC0AZAnJm1yVVsOoB3ScYAmAxgqjtvDwC/AxAD4AoAnQH09pvvZpLt3WF/SVdGRESKL5AzhC4AtpHcQTITwEcAbshVpg2Axe7vS/2mE0A1ACEAQgFUBZBS0kqLiEjpCyQQGgPY7fc6yR3nby2AP7q/DwUQZmYRJH+AExDJ7vAlyU1+873tdhc95e1Kys3MRptZgpklpKamBlBdEREpjkACIb+GmrlePwygt5mtgdMltAdAlpk1B3A5gCZwQqSfmfVy57mZZDsAV7nD/+T35iRnkowlGRsZGRlAdUVEpDgCCYQkANF+r5sA2OtfgOReksNIdgAw0R13BM7ZwgqS6STTAXwBoJs7fY/78xiAD+B0TYmISBkJJBBWAWhhZs3MLATATQAW+hcws/pm5l3WEwBmu7/vgnPmEGxmVeGcPWxyX9d3560K4FoAG0q+OiIiUlyFBgLJLADjAHwJYBOAuSQTzWyymV3vFusDYIuZ/QKgIYDn3PHzAGwHsB7OdYa1JP8N5wLzl2a2DsDPcLqY3iq1tRIRkSIzMvflgPIrNjaWCQkJZV0NEZEKxcxWk4wtrJw+qSwiIgAUCCIi4lIgiIgIAAWCiIi4FAgiIgJAgSAiIi4FgoiIAFAgiIiIS4EgIiIAFAgiIuJSIIiICAAFgoiIuBQIIiICQIEgIiIuBYKIiABQIIiIiEuBICIiABQIIiLiUiCIiAgABYKIiLgUCCIiAkCBICIiLgWCiIgAUCCIiIhLgSAiIgAUCCIi4lIgiIgIAAWCiIi4FAgiIgJAgSAiIi4FgoiIAFAgiIiIS4EgIiIAAgwEMxtoZlvMbJuZPZ7P9EvMbLGZrTOz/5pZE79pL5hZopltMrPXzMzc8Z3MbL27TN94EREpG4UGgplVAfBXAIMAtAEQZ2ZtchWbDuBdkjEAJgOY6s7bA8DvAMQAuAJAZwC93XneADAaQAt3GFjSlRERkeIL5AyhC4BtJHeQzATwEYAbcpVpA2Cx+/tSv+kEUA1ACIBQAFUBpJhZIwC1Sf5AkgDeBTCkRGsiIiIlEkggNAaw2+91kjvO31oAf3R/HwogzMwiSP4AJyCS3eFLkpvc+ZMKWaaIiJxHgQRCfn37zPX6YQC9zWwNnC6hPQCyzKw5gMsBNIHT4Pczs14BLtN5c7PRZpZgZgmpqakBVFdERIojkEBIAhDt97oJgL3+BUjuJTmMZAcAE91xR+CcLawgmU4yHcAXALq5y2xytmX6LXsmyViSsZGRkQGuloiIFFUggbAKQAsza2ZmIQBuArDQv4CZ1Tcz77KeADDb/X0XnDOHYDOrCufsYRPJZADHzKybe3fRrQA+K4X1ERGRYio0EEhmARgH4EsAmwDMJZloZpPN7Hq3WB8AW8zsFwANATznjp8HYDuA9XCuM6wl+W932j0AZgHY5pb5olTWSEREisWcm3wqhtjYWCYkJJR1NUREKhQzW00ytrBy+qSyiIgAUCCIiIhLgSAiIgAUCCIi4lIgiIgIAAWCiIi4FAgiIgJAgSAiIi4FgoiIAFAgiIiIS4EgIiIAFAgiIuJSIIiICAAFgoiIuBQIIiLnQFpaGk6fPl3W1SgSBYKISCnbtWsXmjdvjr59++LEiRN5pqelpeHrr79GdnZ2ocvKyMjAG2+8gfPx7BoFgohIMRw9ehRvvvkmMjIycozPyspCXFwcTp48ieXLl+OWW27J0fD/9ttv6N69OwYMGICWLVvi9ddfx/Hjx/N9j7S0NAwYMAD33nsvfvzxx3O6PoACQUQqEZJ48cUXceONN/qGuLg4zJs3L6CjdX9PPvkkxowZgwEDBuDQoUO+8c888wyWL1+OWbNmYcaMGViwYAEeeughkMSGDRvQo0cPpKSk4OWXX0aDBg1w33334eKLL8ZTTz2FlJQU33KSkpJw1VVXYdWqVZg7dy66du1aatuhQCQrzNCpUyeKiBTX7NmzCYCXXXYZ27Zty7Zt27JRo0YEwEsvvZR/+ctfmJ6eXuhyNm/ezODgYPbs2ZMhISFs27Ytd+/eza+//ppmxpEjR/rKjh8/ngA4ZswYhoeHMyoqiuvXr/dNX7ZsGW+44QaaGUNDQ3nXXXfx888/Z3R0NMPCwrh48eISrzeABAbQxpZ5I1+UQYFwfqWlpfHw4cOFljtx4gRHjBjBrl278tChQ3mmL126lFFRUfzTn/7EzMzMc1FVKYeOHDnCVq1asXXr1hw5ciRnzZrFrVu3lvr7bN++nS1atODtt99+1v1148aNrFGjBvv27cusrCzf+KysLM6fP5/dunUjAAJgUFCQb7j22mt58uTJHMsaMmQIa9WqxX379nHx4sUMCwtjdHQ0L7roIl5++eU5QiU7O5s33ngjAbBVq1b89ddf863fli1bOGbMGFarVo0A2LBhQ/70008l3DoOBYIU2bFjx/j2229z1KhRbNOmDQHQzHjFFVdw9OjRnDNnTp4GPy0tjb179yYABgcHs3fv3jn+edavX886deqwbt26BMCOHTtyw4YNxa5jamoqd+zYUez5pWQ8Hg9//PFHnjhxotCyTz75JAHw6quvZr169Xz704IFC4r8vseOHeOqVavyjE9NTWWLFi0YFhbGKlWqMDo6ml9//XWechkZGYyJiWH9+vW5Z8+eAt9n2bJlfPrppzlp0iROmjSJ9957LwEwLi6O2dnZJMnvvvuOAPjnP//ZN99PP/3Ehg0bMjQ0lOvWrcuz3BMnTvCNN95gampqoeu6f/9+vvrqq6W6nysQpEiSk5PZvn17AmB4eDgHDRrEyZMnc/LkyRw4cCDr1KlDAKxZsyYfeOAB7ty5k3v37mVMTAyrVq3KDz74gO+//z4B8KabbmJ2djaTkpLYpEkTNmrUiL/99hvnz5/PyMhIhoSEcMKECYyPj/cNn3/+ue8friDr1q1jVFQUw8LCuG3btvO0ZSqfX3/9lQsWLKDH48kxPjs7mw8++CABsG3btly9enWBy9i9ezerVavGuLg4kk6QbN68mbGxsQwPD8/3KPmLL77gkiVL8rzvt99+y2bNmvka5oMHD5J0Gvnu3bszNDSUy5Yt48qVK9m6dWsC4N13381vv/2Wx48fJ0mOHTuWAPj5558XeXtMnTqVAPjYY4/R4/Gwa9eujIqK8i3ba+/evfmGQXmgQJCAbd26lc2aNWPNmjW5cOHCfBvm7Oxsrlq1irfeeiuDg4MZFBTEiIgI1qxZk19++aWv3LRp0wiA999/P2NiYlirVi2uWbPGNz0lJYXDhg3znZr7D61bt+Zbb72V79Hnd999xzp16jAqKorh4eHs3LkzT506dW42SBnJzMxkfHw8p0yZwqVLl/q6HdLT07lkyRJOmTKFr7zyylmD8+TJk1y+fDmnT5/Op59+mseOHctT5tSpU3z++edz/F28PB4Pu3TpQgDs378/d+3a5ZtnxIgRBMARI0YwKiqKwcHBfOaZZ/LtBrz99tsZEhLCnTt35hi/detWhoWFsUePHjx9+rTvPZ9//nnfftChQwe+//77PHr0KMePH08z42WXXcYJEyYwODiYF110ERcuXMhhw4bRzDhv3jzf8jMyMnzzeM9ar7zySgLghAkTCv8j5MPj8fCee+4hAN++O3v27GItq6woECQgCQkJjIyMZEREBFeuXBnQPLt37+ajjz7KTp068ccff8wxzePx+I7GgoODc4SFv4yMDB4/fpzHjx/n0aNH+f777/vOUBo0aODrc964cSMXLFjA0NBQX//r/PnzS/QPfj4dOXKEt912G3/++eezlktPT+c111yTIyCrVKnC5s2bs0qVKjnGP/bYY3nm37lzJ3//+98zJCQkR9nOnTtz//79vnLHjh3j1VdfTQBs3rw5MzIycizno48+8h2J16xZk3Xq1OFbb73FAQMGEACnTp1Kj8fDQ4cO8eabb863G/Dnn3+mmRX49/nggw8IgBMnTmR2djYfeOABX9DMnDmTrVq1IgBWrVqVADh27FhfOK5Zs4bt2rXzrd8rr7yS73scOHCA//73v/nEE0+wV69evP7660t0AHH69Gled911BMB27drluAZRESgQykB2dnaF2VE8Hg/nzJnDWrVq8eKLL+bmzZtLbdlZWVl89NFHOX/+/CLXafHixRw2bBgjIiJyNGxdu3bN0f9aUBfAgQMHmJyc7BvWrVvHN998k7feeitbtGjBNm3a8K677uLs2bO5ZcuWgOqVmprK7t2788477wzoIru/cePGEQD79u1bYJkDBw6wW7duDAoK4t///ncePHiQn3/+OSdOnMihQ4dy0qRJXLRoEQ8dOsQxY8YQAF9//XXf/GvXrmWjRo0YHh7Ohx9+mAsWLGBycjI/++wzVqtWjS1btuTOnTu5f/9+du7cmUFBQb6+8YkTJ/qWc/LkSTZr1szX4G3bto09e/b0hVN+R8Xz5s1j/fr1GRoayhdffJFZWVkcMGAA69atm+8NBl4jR46kmbFfv34EwPHjx/vOfLKzs7lw4ULecccd/Oqrr/LMe/LkST777LN88cUXA/oblJb09HSOGzeOCQkJ5/V9S4MC4TzLzs7mgAED2LNnT9+pcHmVnJzM66+/ngB41VVXMSkpqayrlIe3z3n27NmMj4/PcyvgiRMnfBcJX3rpJQ4fPpzR0dH5dkV5zzpuuOEGDh482HeBGwBHjx591r+Xt586JCSEQUFBjI6O5jfffBPQOqxYsYJm5uv/XrZsWZ4yu3bt4uWXX86QkJCAAtR7pBoUFMRPP/3U15XWuHHjHLcyen3//fcMDw9no0aN2LJlS1arVo2fffYZSfq6/7xH9zNmzCCAHGd1WVlZnDlz5lnXOSUlhUOGDPFdWwDAl1566azrkZ6e7uvvj4+Pz3PdQEqXAuEcyMzM5IIFC/Ltl505c6avkZkxY8Z5qY+30Xzvvfe4ZMmSPI3moUOHuGjRIv7tb3/zDfHx8YyIiGBoaChfeumlCnNGk59NmzaxRo0aBMDo6GgOHz6c06dP5xtvvOEb3nvvPW7bti1Hg5Odnc2NGzfyoYceIgAOGTIk3+sWWVlZHDp0KM2M8+fP54oVK3zdGWPHjj3rHSOZmZmMiYlh48aNuW/fPjZo0IBXX311jjLHjx9n27ZtWbt2bS5dujTg9U5PT2eXLl1YvXp1X1fab7/9VmD59evX+669fPfdd77xqampjIiIYI8ePXjw4EHWrVuXAwYMCLge/jweD//5z3+yTp06vPTSS/PcppmfPXv25KiPnDsKhGI4ffo0586dyxkzZuTodyWdf6oOHTr4Lrb5X0hLTk5meHg4e/fuzcGDB7NGjRoF3mvsfZ+PPvqIM2fOLNZ9+fPmzePgwYN9t/L59zl37NiRcXFxvttG8xtiY2O5cePGIr9vebRz507fhc/ieO2112hm7NWrF9PS0nzjPR6Pr7vn1Vdf9Y3PyMjw3WlTvXp1jh07Nt97670X1z/55BOSZHx8PAHkuE4zevToPEfkgUpJSWGrVq3YvXv3gG5lPHToEJOTk/OMnzNnDgHwiiuuoJlx7dq1Ra6Lv4MHDwZUHzm/FAhFcOzYMb766qts2rSpr9GsVq0ax4wZw02bNjE+Pp4hISGMjIzk/fffTwC87bbbfEedw4cPZ0hICDdv3sxff/2VNWvW5ODBg/OcBh89epQvv/wyL7nkEt/7dOrUiYmJiQHV0+Px+G6Ba9asme/C67p167ho0SJOmjSJ/fr1Y+PGjXnNNddwypQpXLJkCffu3ct9+/b5Bp2e5/Thhx+yatWqbNq0Kfv378/+/fvzd7/7HQHwoYceyneexMRE3nnnnQwJCaGZcdCgQXzhhRe4bNkyJiYmsnr16hwyZIiv/NGjR1mvXj1ee+21JMl//etfBV4gDlRmZmaJ/5Yej8fXj3/77beXaFlSfikQAuDxeDhr1ixfn3LPnj356aefMjExkaNGjWJoaKiv4R42bJjvrOGZZ54hAD711FNctGgRAfDZZ5/1LdfbFzt37lySZFJSEh977DHfvfy9evXiwoUL+fHHH+e5IFcQ/3vA4+LiLrhbLsvaN998wz59+rB79+6+YcKECYV+NiI5OZlPPvkkmzdvnuMsLCwsjLt3785RdsqUKQTAefPmsXbt2uzatWu5+OT29u3beeutt3Lv3r1lXRU5RxQIhdizZ4/vNr8+ffrwhx9+yFNm3759nDZtGj/++OMcR2Iej4cjR44kANapU4etW7fO0Wd6+vRpduzYkQ0bNuRtt93GqlWrMigoiDfeeCNXrFiR4z38L8jVr1+f1113HadNm8YlS5YwISHBN3jvAb///vsLbaSkbKSkpPDTTz/l448/zv/85z95ph8+fJjh4eG+/Sb3Pfoi54oCoQAej4cffPAB69aty+rVq/O1114rVgObmZnJgQMHEkC+F8ZWr17NoKAg1qhRg/fddx+3b99+1jotWLCAd9xxh++iZX6D9x5wqbi8Z5cff/xxWVdFKpFAA8GcshVDbGwsExISij1/amoqxo4di3nz5qFbt25455130LJly2Iv79SpU9i5cydat26d7/TExEQ0atQI9erVK9JyDxw4gJ9++gmnTp3yjYuKikKnTp2KXVcpHzweD3755ZcC9xmRc8HMVpOMLbRcIIFgZgMBvAqgCoBZJKflmn4JgNkAIgEcAnALySQz6wvgZb+irQHcRPJTM5sDoDeAI+6020n+fLZ6lCQQFi5ciFGjRiEtLQ2TJ0/Gww8/jODg4GItS0SkIgk0EAptEc2sCoC/AugPIAnAKjNbSHKjX7HpAN4l+Y6Z9QMwFcD/kFwKoL27nHoAtgH4ym++R0jOC3Sliuvuu+/GzJkz0b6+xVz3AAAFDElEQVR9e3zzzTdo167duX5LEZEKJ5AnpnUBsI3kDpKZAD4CcEOuMm0ALHZ/X5rPdAC4EcAXJDPymXZONW/eHE899RRWrlypMBARKUAgfSaNAez2e50EIPez3NYC+COcbqWhAMLMLILkQb8yNwGYkWu+58zsaThh8jjJUzgHHnnkkXOxWBGRC0ogZwiWz7jcFx4eBtDbzNbAuS6wB0CWbwFmjQC0A/Cl3zxPwLmm0BlAPQCP5fvmZqPNLMHMElJTUwOoroiIFEcggZAEINrvdRMAe/0LkNxLchjJDgAmuuOO+BX5vwA+IXnabx7v5+hPAXgbTtdUHiRnkowlGRsZGRnQSomISNEFEgirALQws2ZmFgKn62ehfwEzq29m3mU9AeeOI39xAD7MNU8j96cBGAJgQ9GrLyIipaXQQCCZBWAcnO6eTQDmkkw0s8lmdr1brA+ALWb2C4CGAJ7zzm9mTeGcYXyba9Hvm9l6AOsB1AcwpURrIiIiJVKpPpgmIlIZBfo5hEC6jEREpBJQIIiICAAFgoiIuCrUNQQzSwXwWzFnrw/gQClW50Kl7RQ4bavAaDsF5lxup0tIFnrffoUKhJIws4RALqpUdtpOgdO2Coy2U2DKw3ZSl5GIiABQIIiIiKsyBcLMsq5ABaHtFDhtq8BoOwWmzLdTpbmGICIiZ1eZzhBEROQsKkUgmNlAM9tiZtvM7PGyrk95YWbRZrbUzDaZWaKZPeCOr2dmX5vZVvdn3bKua3lgZlXMbI2Z/cd93czMVrrb6V/ulz9WamYWbmbzzGyzu1911/6Ul5mNd//nNpjZh2ZWrTzsTxd8IPg9AnQQnCe7xZlZm7KtVbmRBWACycsBdANwr7ttHgewmGQLuA8vKsM6licPwPmCR694AC+72ykNwJ1lUqvy5VUA/49kawBXwtle2p/8mFljAPcDiCV5BZxn1d+EcrA/XfCBgMAeAVopuc+k+Mn9/Ricf97GcLbPO26xd+B8PXmlZmZNAAwGMMt9bQD6AfA+E7zSbyczqw2gF4B/AADJTJKHof0pP8EAqptZMIAaAJJRDvanyhAI+T0CtHEZ1aXccr+mvAOAlQAakkwGnNAA0KDsalZuvALgUQAe93UEgMPu18MD2q8A4FIAqQDedrvWZplZTWh/yoHkHgDTAeyCEwRHAKxGOdifKkMgBPII0ErNzGoBmA/gQZJHy7o+5Y2ZXQtgP8nV/qPzKVrZ96tgAB0BvOE+PfE4Knn3UH7cayg3AGgGIApATThd2rmd9/2pMgRCoY8ArczMrCqcMHif5AJ3dIrfE+0aAdhfVvUrJ34H4Hoz+xVOl2M/OGcM4e4pP6D9CnD+15JIrnRfz4MTENqfcvoDgJ0kU93HCi8A0APlYH+qDIFQ6CNAKyu3H/wfADaRnOE3aSGA29zfbwPw2fmuW3lC8gmSTUg2hbP/LCF5M4ClAG50i2k7kfsA7DazVu6o3wPYCO1Pue0C0M3Marj/g97tVOb7U6X4YJqZXQPniK4KgNkknytklkrBzHoC+B7OY0y9feNPwrmOMBfAxXB23v9D8lCZVLKcMbM+AB4mea2ZXQrnjKEegDUAbiF5qizrV9bMrD2cC+8hAHYAuAPOgaf2Jz9m9iyA4XDu9FsD4C441wzKdH+qFIEgIiKFqwxdRiIiEgAFgoiIAFAgiIiIS4EgIiIAFAgiIuJSIIiICAAFgoiIuBQIIiICAPj/SuhkgLlulYEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(valid_accu_list[lower:], color='black', label='validation accuracy')\n",
    "plt.plot(train_accu_list[lower:], color='red', label='train accuracy')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "rand_batch=np.random.randint(train_num//batch_size)\n",
    "grad_lr=train_batch(rand_batch,parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:5: DeprecationWarning: object of type <class 'float'> cannot be safely interpreted as an integer.\n",
      "  \"\"\"\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65c2b16709fe4e8caa58acfb61353bdb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=20), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "lr_list=[]\n",
    "lower=-2.75\n",
    "upper=-0.75\n",
    "step=0.1\n",
    "for lr_pow in tqdm_notebook(np.linspace(lower,upper,num=(upper-lower)//step+1)):\n",
    "    learn_rate=10**lr_pow\n",
    "    parameters_tmp=combine_parameters(parameters,grad_lr,learn_rate)\n",
    "    train_loss_tmp=train_loss(parameters_tmp)\n",
    "    lr_list.append([lr_pow,train_loss_tmp])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XucjeX+//HXZ4ZxGIc9MaIYY+SUUzQkm5lMoiilVFu7kxJ717dz+4vdYX/b9i58O9jfvdukUMkhpZKsEbaSlORQjBBaQ0JIkWgGc/3+WGv8hoZZw8y615p5Px+Pecxyr3td9+dec5v33Nd9res25xwiIiIxXhcgIiKRQYEgIiKAAkFERIIUCCIiAigQREQkSIEgIiKAAkFERIIUCCIiAigQREQkqILXBRRH7dq1XXJystdliIhEleXLl+92ziUWtV5UBUJycjLLli3zugwRkahiZptDWU9dRiIiAigQREQkSIEgIiKAAkFERIIUCCIiAigQREQkSIEgIiKAAkFEJKKtWbOGxx57jB07dpT6thQIIiIRbMmSJQwfPpyDBw+W+rYUCCIiEczv9xMbG0uDBg1KfVsKBBGRCOb3+2nQoAEVKpT+TEMKBBGRCOb3+2nUqFFYtqVAEBGJYAoEERHh4MGD7NixQ4EgIlLeZWdnA4Gp/8NBgSAiEqH8fj+AzhBERMq7iAwEM7vXzLLMbI2Z3Vdg+d1mtj64fFRwWUUze9nMVpvZWjMbdoI2LzazFWb2uZl9ZGbnlMwuiYiUDdnZ2VSqVIm6deuGZXtFDmw1s1bAHUBHIBeYY2azgfrAlUAb51yOmdUJvuRaoJJzrrWZVQW+NLOpzrns45oeA1zpnFtrZncCjwC3lsROiYiUBX6/n+TkZGJiwtOZE8onHVoAS5xzBwDMbCHQF0gFRjjncgCcczuD6zsg3swqAFUIhMi+Qtp1QI3g45rAtlPdCRGRsiicQ04htC6jLCDNzGoF/+LvBTQAmgJdzexTM1toZh2C678B/AxsB7YATznn9hTS7kDAZ2ZbgZuAEYVt3MwGmdkyM1u2a9euYu2ciEg0i7hAcM6tBUYC84A5wBfAYQJnFwlAJ+BPwHQzMwJdS0eAs4BGwINmllJI0/cDvZxz9YGJwDMn2P4451yqcy41MTGxmLsnIhKd9u7dyw8//BBZgQDgnBvvnGvvnEsD9gAbgK3Amy5gKZAH1AZuAOY45w4Fu5EWE+heOsrMEoG2zrlPg4teAzqXyB6JiJQB4R5hBKGPMqoT/J4EXA1MBd4GMoLLmwJxwG4C3UQZFhBP4Axi3XFN/gDUDL4O4BJg7entiohI2eFFIIQ6fd4MM6sFHALucs79YGYTgAlmlkXgwvEtzjlnZs8R6ALKAgyY6JxbBWBmPmCgc26bmd0RbDePQEDcVrK7JiISvSI2EJxzXQtZlgvcWMjy/QSGnhbWTq8Cj98C3gq5UhGRcsTv91OjRg0SEhLCtk19UllEJALljzAKjNUJDwWCiEgECveQU1AgiIhEHOcc2dnZYZvlNJ8CQUQkwuzcuZMDBw7oDEFEpLzzYoQRKBBERCJO/o1xFAgiIuVc/hmCriGIiJRzfr+fxMREqlWrFtbtKhBERCKMF0NOQYEgIhJxFAgiIsKRI0fYsmWLAkFEpLz79ttvOXTokAJBRKS88+ozCKBAEBGJKAoEEREBAoFgZiQlJYV92woEEZEI4vf7Ofvss4mLiwv7thUIIiIRxKshp6BAEBGJKAoEEREhJyeHbdu2KRBERMq7LVu24JxTIIiIlHdeDjkFBYKISMRQIIiICBAIhIoVK3LWWWd5sn0FgohIhPD7/TRs2JDY2FhPtq9AEBGJEF4OOQUFgohIxFAgiIgI+/fvZ/fu3QoEEZHyzusRRqBAEBGJCFETCGZ2r5llmdkaM7uvwPK7zWx9cPmo4LKKZvayma02s7VmNuwEbZqZ/d3Mvgqud0/J7JKISPTJD4Tk5GTPaqhQ1Apm1gq4A+gI5AJzzGw2UB+4EmjjnMsxszrBl1wLVHLOtTazqsCXZjbVOZd9XNO3Ag2A5s65vAKvFxEpd/x+P1WrViUxMdGzGooMBKAFsMQ5dwDAzBYCfYFUYIRzLgfAObczuL4D4s2sAlCFQIjsK6TdPwI3OOfyjnu9iEi5kz/CyMw8qyGULqMsIM3MagX/4u9F4C/7pkBXM/vUzBaaWYfg+m8APwPbgS3AU865PYW02xi43syWmVmmmTU57b0REYlS2dnZnl4/gBACwTm3FhgJzAPmAF8AhwmcXSQAnYA/AdMtEG0dgSPAWUAj4EEzSymk6UrAL865VOAFYEJh2zezQcHQWLZr165i7p6ISORzznn+GQQI8aKyc268c669cy4N2ANsALYCb7qApUAeUBu4AZjjnDsU7AZaTKB76XhbgRnBx28BbU6w7XHOuVTnXKqXfWsiIqVlz549/PTTT9ERCPkXfM0sCbgamAq8DWQElzcF4oDdBLqJMoKjiOIJnEGsK6TZo68H0oGvTn03RESiVyQMOYXQLioDzDCzWsAh4C7n3A9mNgGYYGZZBC4c3+Kcc2b2HDCRwLUHAyY651YBmJkPGOic2waMACab2f3AfmBgie6ZiEiUiKpAcM51LWRZLnBjIcv3Exh6Wlg7vQo8/hHoHXKlIiJlVKQEgj6pLCLiMb/fzxlnnEGNGjU8rUOBICLisUgYYQQKBBERzykQRESEvLy8iPhQGigQREQ8tX37dnJzcxUIIiLlXSTMcppPgSAi4qFIGXIKCgQREU/pDEFERIDALKf16tWjcuXKXpeiQBAR8VKkDDkFBYKIiKcUCCIiwqFDh/jmm28UCCIi5d0333xDXl6eAkFEpLyLpCGnoEAQEfGMAkFERIBAIMTGxlK/fn2vSwEUCCIinvH7/SQlJVGhQqg3ryxdkVFFKdu4cSMA9erVIz4+3uNqREQCImnIKZSTQLj33nvx+XwAVK9enXr16h39qlu37jH/zv9KSEjAzDyuXETKMr/fz+WXX+51GUeVi0B49NFHue6669ixYwfbt28/+rVs2TK2b9/Ozz///KvXxMXFHRMWZ599NsnJyTRs2JCGDRuSnJxMYmKiQkNETsmBAwf47rvvImIOo3zlIhA6depEp06dTvj8/v37jwmK/K/8ANm4cSMffPABP/744zGvq1KlytGAyA+Jgo/r1atHTIwu04jIr2VnZwORM8IIykkgFKVatWo0adKEJk2anHS9vXv3snnzZrKzs9m8efMxj5cvX87u3buPWb9ixYo0aNCA5ORkUlJSaN68Oc2aNaN58+YkJydHzIUkEQm/SBtyCgqEYqlZsyZt2rShTZs2hT7/888//yoo8h/PnDmTF1988ei6cXFxNGnS5GhA5IdFs2bNqFmzZrh2SUQ8ojOEMi4+Pp5zzz2Xc889t9Dnv//+e9avX8/69etZt24d69atY82aNcycOZMjR44cXa9evXq/CoqWLVtSv359XbMQKSP8fj+VK1embt26XpdylAIhjGrVqkXnzp3p3LnzMctzc3P5+uuvjwZF/vfXXnuNH3744eh6tWvXpn379sd8paSkKCREopDf7yc5OTmi/v+ac87rGkKWmprqli1b5nUZYeOcY/fu3axdu5asrCxWrFjBihUryMrK4tChQwDUqFGDdu3aHRMSzZo1IzY21uPqReRk2rdvT926dY8OiS9NZrbcOZda1Ho6Q4hgZkZiYiKJiYmkpaUdXZ6Tk8OaNWtYsWIFK1euZMWKFYwdO5aDBw8CULVqVdq2bXs0INq1a0fLli2Ji4vzaldE5Dh+v58LL7zQ6zKOoUCIQpUqVTr6yz7f4cOHWb9+/dGziBUrVvDKK6/w3HPPAYEhshdeeCFpaWmkp6dzwQUXUKVKFa92QaRc+/HHH/nxxx8j6oIyKBDKjAoVKtCyZUtatmzJTTfdBEBeXh6bNm1i+fLlLFmyhIULF/L444/jnCMuLo6OHTuSnp5Oeno6F154IdWqVfN4L0TKh0gccgohTm5nZveaWZaZrTGz+wosv9vM1geXjwouq2hmL5vZajNba2bDimj7n2a2//R2QwoTExNDkyZN+N3vfsfo0aNZuXIle/bsYdasWdxzzz3k5uYyYsQIevToQUJCAp06dWLIkCHMnj2bvXv3el2+SJkVqYFQ5BmCmbUC7gA6ArnAHDObDdQHrgTaOOdyzKxO8CXXApWcc63NrCrwpZlNdc5lF9J2KvCbktkVCcVvfvMbLr/88qPzp/z00098/PHHfPjhhyxcuJBnn32WUaNGERMTw3nnnXe0iyktLY0zzjjD4+pFyoaoDQSgBbDEOXcAwMwWAn2BVGCEcy4HwDm3M7i+A+LNrAJQhUCI7Du+UTOLBf4XuCHYnnigevXq9OzZk549ewKB+VWWLFlyNCDGjh3L6NGjiY2NpUuXLvTp04c+ffpwzjnneFy5SPTy+/3UrFmThIQEr0s5RpHDTs2sBTATuBA4CPwHWAZ0DS6/FPgFeMg595mZVQQmARcDVYH7nXPjCmn3XiDGOfesme13zhXZgV3ehp1GgpycHJYuXcrcuXOZOXMmq1evBqBFixZceeWV9OnTh44dO2qYq0gx9O7dm23btrFy5cqwbC/UYachfQ7BzG4H7gL2A18SCIZLgAXAvUAH4DUgBegM3AncCiQAi4DLnHNfF2jvLGA6cJFz7vDJAsHMBgGDAJKSks7fvHlzkfVK6fH7/cyaNYuZM2eycOFCjhw5Qp06dbj88svp06cPl1xyCVWrVvW6TJGIdu6559KsWTPeeuutsGwv1EAI6aKyc268c669cy4N2ANsALYCb7qApUAeUJtAF9Ac59yhYDfSYgLdSwW1A84BNppZNlDVzDaeYNvjnHOpzrnUxMTEUMqVUtSoUSPuuece/vOf/7Br1y6mTJlCRkYGb7zxBldddRW1atWiT58+vPjii3z33XdelysScZxzZGdnR9z1Awhx2KmZ1XHO7TSzJOBqAt1HeUAG8IGZNQXigN3AFiDDzF4l0GXUCRhdsD3n3GygboH29zvn1CkdZRISEujfvz/9+/cnNzeXDz/8kHfeeYeZM2cya9YszIwLLriAPn360LdvX5o3b+51ySKe++677zh48GBEBkKok/XPMLMvgVnAXc65H4AJQIqZZQHTgFtcoP/pOaAakAV8Bkx0zq0CMDNfsLtIypi4uDi6d+/O//3f/5Gdnc3nn3/O448/zuHDh/nzn/9MixYt6NSpE88//7yGtEq5FomznObTXEZS6r799lumTZvGxIkTWbNmDZUrV+aaa65hwIABdOvWTTcRknJl6tSp3HDDDWRlZdGyZcuwbLNEryGInI6zzz6bBx98kNWrV7N06VJuvfVW3n33Xbp3705KSgp/+ctfjo7LFinr8o/1SLp1Zj4FgoSNmdGhQwfGjBnD9u3bmTJlCk2bNmX48OGkpKSQkZHBpEmTOHDggNelipQav99PnTp1iI+P97qUX1EgiCeqVKlC//79mTt3LtnZ2QwfPpzNmzdz8803U7duXe644w4++eQToqlLUyQUfr8/Iq8fgAJBIkBSUhKPPPIIGzZs4IMPPuDqq69mypQpdO7cmRYtWjBy5Ei2b9/udZkiJUKBIBKCmJgY0tPTeemll9ixYwfjx48nMTGRoUOH0rBhQ26//XbWrVvndZkip+zIkSNs2bJFgSBSHNWrV+e2225j0aJFrF+/nsGDBzNlyhTOPfdcrr76apYuXep1iSLFtnXrVg4fPqxAEDlVTZs25Z///CebN2/m4Ycf5v333+eCCy4gIyOD9957T9cZJGpE6iyn+RQIEjXq1KnD8OHD2bJlC08//TRfffUVl156Ke3bt2fatGkcPnzY6xJFTkqBIFLCqlevzgMPPMDXX3/NhAkTOHjwIP3796dZs2bH3FtaJNL4/X5iYmJo0KCB16UUSoEgUSsuLo4BAwbw5Zdf8uabb1K7dm3++Mc/kpyczJNPPsmPP/7odYkix/D7/Zx99tnExcV5XUqhFAgS9WJiYujbty9Llizh/fffp127dvz5z38mKSmJ//7v/2bbtm1elygCRPaQU1AgSBliZlx00UXMmTOHFStW0Lt3b55++mkaNWrEH/7wB3bs2OF1iVLOReq01/kUCFImtWvXjqlTp/LVV18xYMAAxo8fT5MmTRgxYgS//PKL1+VJOZSTk8O2bdsUCCJeady4MWPHjmXNmjVkZGQwbNgwWrRowRtvvKHhqhJWmzdvxjmnQBDxWtOmTZk5cybz5s2jWrVqXHvttaSnp7N8+XKvS5NyItKHnIICQcqZ7t27s3LlSsaOHcu6devo0KEDAwYM0IVnKXUKBJEIVKFCBQYPHsyGDRt46KGHmDx5Mk2bNuVvf/ubPsMgpcbv9xMXF8dZZ0XuTSMVCFJu1axZk1GjRrF27Vp69uzJo48+SvPmzZk2bZquL0iJ8/v9NGzYMKLvEBi5lYmESePGjZkxYwbvv/8+Z5xxBv3796dLly6aQE9KVKR/BgEUCCJHXXTRRSxbtowXX3yRTZs2ccEFF3DzzTezdetWr0uTMkCBIBJlYmNjuf3229mwYQPDhg1j+vTpR2/zmZub63V5EqV++uknvv/+ewWCSDSqXr06TzzxBGvXrqV379489thjdOrUiaysLK9LkygUDSOMQIEgclKNGjXi9ddf56233mLr1q2cf/75jBo1iiNHjnhdmkQRBYJIGXLVVVeRlZVF7969GTJkCOnp6WzcuNHrsiRK5AdCcnKyt4UUQYEgEqI6deowY8YMJk2aRFZWFm3btmXMmDEaoipF8vv9xMfHU7t2ba9LOSkFgkgxmBk33ngjWVlZdOnShTvvvJOePXtqJJKcVP4sp2bmdSknpUAQOQX169dnzpw5jBkzhsWLF9OqVSsmTZqkswUpVDQMOQUFgsgpMzP+8Ic/sGrVKlq1asXNN9/MNddcw65du7wuTSKIc06BIFJeNG7cmIULFzJq1Chmz55Ny5Ytefvtt70uSyLE999/z/79+xUIIuVFbGwsf/rTn1i+fDn169enb9++3HLLLbqvs0TNkFMIMRDM7F4zyzKzNWZ2X4Hld5vZ+uDyUcFlFc3sZTNbbWZrzWzYCdqcHHxtlplNMLOKJbNLIt5p1aoVS5Ys4dFHH2Xy5Mm0bt2aefPmeV2WeCh/eHKZCAQzawXcAXQE2gKXm1kTM+sGXAm0cc61BJ4KvuRaoJJzrjVwPjDYzJILaXoy0BxoDVQBBp7erohEhri4OP7617/yySefUK1aNXr06MF//dd/kZOT43Vp4oEFCxZQo0YNWrRo4XUpRQrlDKEFsMQ5d8A5dxhYCPQF/giMcM7lADjndgbXd0C8mVUg8Is+F9h3fKPOOZ8LApYC9U97b0QiSIcOHVixYgX33Xcfzz33HBdddJFuxFPOOOfIzMykR48eVKwY+Z0goQRCFpBmZrXMrCrQC2gANAW6mtmnZrbQzDoE138D+BnYDmwBnnLO7TlR48GuopuAOSd4fpCZLTOzZRq9IdGmSpUqPPvss7z++uusXr2a888/n48//tjrsiRMVq9ezbfffkuvXr28LiUkRQaCc24tMBKYR+CX9hfAYaACkAB0Av4ETLfApy46AkeAs4BGwINmlnKSTfwb+NA5t+gE2x/nnEt1zqUmJiaGvGMikaRfv34sWbKE+Ph4LrroIsaNG+d1SRIGPp8PgEsvvdTjSkIT0kVl59x451x751wasAfYAGwF3gz2+iwF8oDawA3AHOfcoWA30mIgtbB2zewvQCLwwOnvikhka9WqFZ999hkXX3wxgwcPZtCgQbquUMb5fD7atWtHvXr1vC4lJKGOMqoT/J4EXA1MBd4GMoLLmwJxwG4C3UQZFhBP4AxiXSFtDgR6Av2dc3mnvysikS8hIYF3332XYcOG8cILL9CtWzddVyijfvjhBz7++OOo6S6C0D+HMMPMvgRmAXc5534AJgApZpYFTANuCV4gfg6oRuDaw2fAROfcKgAz85lZ/h2mxwJnAp+Y2edm9liJ7ZVIBIuNjeWJJ57g9ddfZ9WqVbquUEbNmzePI0eORFUgWDTNvZKamuqWLVvmdRkiJSYrK4urrrqKLVu28K9//YtBgwZ5XZKUkFtvvZVZs2axc+dOYmNjPa3FzJY75wrtui9In1QW8VD+dYWMjAxdVyhD8vLyyMzMpGfPnp6HQXEoEEQ8lpCQwOzZsxk6dKiuK5QRK1euZOfOnVHVXQQKBJGIEBsby5NPPsn06dP54osvSE1N5ZNPPvG6LDlFPp8PM6Nnz55el1IsCgSRCHLttdeyZMkSqlSpQnp6uj6vEKV8Ph8dOnQg2j47pUAQiTCtW7fms88+o1u3bgwePJjBgwfrukIU2b17N59++mnUdReBAkEkIp1xxhn4fD6GDBnCuHHjyMjI4Pvvv/e6LAnBe++9h3NOgSAiJSc2NpYRI0Ywbdo0li9fTpcuXdiyZYvXZUkRfD4fiYmJnH/++V6XUmwKBJEId/311/Pee++xbds2OnfuzJo1a7wuSU7gyJEjzJkzh8suu4yYmOj79Rp9FYuUQ+np6SxatIgjR47QpUsXFi9e7HVJUoilS5eyZ8+eqOwuAgWCSNRo06YNH3/8MYmJiXTv3p1Zs2Z5XZIcJzMzk5iYGHr06OF1KadEgSASRRo1asTixYtp1aoVffv2ZeLEiV6XJAX4fD4uvPBCEhISvC7llCgQRKJMYmIiCxYsICMjg9tuu40RI0YQTXOSlVU7duxg+fLlUdtdBAoEkahUvXp13n33Xfr378+wYcN44IEHyMvTLPJemjMncNPHaA6ECl4XICKnJi4ujldffZXExERGjx7Nzp07mThxInFxcV6XVi75fD7q1atH27ZtvS7llCkQRKJYTEwMo0ePpl69egwbNozdu3czY8YMqlWr5nVp5cqhQ4eYO3cu/fr1I3An4eikLiORKGdmDB06lPHjxzN//nwyMjLYtWuX12WVK5988gl79+6N6u4iUCCIlBm33XYbb731FqtXr6ZLly5kZ2d7XVK5kZmZSYUKFejevbvXpZwWBYJIGdKnTx/mz5/Pzp076dy5M6tWrfK6pHLB5/PRpUsXatSo4XUpp0WBIFLG/Pa3v+Wjjz4iJiaGtLQ0Fi1a5HVJZdrWrVtZtWpV1HcXgQJBpExq2bIlH3/8MXXr1qVHjx7MnDnT65LKrMzMTCC6h5vmUyCIlFFJSUl89NFHtG3blquvvpopU6Z4XVKZ5PP5SEpK4txzz/W6lNOmQBApw2rXrs38+fNJT0/nxhtv5KWXXvK6pDIlJyeH+fPn06tXr6gebppPgSBSxlWrVo13332X7t27M2DAAN2WswR99NFH7N+/v0x0F4ECQaRcqFq1Ku+88w69e/dm8ODB/Otf//K6pDIhMzOTuLg4MjIyvC6lRCgQRMqJypUr8+abb3LVVVdx99138/TTT3tdUtTz+Xykp6cTHx/vdSklQoEgUo7ExcUxffp0rrvuOh566CGeeOIJr0uKWn6/n7Vr15aZ7iLQXEYi5U7FihWZPHkycXFxPPzww+Tm5vKXv/ylTFwUDaeyNNw0nwJBpByqUKECL730EhUrVuTxxx8nJyeHJ554QqFQDD6fj8aNG9OkSROvSykxCgSRcio2NpYXX3yRSpUqMWLECHJycnj66acVCiE4ePAgCxYsYODAgWXq/QrpGoKZ3WtmWWa2xszuK7D8bjNbH1w+Krisopm9bGarzWytmQ07QZuNzOxTM9tgZq+ZmSZxFwmzmJgY/v3vf3PPPffw7LPPcvfdd+tGOyFYuHAhBw8eLFPdRRDCGYKZtQLuADoCucAcM5sN1AeuBNo453LMrE7wJdcClZxzrc2sKvClmU11zmUf1/RI4Fnn3DQzGwvcDowpkb0SkZCZGaNHjyYuLo6nnnqKnJwcnn/+eWJiNObkRDIzM6lcuTLp6elel1KiQukyagEscc4dADCzhUBfIBUY4ZzLAXDO7Qyu74B4M6sAVCEQIvsKNmiBc6wM4IbgopeB/0GBIOIJM2PUqFFUqlSJv//97xw6dIjx48cTGxvrdWkRyefzkZGRQZUqVbwupUSF8idAFpBmZrWCf/H3AhoATYGuwW6fhWbWIbj+G8DPwHZgC/CUc27PcW3WAn50zh0O/nsrcPZp7ouInAYz429/+xt//etfefnll7nppps4fPhw0S8sZzZs2MDGjRvLXHcRhHCG4Jxba2YjgXnAfuAL4HDwtQlAJ6ADMN3MUgh0LR0Bzgo+v8jM5jvnvi7QbGFXYVxh2zezQcAgCEzWJSKl69FHHyUuLo6hQ4eSm5vLlClTdJ/mAnw+HwCXXXaZx5WUvJA6CZ1z451z7Z1zacAeYAOBv+rfdAFLgTygNoFuoDnOuUPBbqTFBLqXCtoN/CbYrQSB6xHbTrDtcc65VOdcamJiYnH3T0ROwZAhQ3jmmWeYMWMG/fr1Iycnx+uSIobP56N58+akpKR4XUqJC3WUUZ3g9yTgamAq8DaB6wCYWVMgjsAv+i1AhgXEEziDWFewPeecA94H+gUX3QJownaRCHL//ffz3HPPMWvWLK688koOHDjgdUme+/nnn/nggw/KZHcRhD51xQwz+xKYBdzlnPsBmACkmFkWMA24JfiL/jmgGoFrD58BE51zqwDMzGdmZwXbHAI8YGYbCVxTGF9SOyUiJePOO+/khRdeYO7cuVx22WXs27ev6BeVYQsWLCA3N7dMdhdBiB9Mc851LWRZLnBjIcv3Exh6Wlg7vQo8/prA9QYRiWADBw4kPj6em2++mYsvvpg5c+ZQq1Ytr8vyRGZmJvHx8XTt+qtfiWWCBhqLSJH69+/Pm2++yerVq0lPT2f79u1elxR2zjl8Ph/du3enUqVKXpdTKhQIIhKSK664gszMTLKzs+natSvZ2dlelxRWa9euZfPmzWX2+gEoEESkGLp168b8+fP5/vvv6dKlC+vWrSv6RWVEWR5umk+BICLF0qlTJxYuXMjhw4dJS0tj5cqVXpcUFj6fj9atW9OgQQOvSyk1CgQRKbY2bdrw4YcfUrlyZbp168bixYu9LqlU7du3j0WLFpUriUqCAAALTklEQVTp7iJQIIjIKWratCkfffQRderUoUePHsybN8/rkkrN/PnzOXz4cJnuLgIFgoichqSkJBYtWsQ555zD5Zdfzttvv+11SaUiMzOTGjVq0LlzZ69LKVUKBBE5LWeeeSYffPAB7dq1o1+/frz66qtel1Si8oeb9ujRg4oVK3pdTqlSIIjIaUtISGDevHmkpaVx0003MWZM2ZnJ/osvvmDbtm1l/voBKBBEpIRUr14dn8/HFVdcwZ133snIkSO9Lum07d+/n0GDBlG5cmUFgohIcVSuXJkZM2bQv39/hg4dysMPP0xgirPok5ubyzXXXMOKFSt47bXXOPPMM70uqdSFNJeRiEioKlasyKRJk6hWrRpPPPEE+/bt4x//+EdU3ZIzLy+PAQMGMHfuXMaPH0+fPn28LiksFAgiUuJiY2N5/vnnqVmzJk899RR79+5l3LhxVK5c2evSiuSc48EHH2TKlCk8+eST3HbbbV6XFDbRE9kiElXy79M8fPhwJk2aRMeOHVmzZo3XZRVp5MiRjB49mnvvvZchQ4Z4XU5YKRBEpNSYGY888gizZ89mx44dpKamMmbMmIi9rjBhwgSGDRvGDTfcwDPPPINZYXf7LbsUCCJS6nr16sWqVatIS0vjzjvvpG/fvuzevdvrso7xzjvvcMcdd9CzZ08mTpwYVdc8Skr522MR8UTdunXJzMzkmWeewefz0bZtWxYsWOB1WQB89NFHXH/99Zx//vm88cYbxMXFeV2SJxQIIhI2MTEx3H///Xz66adUr16d7t27M2zYMA4dOuRZTatXr+aKK64gKSmJ2bNnU61aNc9q8ZoCQUTCrl27dixfvpyBAwcyYsQIfvvb37Jx48aw17F582YuvfRSqlatyty5c0lMTAx7DZFEgSAinoiPj2fcuHG8/vrrbNiwgXbt2vHKK6+E7YLzrl276NGjBwcOHOC9996jYcOGYdluJFMgiIin+vXrx6pVq2jfvj233HILv//979m7d2+pbnP//v307t2bLVu2MGvWLFq1alWq24sWCgQR8VyDBg1YsGABw4cPZ/r06Zx33nl88sknpbKtglNSTJ8+nS5dupTKdqKRAkFEIkJsbCyPPPIIixYtAqBr164MHz6cI0eOlNg28vLyuPXWW5k7dy7jxo3jiiuuKLG2ywIFgohElAsvvJDPP/+c6667jscee4xu3bqxZcuW027XOccDDzzA1KlTy92UFKHSXEYiEnFq1qzJ5MmTufTSS7nrrrs455xzSElJISUlhcaNGx/zPSUlhapVqxbZ5ogRI/jHP/7BfffdV+6mpAiVRepHyAuTmprqli1b5nUZIhJGmzZt4oUXXmDjxo18/fXXbNq0iX379h2zTt26dX8VFPnfzzzzTCZMmMDAgQP5/e9/zyuvvFLuPoVsZsudc6lFrqdAEJFo4pxjz549bNq06WhAFPy+devWY4auVq1alV9++YVLLrmEd955p1x+CjnUQFCXkYhEFTOjVq1a1KpVi44dO/7q+V9++YXNmzcfExQxMTE8/vjj5TIMikOBICJlSuXKlWnWrBnNmjXzupSoE1JHmpnda2ZZZrbGzO4rsPxuM1sfXD4quOz3ZvZ5ga88MzuvkDbPM7MlwXWWmdmvo15ERMKmyDMEM2sF3AF0BHKBOWY2G6gPXAm0cc7lmFkdAOfcZGBy8LWtgZnOuc8LaXoU8LhzLtPMegX/fdHp75KIiJyKULqMWgBLnHMHAMxsIdAXSAVGOOdyAJxzOwt5bX9g6gnadUCN4OOawLZi1C0iIiUslC6jLCDNzGqZWVWgF9AAaAp0NbNPzWyhmXUo5LXXc+JAuA/4XzP7BngKGFb88kVEpKQUGQjOubXASGAeMAf4AjhM4OwiAegE/AmYbgXuN2dmFwAHnHNZJ2j6j8D9zrkGwP3A+MJWMrNBwWsMy3bt2hXyjomISPGEdFHZOTfeOdfeOZcG7AE2AFuBN13AUiAPqF3gZb/jxGcHALcAbwYfv07gGkVh2x7nnEt1zqWW97nKRURKU6ijjOoEvycBVxP4Rf82kBFc3hSIA3YH/x0DXAtMO0mz24D04OMMAiEjIiIeCfVzCDPMrBZwCLjLOfeDmU0AJphZFoHRR7e4///xwDRgq3Pu64KNmNmLwFjn3DICI5f+YWYVgF+AQSWwPyIicoqiauoKM9sFbD7Fl9cmeAYTYVRX8aiu4lFdxVNW62ronCuyzz2qAuF0mNmyUObyCDfVVTyqq3hUV/GU97rK15R/IiJyQgoEEREBylcgjPO6gBNQXcWjuopHdRVPua6r3FxDEBGRkytPZwgiInISZSoQzOx/zWydma0ys7fM7DeFrNPsuOm59+VP6W1m/2Nm3xZ4rle46gqul21mq/OnBC+w/Awzm2dmG4LfE8JVl5k1MLP3zWxtcJrzews85/X7dWlw+vWNZja0wPJGwTm2NpjZa2ZWIndFMbNrg+9BnpkVOuLDo+OryLqC64X7+Arl/fLi+Ar1/Qr38VXkz8HMuh13fP1iZlcFn3vJzPwFnvvVbQeK5JwrM19AD6BC8PFIYGQR68cCOwiM0QX4H+Ahr+oCsoHahSwfBQwNPh5a1H6VZF1APaB98HF14CvgXK/fr+DPbhOQQuBT8l8UqGs68Lvg47HAH0uorhZAM+ADIDWE9cN1fIVUlwfHV5F1eXR8hVKXF8dXsX4OwBkEphKqGvz3S0C/06mhTJ0hOOfmOucOB/+5hMA9G07mYmCTc+5UP+xWWnUd70rg5eDjl4GrwlWXc267c25F8PFPwFrg7JLY/unURWDuq43Oua+dc7kEpkm50syMwFQobwTXK8n3a61zbn0xXhKu46u4dR2vtI6vIuvy6PgK5f0K+/FF8X8O/YBMF7w1QUkoU4FwnNuAzCLWKWwCvv8KdlVMKKlT52LU5YC5ZrbczApO5XGmc247BP4DAXXCXBcAZpYMtAM+LbDYq/frbOCbAv/eGlxWC/ixQKDkL/eCF8fXyXh5fBXJo+PrRLw4vor7cyjs+Pp78P161swqFbeAqAsEM5tvgdt5Hv91ZYF1HiYwRffkk7QTB/QhMNNqvjFAY+A8YDvwdJjr+q1zrj1wGXCXmaWFuv1SrgszqwbMAO5zzu0LLvby/bJClrmTLC+xukJsJ+zHVwg8Ob5CbCfsx1dRTRSyLJKOr3pAa+C9AouHAc2BDgS6k4YUp00IfXK7iOGc636y583sFuBy4GIX7Fg7gcuAFc657wq0ffSxmb0AvBvOupxz24Lfd5rZWwROWz8EvjOzes657cEDobC705VaXWZWkcB/1snOufwpy71+v7YSuFFTvvoEZtDdDfzGzCoE/4rLX14idRVDWI+vENsI+/EVCi+OrxCE/fgys+L8HK4D3nLOHSrQ9vbgwxwzmwg8FGpd+aLuDOFkzOxSAqnYJ4R+tV/d3jP4Q8jXl8Dd4sJSl5nFm1n1/McELqzmb/8dAvePIPh9ZhjrMgI3L1rrnHvmuOc8e7+Az4AmFhjxEUfg9PmdYHi8T6B/FUrw/SqmsB1fofDi+AqxrrAfXyHy4vgqzs/hhMdX8D29ilN5v0ri6nikfAEbCfT7fR78GhtcfhbgK7BeVeB7oOZxr58ErAZWBX849cJVF4HRDF8Ev9YADxd4fS3gPwTuGfEf4Iww1tWFwCnxqgLr9fL6/Qr+uxeBUSmbjnu/UoClwXZeByqVUF19CfzlmAN8B7wXIcdXkXV5dHyFUpcXx1eoP8dwH1+F/hwI3L/+xQLrJQPfAjHHvX5B8P3KAl4FqhW3Bn1SWUREgDLWZSQiIqdOgSAiIoACQUREghQIIiICKBBERCRIgSAiIoACQUREghQIIiICwP8D35G6s7+HGE8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "upper=len(lr_list)\n",
    "plt.plot(np.array(lr_list)[:upper,0],np.array(lr_list)[:upper,1], color='black')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "path='modelv2_01.pkl'\n",
    "with open(path,'rb') as f:\n",
    "    (parameters,\n",
    "        current_epoch,\n",
    "        train_loss_list,\n",
    "        valid_loss_list,\n",
    "        train_accu_list,\n",
    "        valid_accu_list\n",
    "    )=pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}