CMUHW1P2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CMUHW1P2.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "w4q4BhIZA-tS",
        "Ihvk2-4kCQQO",
        "iUxuR9Kz02QE",
        "8GCB-ci-3eNc"
      ],
      "toc_visible": true,
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Translucent504/Playground/blob/master/CMUHW1P2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HvO3IRiIw_Vb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "%cd /content/drive/My Drive/Colab Notebooks/CMUDL F19/Recitations/Recitation 4/GoodCodeStructureExample/code"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w4q4BhIZA-tS",
        "colab_type": "text"
      },
      "source": [
        "# **Scaffolding**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M2Cfr5LNBtRy",
        "colab_type": "text"
      },
      "source": [
        "## Config"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iMKab0VLxwvY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# config.py\n",
        "\n",
        "# General information\n",
        "cuda = True\n",
        "sanity = True\n",
        "sanitysize = 1100\n",
        "sanityepoch = 5\n",
        "load_model = False\n",
        "# Data information\n",
        "CONTEXT = 2\n",
        "input_size = 40 * (2*CONTEXT + 1)\n",
        "nclasses = 138\n",
        "# Training parameters\n",
        "lr = 0.0001\n",
        "weight_decay = 0.0001\n",
        "batch_size = 256\n",
        "max_epoch = 5\n",
        "patience = 10\n",
        "# Network parameters\n",
        "hidden = [330,330,330,330,330,330,330,330,330,330,330,330,330,330,330,330,330,330,330]\n",
        "dropout = 0.02\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TRXHZl4eBzpy",
        "colab_type": "text"
      },
      "source": [
        "## Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yj6uM4wsAjAf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# data.py\n",
        "\n",
        "import paths\n",
        "import config\n",
        "\n",
        "import numpy as np\n",
        "import torch\n",
        "from torch.utils.data import TensorDataset, DataLoader, Dataset\n",
        "\n",
        "\n",
        "\n",
        "class MyDataset(Dataset):\n",
        "    def __init__(self, X, Y=None):\n",
        "        self.Y = Y \n",
        "        if self.Y is not None:\n",
        "          tmp_Y = np.concatenate(tuple(Y[:]), axis=0)\n",
        "          self.Y = torch.from_numpy(tmp_Y).long()\n",
        "        self.context = config.CONTEXT\n",
        "        tmp_X = np.concatenate(tuple(X[:]), axis=0)\n",
        "        tmp_X = np.pad(tmp_X,((self.context, self.context),(0,0)), mode='reflect')\n",
        "        self.X = torch.from_numpy(tmp_X).float()\n",
        "        \n",
        "    def __len__(self):\n",
        "      if self.Y is not None:\n",
        "        return len(self.Y)\n",
        "      else:\n",
        "        return self.X.shape[0] - 2*self.context\n",
        "\n",
        "    def __getitem__(self,index):\n",
        "        X = self.X[index:index + 2*self.context+1].reshape(-1)  #flatten the input\n",
        "        if self.Y is not None:\n",
        "          Y = self.Y[index]\n",
        "          return X, Y\n",
        "        else:\n",
        "          return X\n",
        "\n",
        "\n",
        "def get_loader(mode=\"train\"):\n",
        "    if mode == \"train\":\n",
        "        data_path = paths.train_data\n",
        "        labels_path = paths.train_labels\n",
        "        shuffle = True\n",
        "    if mode == \"val\":\n",
        "        data_path = paths.valid_data\n",
        "        labels_path = paths.valid_labels\n",
        "        shuffle = False\n",
        "    if mode == \"test\":\n",
        "        data_path = paths.test_data\n",
        "        labels_path = None\n",
        "        shuffle = False\n",
        "    data = np.load(data_path, allow_pickle=True)\n",
        "    if config.sanity:\n",
        "        data = data[:config.sanitysize]\n",
        "\n",
        "    if labels_path:\n",
        "        labels = np.load(labels_path, allow_pickle=True)\n",
        "        if config.sanity:\n",
        "            labels = labels[:config.sanitysize]\n",
        "\n",
        "        print(data.shape, labels.shape)\n",
        "        # dataset = TensorDataset(torch.tensor(data, dtype=torch.float),\n",
        "        #                         torch.tensor(labels, dtype=torch.long))\n",
        "        dataset = MyDataset(data, labels)                                \n",
        "    else:\n",
        "        dataset = MyDataset(data)\n",
        "        # dataset = TensorDataset(torch.tensor(data, dtype=torch.float))\n",
        "    dataloader = DataLoader(dataset, shuffle=shuffle, batch_size=config.batch_size, drop_last=False)\n",
        "\n",
        "    return dataloader\n",
        "\n",
        "\n",
        "\n",
        "def get_test_labels():\n",
        "    labels = np.load(paths.test_labels)\n",
        "    if config.sanity:\n",
        "            labels = labels[:config.sanitysize]\n",
        "    return labels\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d9FpwGcGB49r",
        "colab_type": "text"
      },
      "source": [
        "## Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7UrqVsC2CS1R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# ffnnmodel.py\n",
        "\n",
        "import config\n",
        "import paths\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "\n",
        "\n",
        "class FFNN(nn.Sequential):\n",
        "\n",
        "    def __init__(self):\n",
        "        l = []\n",
        "        sizes = [config.input_size] + config.hidden + [config.nclasses]\n",
        "        for i in range(len(sizes)-2):\n",
        "            l.append(nn.Linear(sizes[i], sizes[i+1]))\n",
        "            if config.dropout > 0:\n",
        "                l.append(nn.Dropout(config.dropout))\n",
        "            l.append(nn.LeakyReLU())\n",
        "        l.append(nn.Linear(sizes[-2], sizes[-1]))\n",
        "        super(FFNN, self).__init__(*l)\n",
        "\n",
        "    @staticmethod\n",
        "    def load():\n",
        "        model = FFNN()\n",
        "        path = paths.model\n",
        "        model.load_state_dict(torch.load(path))\n",
        "\n",
        "        return model\n",
        "\n",
        "    def save(self):\n",
        "        path = paths.model\n",
        "        torch.save(self.state_dict(), path)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3V0bUEBmCYhr",
        "colab_type": "text"
      },
      "source": [
        "## paths.py"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JWrIx4bpCb4d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data_prefix = \"../float16/\"\n",
        "train_data = data_prefix+\"train.npy\"\n",
        "train_labels = data_prefix+\"train_labels.npy\"\n",
        "valid_data = data_prefix+\"dev.npy\"\n",
        "valid_labels = data_prefix+\"dev_labels.npy\"\n",
        "test_data = data_prefix+\"test.npy\"\n",
        "test_labels = data_prefix+\"test_labels.npy\"\n",
        "\n",
        "model = \"../model/model_dict.pt\"\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Rx1vBNMRCeM0",
        "colab_type": "text"
      },
      "source": [
        "## Routine"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Py1F61xZCimb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# coding=utf-8\n",
        "\n",
        "import config\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import numpy as np\n",
        "import time \n",
        "\n",
        "device = \"cuda\" if config.cuda else \"cpu\"\n",
        "\n",
        "\n",
        "def run_one_epoch(model, loader, criterion, update=True, optimizer=None):\n",
        "    cum_loss = 0\n",
        "    for x, y in loader:\n",
        "        x = x.to(device)\n",
        "        y = y.to(device)\n",
        "        output = model(x)\n",
        "        loss = criterion(output, y)\n",
        "        if update:\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "            optimizer.zero_grad()\n",
        "        cum_loss += loss.cpu().detach().numpy().item()\n",
        "    return cum_loss/len(loader)\n",
        "\n",
        "\n",
        "def train(model, train_dataloader, val_dataloader):\n",
        "    max_epoch = config.max_epoch\n",
        "    if config.sanity:\n",
        "        max_epoch = config.sanityepoch\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "    optimizer = torch.optim.AdamW(model.parameters(), lr=config.lr)\n",
        "    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')\n",
        "    model = model.to(device)\n",
        "    best_loss = float(\"+inf\")\n",
        "\n",
        "    print('begin training')\n",
        "    lr = config.lr\n",
        "    epoch = 0\n",
        "    pat = 0\n",
        "    while True:\n",
        "        start = time.time()\n",
        "        epoch += 1\n",
        "        print(\"Epoch\", epoch)\n",
        "        model.train()\n",
        "        train_loss = run_one_epoch(model, train_dataloader, criterion, True, optimizer)\n",
        "        print(\"Training loss :\", train_loss)\n",
        "        model.eval()\n",
        "        val_loss = run_one_epoch(model, val_dataloader, criterion, False)\n",
        "        print(\"Validation loss :\", val_loss)\n",
        "\n",
        "        scheduler.step(val_loss)\n",
        "        if val_loss < best_loss:\n",
        "            best_loss = val_loss\n",
        "            model.save()\n",
        "            pat = 0\n",
        "        else:\n",
        "            pat += 1\n",
        "            if pat >= config.patience:\n",
        "                print(\"Early stopping ! IGNORED\")\n",
        "                #break\n",
        "        if epoch >= max_epoch:\n",
        "            print('reached maximum number of epochs !')\n",
        "            break\n",
        "        stop = time.time()\n",
        "        print(f\"Time :{stop-start}s\")\n",
        "\n",
        "\n",
        "def predict(model, loader):\n",
        "    l = []\n",
        "    for x in loader:\n",
        "        x = x.to(device)\n",
        "        output = model(x)\n",
        "        classes = np.argmax(output.cpu().detach().numpy(), axis=1)\n",
        "        l.append(classes)\n",
        "    return np.concatenate(l)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2c1JsWkKClDb",
        "colab_type": "text"
      },
      "source": [
        "## Run"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Cow7FMzCoqS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import data\n",
        "import ffnnmodel\n",
        "import routine\n",
        "import config\n",
        "import numpy as np\n",
        "\n",
        "# To run, make sure you have a data/ folder containing MNIST and a model/ folder along with your code/ folder\n",
        "\n",
        "def run():\n",
        "    train_loader = data.get_loader(\"train\")\n",
        "    val_loader = data.get_loader(\"val\")\n",
        "    test_loader = data.get_loader(\"test\")\n",
        "    #test_labels = data.get_test_labels()\n",
        "\n",
        "    if config.load_model:\n",
        "        model = ffnnmodel.FFNN.load()\n",
        "    else:\n",
        "        model = ffnnmodel.FFNN()\n",
        "\n",
        "    routine.train(model, train_loader, val_loader)\n",
        "    test_outputs = routine.predict(model, test_loader)\n",
        "    print((test_outputs))\n",
        "    try:\n",
        "    \tnp.savetxt('prediction.csv', test_outputs, delimiter=\",\")\n",
        "    except:\n",
        "        print(\"prediction not saved\")\n",
        "    #accuracy = (test_outputs == test_labels).mean()\n",
        "    #print(\"Test accuracy :\", accuracy)\n",
        "\n",
        "\n",
        "run()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Ihvk2-4kCQQO"
      },
      "source": [
        "# **Playing**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_NUR4tDFCQQF",
        "colab": {}
      },
      "source": [
        "import paths\n",
        "import numpy as np\n",
        "import data\n",
        "import ffnnmodel\n",
        "import torch\n",
        "\n",
        "test = data.get_loader(\"test\")\n",
        "model = ffnnmodel.FFNN.load()\n",
        "\n",
        "\n",
        "def get_predictions(model, loader):\n",
        "    final_result = []\n",
        "    model.eval()\n",
        "    model.to(torch.device(\"cuda\"))\n",
        "    for x in loader:\n",
        "        x = x.to(torch.device(\"cuda\"))\n",
        "        output = model(x)\n",
        "        result = np.argmax(output.cpu().detach().numpy(), axis=1)\n",
        "        final_result.append(result)\n",
        "    return np.concatenate(final_result)\n",
        "\n",
        "results = get_predictions(model, test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "6KMuJdYiCQPw",
        "colab": {}
      },
      "source": [
        "tmp = list(zip(range(len(results)), results))\n",
        "import csv\n",
        "with open('150x4val206.csv', 'w') as f:\n",
        "    writer = csv.writer(f)\n",
        "    writer.writerow(['id','label'])\n",
        "    writer.writerows(tmp)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "9ufjxdShCQPW",
        "colab": {}
      },
      "source": [
        "import data\n",
        "import ffnnmodel\n",
        "import routine\n",
        "import config\n",
        "import numpy as np\n",
        "\n",
        "# To run, make sure you have a data/ folder containing MNIST and a model/ folder along with your code/ folder\n",
        "\n",
        "def run():\n",
        "    train_loader = data.get_loader(\"train\")\n",
        "    val_loader = data.get_loader(\"val\")\n",
        "    test_loader = data.get_loader(\"test\")\n",
        "    #test_labels = data.get_test_labels()\n",
        "\n",
        "    if config.load_model:\n",
        "        model = ffnnmodel.FFNN.load()\n",
        "    else:\n",
        "        model = ffnnmodel.FFNN()\n",
        "\n",
        "    routine.train(model, train_loader, val_loader)\n",
        "    test_outputs = routine.predict(model, test_loader)\n",
        "    print((test_outputs))\n",
        "    try:\n",
        "    \tnp.savetxt('prediction.csv', test_outputs, delimiter=\",\")\n",
        "    except:\n",
        "        print(\"prediction not saved\")\n",
        "    #accuracy = (test_outputs == test_labels).mean()\n",
        "    #print(\"Test accuracy :\", accuracy)\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    run()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b9-81RfCBP5d",
        "colab_type": "text"
      },
      "source": [
        "# **From Scratch**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NIpWf_YD_VEo",
        "colab_type": "text"
      },
      "source": [
        "## **Setup Google Drive**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N5RKs-Ym_X1k",
        "colab_type": "code",
        "outputId": "dbe3e95c-e998-4e93-d4a3-53c56fe4bf4e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "%cd /content/drive/My Drive/Colab Notebooks/CMUDL F19/Recitations/Recitation 4/GoodCodeStructureExample/code"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
            "/content/drive/My Drive/Colab Notebooks/CMUDL F19/Recitations/Recitation 4/GoodCodeStructureExample/code\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6ALyPGif6c_s",
        "colab_type": "text"
      },
      "source": [
        "## **Imports**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rRh-Aq2m6gmQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%matplotlib inline\n",
        "import time\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import numpy as np\n",
        "import csv\n",
        "import math\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from tqdm import tqdm\n",
        "from pathlib import Path\n",
        "from torch.utils.data import TensorDataset, DataLoader, Dataset"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EBMb_Kfm5C3b",
        "colab_type": "text"
      },
      "source": [
        "## **Data Loading Cells**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gU62jx37GJtC",
        "colab_type": "text"
      },
      "source": [
        "### WSJ Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wLWUu-uuGQpY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class WSJDataset(Dataset):\n",
        "    \"\"\"\n",
        "    Expects data in the form of an array of recordings\n",
        "    each recording has N utterances or elements\n",
        "    each element is a 10ms frame represented by a 40 sized frequency/energy vector.\n",
        "    each element can then be interpretted as a phoneme.\n",
        "    \"\"\"\n",
        "    def __init__(self, X, Y=None):\n",
        "        self.Y = Y \n",
        "        if self.Y is not None:\n",
        "          tmp_Y = np.concatenate(tuple(Y[:]), axis=0)\n",
        "          self.Y = torch.from_numpy(tmp_Y).long()\n",
        "        self.context = CONTEXT\n",
        "        tmp_X = np.concatenate(tuple(X[:]), axis=0)\n",
        "        tmp_X = np.pad(tmp_X,((self.context, self.context),(0,0)), mode='reflect')\n",
        "        self.X = torch.from_numpy(tmp_X).float()\n",
        "        \n",
        "    def __len__(self):\n",
        "      if self.Y is not None:\n",
        "        return len(self.Y)\n",
        "      else:\n",
        "        return self.X.shape[0] - 2*self.context\n",
        "\n",
        "    def __getitem__(self,index):\n",
        "        X = self.X[index:index + 2*self.context+1].reshape(-1)  #flatten the input\n",
        "        if self.Y is not None:\n",
        "          Y = self.Y[index]\n",
        "          return X, Y\n",
        "        else:\n",
        "          return X"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y7teM1ky6BPs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_loader(mode=\"train\"):\n",
        "    if mode == \"train\":\n",
        "        data_path = TRAIN_DPATH\n",
        "        labels_path = TRAIN_LPATH\n",
        "        shuffle = True\n",
        "    if mode == \"val\":\n",
        "        data_path = VALID_DPATH\n",
        "        labels_path = VALID_LPATH\n",
        "        shuffle = False\n",
        "    if mode == \"test\":\n",
        "        data_path = TEST_DPATH\n",
        "        labels_path = None\n",
        "        shuffle = False\n",
        "    data = np.load(data_path, allow_pickle=True)\n",
        "\n",
        "    if SANITY:\n",
        "        data = data[:SANITYSIZE]\n",
        "\n",
        "    if labels_path:\n",
        "        labels = np.load(labels_path, allow_pickle=True)\n",
        "        if SANITY:\n",
        "            labels = labels[:SANITYSIZE]\n",
        "        dataset = WSJDataset(data, labels)                                \n",
        "    else:\n",
        "        dataset = WSJDataset(data)\n",
        "    print(f\"{mode} Dataset Size: {len(dataset)}\")\n",
        "    dataloader = DataLoader(dataset, shuffle=shuffle, batch_size=BATCH_SIZE, drop_last=False)\n",
        "\n",
        "    return dataloader"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-1ApfswZuVWi",
        "colab_type": "text"
      },
      "source": [
        "## Routine :"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mIbPcstwun2w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def run_one_epoch(model, loader, criterion, update=True, optimizer=None, scheduler=None):\n",
        "    cum_loss = 0\n",
        "    plotloss = []\n",
        "    lrs = []\n",
        "    i = 0\n",
        "    corr_preds = 0\n",
        "    for x, y in tqdm(loader):\n",
        "        x = x.to(device)\n",
        "        y = y.to(device)\n",
        "        output = model(x)\n",
        "        loss = criterion(output, y)\n",
        "        if update:\n",
        "            lrplot = scheduler.get_lr()\n",
        "            lrs.append(lrplot)\n",
        "            if i % 1000 == 0:\n",
        "                print(f\"Learning rate: {lrplot}\")\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "            scheduler.step()\n",
        "            optimizer.zero_grad()\n",
        "        plotloss.append(loss.cpu().detach().numpy().item()) \n",
        "        # Per batch Accuracy data:\n",
        "        out = np.argmax(output.cpu().detach().numpy(), axis=1)\n",
        "        true = y.cpu().detach().numpy()\n",
        "        if WSJ and i % 1000 == 0:\n",
        "            print(f\"Current Batch Accuracy: {(np.sum(true==out)/len(true))*100}%\")\n",
        "        corr_preds += np.sum(true==out)\n",
        "        i+=1   \n",
        "        cum_loss += loss.cpu().detach().numpy().item()\n",
        "    # Overall per Epoch Accuracy\n",
        "    if update:\n",
        "        print(f\"Training Epoch Accuracy: {100*corr_preds/len(loader.dataset)}%\")\n",
        "        plt.plot(lrs, plotloss)\n",
        "        plt.xscale(\"log\")\n",
        "        plt.show()\n",
        "    else:\n",
        "        print(f\"Validation Epoch Accuracy: {100*corr_preds/len(loader.dataset)}%\")\n",
        "        plt.plot(plotloss)\n",
        "        plt.show()\n",
        "    return cum_loss/len(loader)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2FC6Fq8Vzkc-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# def lr_find(model, loader, optimizer, min_lr=1e-7, max_lr=1.0):\n",
        "#     cum_loss = 0\n",
        "#     plotloss = []\n",
        "#     lrs = []\n",
        "#     i = 0\n",
        "#     n = math.log(number, 1.01)\n",
        "#     corr_preds = 0\n",
        "#     for x, y in tqdm(loader):\n",
        "#         x = x.to(device)\n",
        "#         y = y.to(device)\n",
        "#         output = model(x)\n",
        "#         loss = criterion(output, y)\n",
        "#         lrs.append(lrplot)\n",
        "#         loss.backward()\n",
        "#         optimizer.step()\n",
        "#         optimizer.zero_grad()\n",
        "#         lr = optimizer.param_groups[0]['lr']\n",
        "#         optimizer.param_groups[0]['lr'] = lr * 1.1\n",
        "#         plotloss.append(loss.cpu().detach().numpy().item()) \n",
        "#         i+=1   \n",
        "#         cum_loss += loss.cpu().detach().numpy().item()\n",
        "\n",
        "#     print(f\"Training Epoch Accuracy: {100*corr_preds/len(loader.dataset)}%\")\n",
        "#     plt.plot(lrs, plotloss)\n",
        "#     plt.xscale(\"log\")\n",
        "#     plt.show()\n",
        "#     return cum_loss/len(loader)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kUU_SFfTuZXw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train(model, train_dataloader, val_dataloader):\n",
        "    max_epoch = MAX_EPOCHS\n",
        "    lr = LEARNING_RATE\n",
        "    if SANITY:\n",
        "        max_epoch = SANITYEPOCH\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE, weight_decay=0.0001)\n",
        "    scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer,max_lr=SCH_RATE,steps_per_epoch=len(train_dataloader), epochs=max_epoch)\n",
        "    model = model.to(device)\n",
        "    val_losses = []\n",
        "    train_losses = []\n",
        "    best_loss = 10000\n",
        "    print('Begin training')\n",
        "    for epoch in range(max_epoch):\n",
        "        start = time.time()\n",
        "        print(\"Epoch: \", epoch+1)\n",
        "        #print(\"Learning Rate: \", lr)\n",
        "        model.train()\n",
        "        train_loss = run_one_epoch(model, train_dataloader, criterion, True, optimizer, scheduler)\n",
        "        train_losses.append(train_loss)\n",
        "        print(\"Training loss: \", train_loss)\n",
        "        model.eval()\n",
        "        val_loss = run_one_epoch(model, val_dataloader, criterion, False)\n",
        "        val_losses.append(val_loss)\n",
        "        print(\"Validation loss: \", val_loss)\n",
        "        stop = time.time()\n",
        "        if val_loss < best_loss:\n",
        "            best_loss = val_loss\n",
        "            print(\"Saving Model....\")\n",
        "            model.save()\n",
        "        \n",
        "        print(f\"Time :{stop-start}s\")\n",
        "    return val_losses, train_losses\n",
        "    \n",
        "\n",
        "\n",
        "def get_predictions(model, loader):\n",
        "    final_result = []\n",
        "    model.eval()\n",
        "    model.to(torch.device(\"cuda\"))\n",
        "    for x in loader:\n",
        "        x = x.to(torch.device(\"cuda\"))\n",
        "        output = model(x)\n",
        "        result = np.argmax(output.cpu().detach().numpy(), axis=1)\n",
        "        final_result.append(result)\n",
        "    return np.concatenate(final_result)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NSuoxXaz6pXW",
        "colab_type": "text"
      },
      "source": [
        "## Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ewusyja36qYS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class FFNN(nn.Sequential):\n",
        "\n",
        "    def __init__(self):\n",
        "        l = []\n",
        "        sizes = [INPUT_SIZE] + HIDDEN_SIZE + [NUM_CLASSES]\n",
        "        for i in range(len(sizes)-2):\n",
        "            l.append(nn.Linear(sizes[i], sizes[i+1]))\n",
        "            l.append(nn.BatchNorm1d(sizes[i+1]))\n",
        "            l.append(nn.ReLU())\n",
        "        l.append(nn.Linear(sizes[-2], sizes[-1]))\n",
        "        super(FFNN, self).__init__(*l)\n",
        "\n",
        "    @staticmethod\n",
        "    def load():\n",
        "        model = FFNN()\n",
        "        path = MODEL_PATH\n",
        "        model.load_state_dict(torch.load(path))\n",
        "\n",
        "        return model\n",
        "\n",
        "    def save(self):\n",
        "        path = MODEL_PATH\n",
        "        torch.save(self.state_dict(), path)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0otijjTb6HVV",
        "colab_type": "text"
      },
      "source": [
        "## **Config Cell**\n",
        "\n",
        "(𝐿𝑅 ∗ 𝑊𝐷)/(𝑇𝐵𝑆 ∗ (1 − α)) ≈ 𝑐𝑜𝑛𝑠𝑡𝑎𝑛𝑡\n",
        "\n",
        "α - momentum"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8hHH-jMi6ItZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Data Parameters\n",
        "data_folder = Path(\"../float16/\")\n",
        "mnist_data_folder = Path(\"../data/\")\n",
        "WSJ = True\n",
        "if WSJ:\n",
        "  TRAIN_DPATH = data_folder/'train.npy' \n",
        "  TRAIN_LPATH = data_folder/'train_labels.npy'\n",
        "  VALID_DPATH = data_folder/'dev.npy'\n",
        "  VALID_LPATH = data_folder/'dev_labels.npy'\n",
        "  TEST_DPATH = data_folder/'test.npy'\n",
        "  MODEL_PATH = Path(\"../model/tqdm.pt\")\n",
        "  # Model Parameters\n",
        "  torch.manual_seed(0)\n",
        "  np.random.seed(0)\n",
        "  LOAD_MODEL = False\n",
        "  MAX_EPOCHS = 1\n",
        "  BATCH_SIZE = 1024\n",
        "  LEARNING_RATE = 1e-5\n",
        "  SCH_RATE = 5e-3\n",
        "  MOMENTUM = 0.9\n",
        "  CONTEXT = 15\n",
        "  INPUT_SIZE = 40 * (2*CONTEXT + 1)\n",
        "  HIDDEN_SIZE = [1240]*6\n",
        "  NUM_CLASSES = 138\n",
        "else:\n",
        "  TRAIN_DPATH = mnist_data_folder/'train.npy' \n",
        "  TRAIN_LPATH = mnist_data_folder/'train_labels.npy'\n",
        "  VALID_DPATH = mnist_data_folder/'dev.npy'\n",
        "  VALID_LPATH = mnist_data_folder/'dev_labels.npy'\n",
        "  TEST_DPATH = mnist_data_folder/'test.npy'\n",
        "  MODEL_PATH = Path(\"../model/MNISTMODEL.pt\")\n",
        "  # Model Parameters\n",
        "  torch.manual_seed(0)\n",
        "  np.random.seed(0)\n",
        "  LOAD_MODEL = False\n",
        "  MAX_EPOCHS = 10\n",
        "  BATCH_SIZE = 64\n",
        "  LEARNING_RATE = 1e-6\n",
        "  SCH_RATE = 1e-5\n",
        "  MOMENTUM = 0.9\n",
        "  INPUT_SIZE = 784\n",
        "  HIDDEN_SIZE = [512,512,512,512,512,512,512,512]\n",
        "  NUM_CLASSES = 10\n",
        "\n",
        "# Sanity / Training\n",
        "SANITY = False\n",
        "SANITYSIZE = 1100\n",
        "SANITYEPOCH = 2\n",
        "cuda = True\n",
        "device = \"cuda\" if cuda else \"cpu\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NNdP2TTc6YMu",
        "colab_type": "text"
      },
      "source": [
        "## Runtime"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KSV0_mTw6ZiI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def run():\n",
        "    train_loader = get_loader(\"train\")\n",
        "    val_loader = get_loader(\"val\")\n",
        "\n",
        "    if LOAD_MODEL:\n",
        "        model = FFNN.load()\n",
        "    else:\n",
        "        model = FFNN()\n",
        "\n",
        "    val_losses, train_losses = train(model, train_loader, val_loader)\n",
        "    plt.plot(val_losses)\n",
        "    plt.plot(train_losses)\n",
        "    plt.title(\"Validation Loss\");\n",
        "    plt.xlabel(\"Iterations\");\n",
        "    plt.ylabel(\"Loss\");\n",
        "    return (val_losses, train_losses)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Hcrp6RL_IkP",
        "colab_type": "code",
        "outputId": "74f40563-8c94-4503-a8a3-e57391ca47e5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "v,t=run();"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "train Dataset Size: 15388713\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/15029 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "val Dataset Size: 675836\n",
            "Begin training\n",
            "Epoch:  1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 1/15029 [00:01<7:45:47,  1.86s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Learning rate: [0.00019999999999999966]\n",
            "Current Batch Accuracy: 0.48828125%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "  7%|▋         | 1003/15029 [01:04<17:51, 13.09it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Learning rate: [0.0007596549238958116]\n",
            "Current Batch Accuracy: 32.8125%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 13%|█▎        | 2003/15029 [02:07<13:18, 16.31it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Learning rate: [0.002177608334049224]\n",
            "Current Batch Accuracy: 36.328125%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 20%|█▉        | 3003/15029 [03:10<11:40, 17.17it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Learning rate: [0.0037925563905875587]\n",
            "Current Batch Accuracy: 40.72265625%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 27%|██▋       | 4001/15029 [04:12<10:15, 17.91it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Learning rate: [0.004851321067112773]\n",
            "Current Batch Accuracy: 35.44921875%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 33%|███▎      | 5003/15029 [05:14<09:53, 16.89it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Learning rate: [0.0049730332180343674]\n",
            "Current Batch Accuracy: 41.796875%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 40%|███▉      | 6003/15029 [06:14<08:19, 18.06it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Learning rate: [0.004755845387529389]\n",
            "Current Batch Accuracy: 42.08984375%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 47%|████▋     | 7003/15029 [07:14<07:26, 17.99it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Learning rate: [0.004338983727986821]\n",
            "Current Batch Accuracy: 42.67578125%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 53%|█████▎    | 8003/15029 [08:14<07:28, 15.68it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Learning rate: [0.003759346483046238]\n",
            "Current Batch Accuracy: 41.50390625%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 60%|█████▉    | 9003/15029 [09:14<05:39, 17.74it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Learning rate: [0.0030682398733609304]\n",
            "Current Batch Accuracy: 46.6796875%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 67%|██████▋   | 10003/15029 [10:14<04:35, 18.27it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Learning rate: [0.002326836759234407]\n",
            "Current Batch Accuracy: 47.8515625%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 73%|███████▎  | 11003/15029 [11:14<04:33, 14.71it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Learning rate: [0.0016007619639478443]\n",
            "Current Batch Accuracy: 47.0703125%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 80%|███████▉  | 12003/15029 [12:14<02:49, 17.87it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Learning rate: [0.0009542835344270105]\n",
            "Current Batch Accuracy: 47.16796875%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 87%|████████▋ | 13003/15029 [13:14<01:50, 18.38it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Learning rate: [0.00044462409610203905]\n",
            "Current Batch Accuracy: 51.5625%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 93%|█████████▎| 14001/15029 [14:13<00:55, 18.58it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Learning rate: [0.0001168958287525649]\n",
            "Current Batch Accuracy: 50.09765625%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|█████████▉| 15003/15029 [15:13<00:01, 17.04it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Learning rate: [1.0739071743812784e-07]\n",
            "Current Batch Accuracy: 51.171875%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 15029/15029 [15:15<00:00, 11.16it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Training Epoch Accuracy: 43.28398352740739%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hUZfbA8e9JIYUSSkINEDoISosU\nAUFUBCysde1iQ91Vf+6uu4uy6oKNdVd37YgFdXVVVsEFBBTpIC10CC0JJdQ0SAghCcm8vz9mMplJ\nZpLJMEkmw/k8Tx5m7rxz51wmOfe97z33vWKMQSmlVN0XVNsBKKWU8g1N6EopFSA0oSulVIDQhK6U\nUgFCE7pSSgUITehKKRUgQmrrg6Ojo01cXFxtfbxSStVJGzduzDDGxLh6rdYSelxcHAkJCbX18Uop\nVSeJyEF3r+mQi1JKBQhN6EopFSA0oSulVIDQhK6UUgHCo4QuIgdEZLuIbBGRcmcyxeotEUkSkW0i\n0s/3oSqllKpIVapcrjDGZLh5bQzQxfYzEHjf9q9SSqka4qshl3HA58ZqLdBYRFr5aN1KKT+zNiWT\nTYdO1nYYqgxPE7oBfhKRjSIywcXrbYBUh+eHbcuciMgEEUkQkYT09PSqR6uU8guvzN/F24v31XYY\nqgxPE/pQY0w/rEMrvxWRy735MGPMdGNMvDEmPibG5YVOSimlvORRQjfGHLH9mwbMBgaUaXIEaOvw\nPNa2TCmlVA2pNKGLSH0RaVjyGBgF7CjTbA5wr63aZRCQbYw55vNolVJ+Q29e6X88qXJpAcwWkZL2\n/zHGLBSRRwGMMdOA+cBYIAnIA+6vnnCVUv5AajsA5VKlCd0YkwL0drF8msNjA/zWt6EppZSqCr1S\nVCmlAoQmdKWUV4wOovsdTehKqaoTHUX3R5rQlVIqQGhCV0p5RUdc/I8mdKVUlemAi3/ShK6UUgFC\nE7pSSgUITehKKa8YrVv0O5rQlVJVplWL/kkTulJKBQhN6EopFSA0oSulVIDQhK6UqjIdQvdPmtCV\nUipAaEJXSnlFqxb9jyZ0pVSVidYt+iVN6EopFSA0oSulVIDQhK6U8orRCXT9jscJXUSCRWSziMxz\n8dp4EUkXkS22n4d8G6ZSyp/oCLp/CqlC2/8DdgGN3Lz+jTHm8fMPSSmllDc86qGLSCxwLfBR9Yaj\nlFLKW54OufwL+BNgqaDNzSKyTUS+FZG25x+aUsqfaR26/6k0oYvIdUCaMWZjBc3mAnHGmEuARcBn\nbtY1QUQSRCQhPT3dq4CVUrVPy9D9kyc99CHADSJyAPgaGCkiXzg2MMZkGmMKbE8/Avq7WpExZrox\nJt4YEx8TE3MeYSullCqr0oRujHnGGBNrjIkDbgeWGGPudmwjIq0cnt6A9eSpUiqA6ZCL/6lKlYsT\nEZkCJBhj5gBPisgNQBGQBYz3TXhKKX8kWrjol6qU0I0xy4BltsfPOyx/BnjGl4EppZSqGr1SVCml\nAoQmdKWUV/TSf/+jCV0pVXU6hO6XNKErpVSA0ISulFIBQhO6UsorWofufzShK6WqTIfQ/ZMmdKWU\nChCa0JVSKkBoQldKeUWH0P2PJnSlVJXp9Ln+SRO6UkoFCE3oSinv6JiL39GErpSqMp0+1z9pQldK\nqQChCV0ppQKEJnSllFd0+lz/owldKVVlWrbonzShK6VUgNCErpRSAUITulLKKzp9rv/xOKGLSLCI\nbBaReS5eCxORb0QkSUTWiUicL4NUSvkXHUP3T1Xpof8fsMvNaw8CJ40xnYF/An8738CUUkpVjUcJ\nXURigWuBj9w0GQd8Znv8LXCliO7DlQpkOuLifzztof8L+BNgcfN6GyAVwBhTBGQDzco2EpEJIpIg\nIgnp6elehKuU8gd66b9/qjShi8h1QJoxZuP5fpgxZroxJt4YEx8TE3O+q1NKKeXAkx76EOAGETkA\nfA2MFJEvyrQ5ArQFEJEQIArI9GGcSimlKlFpQjfGPGOMiTXGxAG3A0uMMXeXaTYHuM/2+BZbGx1i\nUyqA6Z+4/wnx9o0iMgVIMMbMAT4G/i0iSUAW1sSvlApQWvLgn6qU0I0xy4BltsfPOyzPB271ZWBK\nKaWqRq8UVUqpAKEJXSnlFR1B9z+a0JVSKkBoQldKqQChCV0p5RWtWvQ/mtCVUlWmUzX5J03oSikV\nIDShK6VUgNCErpTyig6h+x9N6EqpKtMRdP+kCV0ppQKEJnSllAoQmtCVUt7RQnS/owldKVVlWobu\nnzShK6VUgNCErpTyig64+B9N6EqpKtMRF/+kCV0ppQKEJnSllAoQmtCVUl7RqkX/U2lCF5FwEVkv\nIltFZKeITHbRZryIpIvIFtvPQ9UTrlLKH+j0uf4pxIM2BcBIY0yuiIQCq0RkgTFmbZl23xhjHvd9\niEoppTxRaUI3xhgg1/Y01PajB1tKKeVnPBpDF5FgEdkCpAGLjDHrXDS7WUS2ici3ItLWp1EqpfyO\n0X6d3/EooRtjio0xfYBYYICI9CrTZC4QZ4y5BFgEfOZqPSIyQUQSRCQhPT39fOJWStUiHUH3T1Wq\ncjHGnAKWAqPLLM80xhTYnn4E9Hfz/unGmHhjTHxMTIw38SqllHLDkyqXGBFpbHscAVwN7C7TppXD\n0xuAXb4MUimlVOU8qXJpBXwmIsFYdwAzjTHzRGQKkGCMmQM8KSI3AEVAFjC+ugJWSvkHrUP3P55U\nuWwD+rpY/rzD42eAZ3wbmlLKX2kZun/SK0WVUipAeDLkopRSTg5m5rEvLbfyhqpGaQ9dKVVlmsz9\nkyZ0pZQKEJrQlVJeyz9XXNshKAea0JVSXlu+V6/49iea0JVSXtMeun/RhK6UUgFCE7pSymt6tah/\n0YSulPLaybzC2g5BOdCErpTy2uS5ibUdgnKgCV0ppQKEJnSllAoQmtCVUipAaEJXSqkAoQldKaUC\nhCZ0pZQKEJrQlVIqQGhCV0qdlxU6QZff0ISulKqy3rFR9sf3frK+FiNRjjShK6Wq7B+39q7tEJQL\nlSZ0EQkXkfUislVEdorIZBdtwkTkGxFJEpF1IhJXHcEqpfxD84bhtR2CcsGTHnoBMNIY0xvoA4wW\nkUFl2jwInDTGdAb+CfzNt2EqpfxJVGRobYegXKg0oRurkjvChtp+yk6aOQ74zPb4W+BKERGfRamU\nUqpSHo2hi0iwiGwB0oBFxph1ZZq0AVIBjDFFQDbQzMV6JohIgogkpKfrmXGlAsWhzLzaDkHhYUI3\nxhQbY/oAscAAEenlzYcZY6YbY+KNMfExMTHerEIp5Ydu+2BNbYegqGKVizHmFLAUGF3mpSNAWwAR\nCQGigExfBKiU8n/Hc/JrOwSFZ1UuMSLS2PY4Arga2F2m2RzgPtvjW4AlxujNqZRSqiaFeNCmFfCZ\niARj3QHMNMbME5EpQIIxZg7wMfBvEUkCsoDbqy1ipZRfyi0ookGYJylFVZdK//eNMduAvi6WP+/w\nOB+41behKaXqkpd/SOTVmy6p7TAuaHqlqFLKK9v/Osrp+VfrU2spElVCE7pSyisNw8tfXFRs0VNn\ntUkTulLKZw6fzGP74Ww+WplS26FckPQMhlLKZwTh+ndWAXBL/1gaR9ar5YguLNpDV0r5zBfrDtof\n6+hLzdOErpTymekrSodadDKnmqcJXSnltR2Tr3H72pFTZ5m6YDfr92ex6dDJGozqwqVj6Eopr1V0\nIdF1b1vH0qctTwbgwNRraySmC5n20JVSNSI1K48v1x1k8KuLmbY8mePZ+fbl8S8t0hkbfUATulKq\nRuQWFDFp9g6OZeczdcFufvXuagC+23SYjNxCvt2oFyadL03oSqkaMebNlU7Pj+fk87eFuxHb6VOt\nijl/mtCVUudl4VPDvH7v+8uS7SdMLS4maM0+e45zxRaW7D7B1+sPef05Fwo9KaqUOi/dWzYi6eUx\ndJ60wKv3L99rvXuZqx5678k/cfVFLViUeAKA2we08+ozjmfn0zA8hPoBPhuk9tCVUuctJPj8U8my\nPWm8sWgvO49mOy0vSeauGGNYsvsElkrGawa9uphbpnl/V6WU9FzOFVu8fn9N0YSulPILu4+f5q3F\n+7j2rVWs35/lss3I15eRmVvAbR+sIS0nn+82HeGBTxP4akPpcMzhk3l0+8sC9p047fTeXcdyOJBx\nhqS03LKrBWDJ7hMUFBWXW348O5+Rry/n5R92ebQdy/em88CnG6iNe/xoQldK+cTLN3p1q2GXJs/d\nyX8Tyle9pKSfof9LP7N+fxa3frCGWZsOA3DsVOkt8BZsP05BkYU3Fu0t9/4R/1jGVW8sL7d848Es\nHvg0gakLyt6MDbLOFAKwNsX1XTUtFsOuYzn25w9+uoElu9M4V6wJXSlVR/2qTxufrWvn0Rz++O22\nCtsczMzjl+TySfbl+dae9IIdxwFYaPvX0ZmCIqfnmbnWpD1j9QH6vbjI5ee5G3KZtiKZMW+uZEvq\nKQCKbMM/BuP03prosWtCV0r5RP2wEP732yHcPySuxj973f5MHvosgdwyiTrrTCGPfrGxXPueL/zI\nRytTOHLqLADpuQVO74mb+APPzNrGrE2H2ZxqrcJJTj/Dk19tLpeYZ6w+AMDMhFSKHJL+kZNnWZeS\nSVpOPl0mLeD5/+2ksKh6x+Gltu7lHB8fbxISEmrls5VS1Stu4g+1HYLHDky91qt4r+/dmiu6xfD7\nmVur/HnnQ0Q2GmPiXb0W2DU8SilVCW93PnO3HmXu1qM+jub8VDrkIiJtRWSpiCSKyE4R+T8XbUaI\nSLaIbLH9PO9qXUqpC8O0u/vz+q29azsMv+TqZK+veDKGXgT8wRhzETAI+K2IXOSi3UpjTB/bzxSf\nRqmUqlNG92rJzf1jazsMv/THb7eRmlU9E5FVmtCNMceMMZtsj08DuwDfnc5WSl0QJt/Qs7ZD8BvD\nXltaLeutUpWLiMQBfYF1Ll4eLCJbRWSBiLj85kRkgogkiEhCenp6lYNVStUt2/86yp7I+7RtXMvR\nBD6Pq1xEpAGwHHjZGDOrzGuNAIsxJldExgJvGmO6VLQ+rXJR6sKRfrqAmIZhLN2Txs+JJ5iZkFor\nF974E2+rXSqqcvGohy4iocB3wJdlkzmAMSbHGJNrezwfCBWRaK+iVUoFnJiGYQBc0a05L994Mfte\nHsvEMd1rOarA40mViwAfA7uMMW+4adPS1g4RGWBbr+vrZJVSCrhzoHczJyr3POmhDwHuAUY6lCWO\nFZFHReRRW5tbgB0ishV4C7jd1NYVS0qpOqFReGi5YYeXftWL63u3rqWI6r5KLywyxqwC2y1F3Ld5\nB3jHV0EppS48/35wAMO6xHDHgHb85doeDHxlcZXeP6xLNCv3ZVRTdHWDzuWilKpVi353OTPGX8qw\nLjEABAcJLRqFu2w7sENTt+uZMf7SaomvLtGErpSqVV1aNOSK7s0rbXdg6rX8w83Vp188OJCQ4CD6\ntbuwSyM1oSul/NIAh9743YOsJ1DbNo2kXkj5tBVkGxS+se+Ffc2jJnSllF/6ZPylPDi0AwARocH2\n5Yt+d7nb99w1sD29Y6MA+OjeeJ4e1dXp9XXPXsnUmy6uhmj9gyZ0pZRfahAWwsQx3Xnqqi787urS\nxNy+WX362oZWWjSy1re3j64PQFCQMOs3Q1j29AiuuqgFj4/swp6XRnPv4PaAtbLm9gHt+PKhgfb1\nTRrbo9xnR9YLLresLtCErpTyW6HBQTx1VVci6zkX5L1xWx9u6R/L6j+PZPeLo2nTOML+WnCQEGdL\n8ABhIcFMvqEnSS+PIcKWqC+OjaJZ/XrMfGQwsU0inNbdolEYcx4fWmlsB6Zey29GdDqfzfM5nQ9d\nKVXndIiubz9BGuJBZ1pECAkurb5uFB7KxueuBqw3gQZ4aGgHxg+JIyoilBM5+U7vf+H6i5i79Sib\nDp1yWh4kFVZ01zhN6EqpC1rLqPByFziFhzrvJfq1a+LyZhbNbUM+/kKHXJRSqozQ4CDmPD4EgHfu\n7Evvto15YqTzfIOHMvO4a2B77hjgP1MYaEJXSikXLoltzK4po7nuEutUBFd0b+7Uk3/iq00cPXWW\nr9Yfqq0Qy9GErpRSbkRUUO2y9XA2D362oQajqZwmdKWU8sDMDamsS8nkkeEd7cv2nsitxYjK05Oi\nSilVieV70/nTd9sAa8WLv9IeulJKuVBYZGH0v1awOimDVftKb5k5eW5iLUZVMe2hK6WUg3Upmbwy\nfxfdWzZi9/HTTJq9nZ5tomo7LI9oQldKXXCS0nJp0SiMhuGhTsu/23iYP/x3K2A96QlwIDOPA5l5\nNR6jNzShK6UuGDNW7ycuuj73z7BWp/z8++F0bt7A/npJMq9ur7uZBvh86Ri6Uirgzd58mFN5hUye\nm2hP5gBXvbGc1Kw8PlyRQlJazVWs3Nw/tlrWqz10pVRAyj57DhFYuTeD333jvuc97LWlALw8f1dN\nhVZtNKErpQJS78k/1XYINa7SIRcRaSsiS0UkUUR2isj/uWgjIvKWiCSJyDYR6Vc94cLBzDN8ue4g\n2XnnqusjlFJ1RHJ6LiNfX0bcxB948qvNHDl1FoC4iT/UcmTu/TJxZLWt25MeehHwB2PMJhFpCGwU\nkUXGGMdizDFAF9vPQOB9278+t+NIDpNm7+DSuKZERYZW/gallF8rLLJQWGyhQVjl6Sg1K4/gIKG1\nbf7zK19fbn9tztajJKXlEhzkX1PaOvrhyaH22KtDpf+DxphjwDHb49MisgtoAzgm9HHA58YYA6wV\nkcYi0sr2Xp8q+a4sxvh61Upd8IqKLQQHCVID83yv359FkcXCW4v3sTYlq9wUtiUOZp5hTXImQzpH\n28e73Uk8llMdofrEo8M70bN19dazV6nKRUTigL7AujIvtQFSHZ4fti3zuZJfNIulOtauAk3+uWJ+\nScqo7TAqdSqvkMKi6vulzissothScSfoTEERnSct4M3F+ypsd67YwpS5iWSdKXT5+o4j2UyavZ3c\ngiJ+SXb/f3/bB2u488N1rE3JAqz/B2D9zv62cDcr9lqvzhz+92VMnLW90mTu78JDq7+o0ONPEJEG\nwHfAU8YYr3aDIjJBRBJEJCE9Pb3yN7hch/Vf7aErT0yem8idH61j74nTtR1KhfpMWcRjX2ystF1h\nkQVLJYm5LIvFcNHzP/Lc/3a4bXPk1Fnm2G7g8M2GVLftABYlnuCT1fuZPHcnYE3Ep/IKmb35MBaL\n4a6P1vHlukPcOs2asI9n5xP/0iLiJv7ALlsP2tU5sD5TFrF8bzrdn1vI+8uSufeT9X49Fl4Vj1ze\nkUcur/7b1XlU5SIioViT+ZfGmFkumhwB2jo8j7Utc2KMmQ5MB4iPj/cqI/vbLZ+Uf9tz3JpAcs5W\n/0n0/HPFvL1kH0+M7FLujjeeWLw7rdI2Xf+ygLEXt+S9u/p7vN5iW+dn5oZUXrnR9R3vx765kuwy\n/0cLth8j9WQeY3q1YsOBLH4/cys7J19j70wdyMxjye4TPPBpgv09C3cct6+nJHn/Z91BMnIL7ev8\naOV+vtt02GUc932y3uPtqiuWPT3C6R6n1cmTKhcBPgZ2GWPecNNsDnCvrdplEJBdHePnoGPo1elc\nsYXle52PnLakniq3rMTz/9vB/O3V8jWXk5yey6p9VR86KenMejomPO6dVbw4r/T00Mh/LOOleZ5N\nxjR9RQrvLk3m8zUHqhhlKWOM26GXvMIiAOZvP857y5JYuOM4GbkFjPzHMg5knLG3W5R4griJP5B2\n2npfzJkJ1h53kcXw087jxE38gaOnzvLF2oPcOu0XCoqKnZJ5bn4Ry/ak8diXm3hl/m6GvbaUd5cm\nAXD01FkW7jgOwNbUU07JHODHnSfKxf3WkiSnx+6SeSBqFRVeY8kcPOuhDwHuAbaLyBbbsmeBdgDG\nmGnAfGAskATkAff7PlSrkh56FY8667wTOfk0CAuhvgeVAN56e0kSby3ex38eHshlnaIB+NW7qwFc\nnrD6fM1BPl9z0O3JLF8qqWao6LMW7zpBh+j6dIwpvZTb2Hb8nhY+bD2czdbD2fRp25ggEVIyzpCy\naj9/ua78lKk7jmSTdaaQhIMnuXNAOzJyCwDKJWSLxTBtRTJ3DWxPVET5yqyComL74w7PzAfgD1d3\n5fVFe1n0u8vp0qIhUPpdALy2cI/TOj5ZvZ8p43oB2Hcou4+dpnnDcCbNLh1qKUnuO45k85fvrctv\neHu107pOFxQxfobzjRuS0607jCKLYd62mtmJ12VXdIth+r3x1PR4gidVLqug4rhs1S2/9VVQFQqA\nHroxhp93pTGsSzThocG8uzSJpbvT+Paxy9y+55Zpv3B1j5Y87zAX87I9aQzq2AywTjbU6zxnhDuU\naf2jPXoqv5KW1uEFX5mxej+T5yaS/MpYe8lZUbEFi4F6Ic4HkT8nnuCqi1q4XM+Dn1l7i89fdxH3\nXRZHcJDYd/zTliezdHc6Xz48kEvjmgKQW1CEMYbk9DPsPJrNLQ6XYz/x1WandRdbDJ/+coC7BrYj\nPDSY3IIirnt7lf31dSmZrNtvPblX8qv5S3IGraIi2J+Ry2sL95CcdobJ43o6lef9tPM4249kl9uW\n1xftBeDqf65g2t39GN2rVYU3U/h8zUFu7hdLaHAQK21HMvd+sr7cDvDnXdZhnf0OPfo9VTi/8GSZ\n/xfl7Lb4WF696RKCxPOjQl+qc1eKlvTQy473VafZmw+TeDSHSdf6ZmL7jQdP8vDnCVx7cSseHNaB\nNxbtpdhiyMk/R6Pw8j24/HPFpGaddTqxl5SWy/gZG7i+d2vOFBSxZHcaW18Y5bIH6KmwEOu4b0mP\nsajYfdVFTn7p/39eYRGR9ay/Sqv2ZRDdsB7dWzZy+97M3AL+8dNenr/uIiLqBdvnl16VlMHwrjEA\nXPf2KvaeOE3Kq84J6aHPE7ipbxve+HUf+zJjDGcddjBT5iWyKPEEX00YZN/xlwwFPPrvjXxwT3+y\nzhQy4d/OJyFLtt+VmQmpvDgvkfTTBVx9UXPeXJzk9LrjDu6c7f/tzg+txWAlEzHN3XaU7zYdZtZv\nLqNfuyYkpZ0uF4Mrj36xiaGdoyttN+7d1eWW/f3H3S7bvrrA9fLK7KvB+U7qmuFdY3jtluqZdMtT\ndW5yrpJD56dnejYrmruKAIvFsOFAVqWlXADvLU3mw5X77VehubM2JdM+zlmRhIMnAfhh+zFueu8X\n+7CA4zioo7Qc66H84ZOlU3impFv/sOZuPcqS3eV7Xe5knz3HS/MSnca+tx0+RWpWnr03fDrfug0P\nf146PrpqXwZfrz9Eh2d+IP9cMZts2wBwwhYfwN0fr2P0v1YC1kTrqif/5uJ9fLX+EH/8ditJaaU7\nKccTYruPn7b3rsuOSc/afISU9Fwe+mwD7y5NottzC8sdVaxJyQTKD81lninklmlrXCZSx//fsp6Z\ntR2w9vRvfn+NvaSuRMlUqwCJx07zwfJk+/NnZ1vfWzIUs/1wNnd9tJar3ljh9vPKWuVl6eW7S5Mr\nb6S89tRVXTgw9VpeufFi/uXQyagtdTChWzN6ppsaWEdfrz/EoFcXu0yyX22wllX903Zo686BjDP2\nXslcW1mXK6lZedw+fS2//2arPUG7s/HgSTpE1+eZMd358+jufHzfpYA1IRdbDDNW73fqAR/Ltu5I\njpw6a98BHcgsn7yf+nozf/zvVnviWL43ne83lxYb/ZKcwZS5iXy0aj+TZm/nwxUpPDNrGze8s5ob\n31tt780u3HGcwiILS/eUJq27P17HxFnbMQaOZefz6Beb7K+l5ZQforl9+hr+/uMeuj+30CmpWyyG\nY9nW9vO2HXOZ1Bx3wj8nnuD5/+0s12bk68v5eVcaf/9xD4VFFm6fvrZcG2OMvdLCE//6ueL6a0/9\nvOuEUw+4oMyY+rliC6uTMn3yWapmTLu7H73alD/qvLGv9XKbOwe2o0n9ejUdVjl1bsjFcVgq7XQ+\nzRuGu227dE8aaacL+HlXGjf0bm1fXlhk4b2lyYQECe8uS2JQx2YM7eL6kPbnXdZD9bZNI5iz5SiP\nDnddS7ol9RQAC3ce54u1B7lncJw1ht1p/PHbbcx/cijNG4VjjGHTwZNc0b05j9jWlX+uGBFrQl+/\nP4vJcxMJDQ7i7kHtAThuS5jnig3Hc/L59QdrOHzyLI0jQ3lwSAc2p55iye40+0T8TRvU45kxPXjg\n0w0UWwxN6tfjsk7N7EMAACfzzjnNLpeRW2jf1i2pp5g4y3r/xDaNI8odmbxV5sKTpXvS6dUmitUO\nvci1KVn2C0a+2ZDKltRTJKXlMrJ7cxYllq+EKFFYZCH+pUX25w99nuC2raOSE5KO/PXk3Us/1P1Z\n/S4Edw1sx5frDgFwcWxj5j0xjM/XHLB3MGqiGKCq6lwPXRzOz+46VvHJnK2p1sPgOVuce9bfbTrM\nkVNneeuOvnSKacDvZm5xOya/KPEE3Vs2ZPxlHUg8luN2zuStqacICwliaOdonvvfTu79ZD0nzxQy\nb9sxMnIL7NUFBzPzyDxTSL92TezvDQ8NpnVUBAcyzrBuv7Xn5jh8cjy7tAe8NjmTwyetCbZL8wY8\ncWUXPhl/qVMsHyxP4dUFu+y9+e83H+Ggwx1XhpXZeXVp3oAmkaFOQyezNll79le7OAE5e7PzJQbT\nlicz4OWf3Y4HvzBnJ7M3H2H7kexKr0L8Yu1BcvIrH7byRNkTm0qdr3sHx3FFtxjG9WldeeNaUOcS\numP5WeJR6+H0e8uSeO77HaxNybQPdxzPzud4Tj5NIkNZvjeN7LxzLNxxjBfnJfLPRXu5JDaKMb1a\n8s/b+pCRW8B7y5IoKCrm3k/W8/tvtlBYZOGkrSTtqh4tuO6SVojAawt3k3+umIOZZ5x6rltST9Gr\nTRQf3RfPM2O6s3JfOjN+OWDvtX61PpVii2Gl7Waz/duXJnSADtH17T10KE3oiUdz2H28dMe1xOHi\nE1cXWb34K2vp2gfLU+zLUtJzSU4v3RFd1Mr50HHC5R25qV9phUf9eqUnBy+J9axy5kyhb6pepnhY\n862UL00Z15PvfzsEgHfu7Mvtl7bl6VHd7JfrRzhcKDbj/gG8eXvfWomzMnVwyMWxh55DQVEx//p5\nH4VFFv699iBdWzRgyrhe9jlaO3cAAA+ySURBVB73U1d15YU5O7n743VsP5JNRGgwbZtG8ML1PRER\nLo6N4sa+bZix6gCHs87aT3Ydz8kn60whxRbDNT1b0qJROBNHd2fqwt0M/dsSMnILad4wjJ//MJyI\n0GB2HM3mzgHtCQ8N5pHhnViVlMGM1fs5nV/EsC7RrNyXwd9/3MOXaw/St11jujjc9gogLjqSWZuO\n2MexU9JzKbYYxr5lPcHYrmkkh7LyWLy7dLiih0Ni/nV8W75JSOWuAe04V2SxJ8ZhXaLZfOiUU0Kv\nFxKEiLW87t8PDmBYlxj7Zd/tm0Uy/rI4e+VJ26aR9vc1DAuhY0x9+rVvws39YjmRk0+7ppGMn7HB\naec2cUx3ftx5nM2HTnn7NZfz1FVdWLwrzWWJn1JVlfTyGDpPWgDAP27tbS9ZLSmdve4Saw9803NX\ns/dELk39YHzcE3UuoTuW0iUey2Hn0RwKiyy8cVtvii2Gf/28j4nfbeOaXi0JCRJ+fWlbZqzez/Yj\n2Uy4vCN/Ht293PSaf7ymG/O3H+OH7ceYcHlHOkTX59nZ22nbJJL37+rHxbZe6iPDO9G5eQM+/eUA\nPVo14sOVKfxz0V5u6R9L/jkLvduW9mZv6R9rrwf+6w09eXbWdqYtT6ZReAhv39GXoDIxdIxuQJ6t\nl9u1RQOS08/YK1kAQoKEjtH1Sck4Q6uocD4ZfykdHK5Am3rzxbx8Yy+CgoQHhnagXdNIvlx3kOFd\nY1i5L4M1yaUn4QZ3asaixBPsPn7avlPo2dr67+GTZ2nZyPV5idduuYQxF7eyPy+pe39keEenE5e9\nWkexP/2Mxwm9XnAQhRWUSAI8MbILJ3IKNKErr2x9fhS9p5Te8CIkOIiNf7mKkOAgp1Lfsrkhsl4I\nfdo2rrE4z1edS+iOFQMp6bmstiXNoV2iad4wnMh6Ifz2P5v4z9pD9GjViPDQYF6/rQ9ZZwpdjgcD\ntIqK4PnrepJwMIs/XdONkGDrWHiLRuHlLmy5skcLruxhXU9eYRGf/XLAXm7o+MVf07MlDcNDaBJZ\nj04xDfh6wiC2Hc4msl4wsU0iKevmfrEUWSyEBAXRICyEP323zWluj5Bg4Y4B7Zg8N5Hc/CKn3jlY\nj1xCgkt/Ga+6qAVXXdTCPuSzcl8GY3q15O+39qZBWAifjL/UWjPeIAyAuGbWncPwrjG0iCpN6N1a\nNqRxZCjv3dmPy9zUQneKcT7aaFI/lAnDO7Jw5/Fy5yY+vf9Sxs/YQERoMHcMaMcnq/fTo3Uj0nPy\nOZrtXC3z6PBOTFueTPtmkQQHCfs8vABmSOdmWkVygWnbNILULPdlxeH1glj29AgOZuVRYKu6amb7\n3Q8kdW4M3akEzsDXG1Jp1zTSXu1yTc8WtG0awemCInuPuX/7Jm6TeYk7B7bjjdv6EBJs/S9p2zSy\nXDIv64+jujOgQ1OW7kmnecMw2jkMT4SHBvPKjRfz7NgegDXh9m7b2H4Zd1lRkaFMuLwTDwztQJcW\n1gT5407rnBlv3t6HD+6J59eXtrXH6inHMfBxfVrbr1Js3TiC2y4tnU8tOEj4ZeJI3rmzLy0ceuiN\nwkPZ8vwot8kccLprOlgTfKeYBqx2uDPLganXcmDqtfadXv2wYJrWt/aMYhrUI7eg/InQIZ2tV8GG\n2r4Tx6tkK/LGbX2YfENPmjd0/Qfry+qEi8tcnfvfRwf7bN3KMx2j67PyT853AWpsu/nN4I7NmPWb\nywgLCSYuuj7Du8YwqmfL2gizRtS5hF7SQ7+iWwzRDepx5NRZpxOMIcFBPDCkAwC9Y6v3UCkqMpSv\nJwxm/bNX8r/Hh5S71Pf63q0Z3avqvzydmjcgOEjYfOgUraPCGdenDR2i6xNZL4RdU0bz59HdPV5X\nw/BQvnp4EKN7tuSK7s0rbNu6cQSR9ULcJkJ3WjQKZ/2kK+3PS2YadHUHmqiIUB4d3okvHhpIc9uO\nIyQoiLfv7MdlnZo5tS254rJFI2s8jkNMX08YxIf3xrtMzi0ahXPfZXE8MNT6e3DHgHZs++sopzYf\n3xfPkyM7065pJD8+dTkdY8pPoFT2xLUrcx4fYn8c2yTCPq1AbZs4xvPfEUf3DW5/Xp/bKiqcHq0a\n0b2ltePy+6u7um376PBObH7ualZPHMm8J4by6k0X88jwjk5tNj93NQCtHY4ay35XC5+63On50qdH\nsPJPV7D3pTF8NWGQU0VZoKtzQy4lveYuLRryq75t+L+vt5T7I7pjQDvyz1kY6zDeW52auxlz9laj\n8FCuv6QV3285SkyZ5BpRr+rTsg7u1IzBZZJlRUKDg3jyyi4eXW5eonnDcGY+MpiiMnceeeO23hzK\nKi2ZFBF7sim56Cc4WBjeNYbhXWP4b0Iq7y1L5g+jutIqynqrrpttFTglO4iOMfXtc9gAPDaiE22b\nRDKye3OnI7irerRg6oLd3DGgLY3CQ7l/SBz556zxlQyd/X5UNwBuv7Qtr8zfzYzxl3L/p9aJqb57\n7DKG/m2JvUy0YVgIb9uOYFpFhRMWEoyIEBYSREGRha8eHgTA+mevZOCri/FkuqGHh3Xgw5X7nZY9\nO7Y7EaHB3D6gHf1eXMTp/CIeGNKBT1Zb2z04tAMjusVwz8eup5pt0ziCR4d3YmoVL+8f1iWayeN6\nsXJfBilurjp+7ZZL+NO325j3xFCnuWxK/OfhQXSIro8xhgU7jnNNz5a84eLivV/Ht7X/HjSxxVxy\nTuaZMT1IO51PsAhN6tdjw6SriKgXzFuL9zF9RQqf3T+AZXvSeM523qYkJ0y7uz/554qddvwXmjqX\n0Ef3bMmLv+rFrf1jCQsJIqZhWLmeVHhoMI+NqP7J5KvTb67ozPdbjtqTWk2rqGflzoAO5XunjuWQ\nZV3WKZrgILEfUQHcGt+WW+NLh4I2P3e1/Qo8EeHbRwfTvpnzH6y7I5bOzRs49eBfuL6n21geHmYt\n3YwuM67aOirCntBbRoUzolv5o5yS8tGSOJs3Cqd5wzBO5BQw9/GhXP+Oc+JrHBlKYZGFp0d1477L\n4thxJMc+VQFA95aNuNw2p01MwzBO5xc5XVD3nG3mx3lPDOWtxfsY1bMlk2Zvp6DIwtpnriQyrPxO\nf/+rY7np/V/YfOgU1/RswbWXtGbzoZPMWH2A4V1jWL433b4Dn/3bIZzIyWfUP8tfxXvNRS25bar1\n+wkNFs4VO++1SpKpiNg7VMueHsGqpAyuu6QV328+wl/nJrq86tKR4wWDJZ2aZ8f2sA9h3jM4zp7Q\nS3hzNBxo6lxCDwoS7hlUelhYMs1roOnaoiH/eWggXVu6HnMPBC0ahZP8ytgK25S9nDq+moY0RMSe\nzAd3bMZJ2+3Q3r+7H9NXpPDBihS3N1eZevPFvLZwj1Ot8hcPDuTbjYftiSskSCiyXej101OXOx3V\nfTVhkP3OPD/97nK6OpxnKfnEuGblT6T3ahPF9HvjAWsyK7YYp4qNlFfG0vHZ+YTY7hE6+zdDnN5/\nQ+/W/Hl0d8JCgkg4eJL+tqGJqIhQp/U4HkUEOQzSzn1iKJNm7+CJkZ0ZP2OD24tt4qLr2+cEv2dw\nHI0iQhnX5/zvUPnHa7rRKsq3R8d1nVQ270h1iY+PNwkJnl3WrVRtys47R+8pP/HX6y9ivMPRhKc2\nHjxJq6hwmtavR0iQ2E+8O+o9+SeaNajHkj+McFqelJbLjNX7eXFcLzo+a50rvSondQ9l5hEZFlzu\nyMMTJTuZA1OvJfvsOXYeya7w5LiqGSKy0RgT7+q1OtdDV6qmRUWGnldljCcnV7e+MMrl8s7NG/Cy\n7bZxJWWcVdHORc/eG1ERoZrM6wBN6ErVERPHdPe6esUb9w1uT1Rk3bhCUllpQldKuTTZdks7VXfU\nuTp0pZRSrmlCV0qpAKEJXSmlAkSlCV1EPhGRNBHZ4eb1ESKSLSJbbD/P+z5MpZRSlfHkpOinwDvA\n5xW0WWmMuc4nESmllPJKpT10Y8wKIKsGYlFKKXUefDWGPlhEtorIAhFxO2GGiEwQkQQRSUhPT3fX\nTCmllBd8kdA3Ae2NMb2Bt4Hv3TU0xkw3xsQbY+JjYmJ88NFKKaVKeDSXi4jEAfOMMZVeaSAiB4B4\nY0xGJe3SgYMeRVleFODJvcgqaufutbLLK3ru7nE0UOH2e6C6trGyZZ5srz9vn6vlgf4devq8rm6f\nq2WVfYf+vH2ulldl+xobY1z3iI0xlf4AccAON6+1pHTHMAA4VPK8un6A6efbzt1rZZdX9LyCxwn+\nuo2VLfNke/15+y7E79DT53V1+7z5Dv15+6r6HVb091n2p9IqFxH5ChgBRIvIYeAFIBTAGDMNuAV4\nTESKgLPA7cb2ydVorg/auXut7PKKnrt77AvVtY2VLfN0e8+XfoeetfPkO/T0eV3dPlfLauI7rHO/\no7U2fW4gE5EE42Z6y0AQ6NsHgb+Nun2BSa8UrR7TazuAahbo2weBv426fQFIe+hKKRUgtIeulFIB\nQhO6UkoFCE3oSikVIDSh1zARGSYi00TkIxH5pbbj8TURCRKRl0XkbRG5r7bj8TXb7KIrbd/hiNqO\np7qISH3bNB0BN+meiPSwfX/fishjtR2PL2lCrwJ3UwmLyGgR2SMiSSIysaJ1GGNWGmMeBeYBn1Vn\nvFXli+0DxgGxwDngcHXF6g0fbZ8BcoFw/Gz7wGfbCPBnYGb1ROk9H/0N7rL9Dd4GDKnOeGuaVrlU\ngYhcjvWP+XNjmwZBRIKBvcDVWP/ANwB3AMHAq2VW8YAxJs32vpnAg8aY0zUUfqV8sX22n5PGmA9E\n5FtjzC01FX9lfLR9GcYYi4i0AN4wxtxVU/F7wkfb2BtohnWnlWGMmVcz0VfOV3+DInID8Bjwb2PM\nf2oq/uqmN4muAmPMCtu8No4GAEnGmBQAEfkaGGeMeRVwebgqIu2AbH9K5uCb7bNdTVxoe1pcfdFW\nna++P5uTQFh1xHk+fPQdjgDqAxcBZ0VkvjHGUp1xe8pX36ExZg4wR0R+ADShK7s2QKrD88PAwEre\n8yAwo9oi8q2qbt8s4G0RGQasqM7AfKRK2yciNwHXAI2x3vilLqjSNhpjJgGIyHhsRyTVGt35q+p3\nOAK4CesOeX61RlbDNKHXAmPMC7UdQ3UxxuRh3WEFJGPMLKw7rYBnjPm0tmOoDsaYZcCyWg6jWuhJ\n0fN3BGjr8DzWtixQ6PbVfYG+jYG+fR7ThH7+NgBdRKSDiNQDbgfm1HJMvqTbV/cF+jYG+vZ5TBN6\nFdimEl4DdBORwyLyoDGmCHgc+BHYBcw0xuyszTi9pdtXt7cPAn8bA337zpeWLSqlVIDQHrpSSgUI\nTehKKRUgNKErpVSA0ISulFIBQhO6UkoFCE3oSikVIDShK6VUgNCErpRSAUITulJKBYj/B1DhzOPw\natpvAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/660 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Training loss:  2.171063547196586\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-19-f0381787101f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mv\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-13-4e591606ccc1>\u001b[0m in \u001b[0;36mrun\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFFNN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0mval_losses\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_losses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m     \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_losses\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_losses\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-7-7eb90c684dd7>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(model, train_dataloader, val_dataloader)\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Training loss: \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m         \u001b[0mval_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrun_one_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_dataloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m         \u001b[0mval_losses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Validation loss: \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-18-34af15b768d5>\u001b[0m in \u001b[0;36mrun_one_epoch\u001b[0;34m(model, loader, criterion, update, optimizer, scheduler)\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m         \u001b[0mlrplot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscheduler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_lr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m         \u001b[0mlrs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlrplot\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mupdate\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'get_lr'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0fVxkeVnlebI",
        "colab_type": "code",
        "outputId": "44b27386-651b-4051-9d9c-8d1986d16071",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        }
      },
      "source": [
        "tempo = np.array([2.3753485043843585,2.0615390134580207,1.8913816471894582,1.7839566740122708,\n",
        "         1.7331338983593565,1.6891762694626145,1.7151012378208566,1.6674280624498021,\n",
        "         1.6835154702266057,1.6134144763151805,1.5996342856775632,1.5694274478789532,\n",
        "         1.6674274412971555,1.5510204764026585,1.6405746139360198,1.5249536400491541,\n",
        "         1.4834835471528949,1.5473498891700397,1.4986431845209816,1.4536923199892045,\n",
        "         1.542754511851253,1.4373818435452201,1.3993541067296809,1.433429891051668,\n",
        "         1.3958047336701191,1.4357629985520335,1.5100324656927224,1.399349873625871,\n",
        "         1.3719498879981764,1.3774904342311802,1.3632555172298895,1.3484137666044813,\n",
        "         1.3477942674448995,1.3707056564815117,1.3369793112530852,1.3402825184843756,\n",
        "         1.356912567579385,1.3421990789247282,1.334889861038237,1.3316648706342236,\n",
        "         1.336125900889888,1.335822554339062,1.3355693424289876,1.337578166163329,\n",
        "         1.336651961460258,1.3398560769630201,1.3405852402701521,1.3392708445137198,1.3387861782854253])\n",
        "plt.plot(tempo)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f90f9c0a710>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXicZb3/8fd3sjZ7l2xNmy60Tdt0\nb0o9bG2otCC7VlAQkYNWPBXxgNtPzxGPRz2il3BQQA4CoqgoS1kVpGBpKciSdN/3JU2bpFuSJs02\nc//+mGlJ26ztpJOZ+byuK1dmueeZ7wPDJw/33Is55xARkcjiCXUBIiISfAp3EZEIpHAXEYlACncR\nkQikcBcRiUCxoXrjAQMGuKFDh4bq7UVEwlJpael+51xmZ+1CFu5Dhw6lpKQkVG8vIhKWzGxnV9p1\n2i1jZoPNbJGZrTOztWZ2Rwdtp5lZi5nN7U6xIiISXF25cm8B7nLOLTOzVKDUzBY659a1bmRmMcA9\nwOs9UKeIiHRDp1fuzrm9zrllgdu1wHogr42mtwPPAZVBrVBERLqtW6NlzGwoMBl4/6TH84BrgV8H\nqzARETl9XQ53M0vBf2X+dedczUlP/y/wbeecr5NjzDOzEjMrqaqq6n61IiLSJdaVhcPMLA54Bfi7\nc+7eNp7fDljg7gCgHpjnnHuhvWMWFRU5jZYREekeMyt1zhV11q7TL1TNzIDHgPVtBTuAc25Yq/ZP\nAK90FOwiItKzutItcz5wE3Cxma0I/HzCzG4zs9t6uL5TbNxXy89e28Dh+qaz/dYiImGj0yt359xS\nPupy6ZRz7gtnUlBndhyo46G3tnLZuFwykuJ78q1ERMJW2K0tk5ueCMDe6qMhrkREpPcKu3DPSfOH\ne0VNQ4grERHpvcIu3PunJBDjMfYp3EVE2hV24R7jMbJTE9hbrXAXEWlP2IU7QHZ6orplREQ6EJbh\nnpOWyD5duYuItCsswz07LZGKmsZQlyEi0muFZbjnpidypLGF2obmUJciItIrhWW456RrOKSISEfC\nMtyzA2Pd91Wra0ZEpC1hGe6apSoi0rGwDPdszVIVEelQWIZ7YlwMGUlxmqUqItKOsAx30Fh3EZGO\nhG+4pyfqyl1EpB3hG+5piRotIyLSjrAN9+y0RA7UNdLU0uGe3CIiUSlswz03PRHnoLJWXTMiIicL\n23DP1ixVEZF2hW2452iWqohIu8I23DVLVUSkfWEb7ul94kiI9ahbRkSkDWEb7mYWGOuubhkRkZOF\nbbjDsbHu6pYRETlZeIe7ZqmKiLQpvMM9sN2ecy7UpYiI9CphHe7ZaYk0tfg4VK/t9kREWgvrcNdw\nSBGRtoV1uGuWqohI2zoNdzMbbGaLzGydma01szvaaHOjma0ys9Vm9q6ZTeyZck+kWaoiIm2L7UKb\nFuAu59wyM0sFSs1soXNuXas224EZzrlDZnYZ8AgwvQfqPUFmagIeQ8MhRURO0mm4O+f2AnsDt2vN\nbD2QB6xr1ebdVi95DxgU5DrbFBfjYUBKgoZDioicpFt97mY2FJgMvN9Bs1uBV9t5/TwzKzGzkqqq\nqu68dbs0S1VE5FRdDnczSwGeA77unKtpp00x/nD/dlvPO+cecc4VOeeKMjMzT6feU2iWqojIqboU\n7mYWhz/Y/+icW9BOmwnAo8DVzrkDwSuxYznp2ihbRORkXRktY8BjwHrn3L3ttMkHFgA3Oec2BbfE\njmWnJVLT0MLRJu/ZfFsRkV6tK6NlzgduAlab2YrAY98F8gGccw8D3wf6Aw/5/xbQ4pwrCn65pzo2\nkWlfTQPDBiSfjbcUEen1ujJaZilgnbT5IvDFYBXVHcfGuu+tPqpwFxEJCOsZqqBZqiIibQn7cNcs\nVRGRU4V9uCcnxJKaGKvhkCIirYR9uENgrLu6ZUREjouMcNcsVRGRE0RGuGuWqojICSIj3NMTqapt\npMXrC3UpIiK9QkSEe3ZaIj4H+480hboUEZFeISLCvfUsVRERiZBwzz4+1l397iIiECHhnnPsyl2r\nQ4qIABES7v2S4omLMQ2HFBEJiIhw93iMbA2HFBE5LiLCHTRLVUSktYgJ9+z0RCrULSMiAkRQuOem\nJbK3+ijOuVCXIiISchET7jnpiTQ0+6g52hLqUkREQi5iwv34WHf1u4uIRE64a5aqiMhHIibcNUtV\nROQjERXuZrDnsK7cRUQiJtzjYz0M65/Mhr01oS5FRCTkIibcAQrz0llbrnAXEYmocB+fl8aew0c5\nWKd13UUkukVUuI8bmA7Amj3VIa5ERCS0IircC4+Fe7nCXUSiW0SFe3pSHPn9knTlLiJRL6LCHWBc\nXhpr9uhLVRGJbhEY7unsOlhPdX1zqEsREQmZTsPdzAab2SIzW2dma83sjjbamJn90sy2mNkqM5vS\nM+V27tiXqmvV7y4iUawrV+4twF3OubHAx4D5Zjb2pDaXASMDP/OAXwe1ym4Yl6cvVUVEOg1359xe\n59yywO1aYD2Qd1Kzq4HfO7/3gAwzyw16tV3QLzmevIw+rFa/u4hEsW71uZvZUGAy8P5JT+UBu1vd\nL+PUPwCY2TwzKzGzkqqqqu5V2g2FA9NYqxEzIhLFuhzuZpYCPAd83Tl3WpfFzrlHnHNFzrmizMzM\n0zlEl4zLS2fb/jpqG/SlqohEpy6Fu5nF4Q/2PzrnFrTRZA8wuNX9QYHHQmJ8oN99ndaZEZEo1ZXR\nMgY8Bqx3zt3bTrOXgM8HRs18DKh2zu0NYp3dUpiXBsAahbuIRKnYLrQ5H7gJWG1mKwKPfRfIB3DO\nPQz8DfgEsAWoB24Jfqldl5WaSHZagvrdRSRqdRruzrmlgHXSxgHzg1VUMIwbmM5qhbuIRKmIm6F6\nTGFeOlurjlDf1BLqUkREzrqIDffxeen4HKzXzkwiEoUiNtzHHftSVZOZRCQKRWy456Ql0j85Xsv/\nikhUithwNzPG5elLVRGJThEb7uDvmtlceYSGZm+oSxEROasiOtzH56Xj9Tk27qsNdSkiImdVRIf7\nsT1V1TUjItEmosN9UN8+pPeJ08YdIhJ1IjrczYzx+lJVRKJQRIc7+BcR27ivlqYWX6hLERE5ayI+\n3McNTKfZ69hUoS9VRSR6RHy4H1vbXZOZRCSaRHy45/dLIjUhVhtmi0hUifhw93iMwrw0Vpcp3EUk\nekR8uAOcO6w/q/dUc+BIY6hLERE5K6Ii3GePzcbn4M0NlaEuRUTkrIiKcC8cmEZeRh9eX1sR6lJE\nRM6KqAh3M+OSsdm8vblKOzOJSFSIinAHf9dMY4uPJZv2h7oUEZEeFzXhPm1YP9L7xLFwnbpmRCTy\nRU24x8V4mDU6izc3VNDi1VIEIhLZoibcAWYXZnO4vpkPdxwKdSkiIj0qqsL9olGZJMR6eH3dvlCX\nIiLSo6Iq3JPiY7lgxABeX1uBcy7U5YiI9JioCnfwd83sOXyU9Xu1SqSIRK6oC/dZY7IxQ10zIhLR\noi7cB6QkUDSkr2arikhEi7pwB5g9Nod1e2vYfbA+1KWIiPSITsPdzB43s0ozW9PO8+lm9rKZrTSz\ntWZ2S/DLDK5LxmYD8MZ6Xb2LSGTqypX7E8ClHTw/H1jnnJsIzAR+YWbxZ15azxk6IJlR2SnqmhGR\niNVpuDvnlgAHO2oCpJqZASmBtr1+da7ZY3P4YMdBDtU1hboUEZGgC0af+wPAGKAcWA3c4Zxrc36/\nmc0zsxIzK6mqqgrCW5++2YXZeH2Of2iNdxGJQMEI9znACmAgMAl4wMzS2mronHvEOVfknCvKzMwM\nwlufvvF56eSkJWpIpIhEpGCE+y3AAue3BdgOjA7CcXvUsTXel2zaz9Emb6jLEREJqmCE+y5gFoCZ\nZQMFwLYgHLfHXTlxIEebvTy8eGuoSxERCaquDIV8CvgnUGBmZWZ2q5ndZma3BZr8N3Cema0G3gS+\n7ZwLix0xzh3Wj2sn5/Hgoi2sK68JdTkiIkFjoVpAq6ioyJWUlITkvVs7XN/Ex+9dQnZaAi/MP5+4\nmKic1yUiYcLMSp1zRZ21i/oky0iK50fXFLK2vIZHloRFb5KISKeiPtwBLh2Xy+UTcrn/jc1sqtBq\nkSIS/hTuAf91VSEpibF889lVeH1a611EwpvCPWBASgI/uKqQlbsP89hSdc+ISHhTuLdy5YRcLhmb\nzS9e38S2qiOhLkdE5LQp3FsxM358zTgSYj1869lV+NQ9IyJhSuF+kqy0RL5/ZSElOw/x2NLtoS5H\nROS0KNzb8Kkpecwem809r22gdOehUJcjItJtCvc2mBk///REcjMS+eqflnFQywKLSJhRuLcjvU8c\nD90wlQNHmvj6X1Z02v9e09DMiyv20Oxtc7VjEZGzSuHegfGD0vn+lWNZsqmKBxdtabfd+r01XPWr\npdzx5xXct3DTWaxQRKRtCvdO3Dg9n6snDeS+Nzbx7pZT10NbsKyMax96h/omL8UFmfx68VY+2N7R\nxlUiIj1P4d4JM+Mn145neGYKX/vzcipqGgBobPHyvedXc+fTK5k0OIO/fu1CHrhhCvn9kvj3v6yg\npqE5xJWLSDRTuHdBckIsv75xCnWNXm5/ajm7DtRz3cP/5I/v7+K2Gefwh1unk5maQHJCLPddP4l9\nNQ384MW1oS5bRKKYwr2LRman8pNPjuOD7Qe5+Bdvsa2qjv+7aSrfuWw0sa2WCZ6S35evFo9gwfI9\nvLKqPIQVi0g0iw11AeHk2smDWFdeQ8nOQ9x73SSGDUhus91XLx7B4k1VfO/5NUwd0pfc9D6ntHHO\n8cH2gzR5fVw4MrT7yYpI5In6zTp6yvb9dXzi/reZMiSDJ/91Oh6PAf5Qf2tjFQ8s2kLpzkPExRhL\nvlXc5h8AEZGTabOOEBs2IJnvXzmWd7Yc4PF3tuPzOf62ei9X/GoptzzxIfuqG/jmnAKcg98s0TIH\nIhJc6pbpQZ+ZNpg311fys9c28tQHu9haVcewAcn8bO4ErpmUR3ysh21VdTz1wS7mF59D/5SEoNfg\n9TkWrtvHJWNziAn834OIRD5dufcgM+OeT42nf0o8cTEefvXZybxx5wyuKxpMfKz/H/1XZg6nocXL\nE+/u6JEa/rp6L7f9YRmvrtnbI8cXkd5JV+49rH9KAm9/q5gYj2F26pXziKxULi3M4Yl3d/Cli4aT\nlhgX1Pd/eaV/xM6iDVVcMWFgUI8tIr2XrtzPgtgYT5vBfsz84hHUNrTwh/d2BvV9axqaWbyxCjNY\nvKlS69OLRBGFey8wLi+dGaMyeezt7Rxt8gbtuK+vraDJ6+Omjw1h/5Em1pRXB+3YItK7Kdx7ifnF\nIzhQ18RfPtwVtGO+sqqcvIw+fG3WSMzgrY1VXXpddX0zG/bVBK0OETn7FO69xLnD+jFtaF8eWbKN\nppYzXzb4UF0TSzfv54qJuQxISWDCoAwWbazs0mvvfmkNn3zoXRqag/d/ESJydince5H5xSMor27g\nhRV7zvhYr63dR4vPcWXgS9TigkxW7D7c6cYjNQ3NvLpmH/VNXv657cAZ1yEioaFw70VmjMqkcGAa\nv35rK94z/PLz5ZXlDBuQTOHANABmFmThHLy9ueOumb+u2ktjiw+PwVsbunalLyK9j8K9FzEz5heP\nYPv+ujMal15Z28B72w5w5YTc46N0JuSl0z85nkWdBPazpWWMyEqhuCCLRRurCNXyFCJyZhTuvcyc\nwhyGZybz4KKtpz108dXV+/A5uGLiR+PaPR5jxqhMlmze3+7/FWzfX0fpzkPMnTqI4tFZ7DpYz7b9\ndadVg4iEVqfhbmaPm1mlma3poM1MM1thZmvNbHFwS4wuMR5j/swRrN9bwyX3LebxpdupPtq9jT9e\nWVVOQXYqo7JTT3h8RkEmB+uaWFV2uM3XPVdahsfg2sl5zCzwr1TZ2ZW+iPROXblyfwK4tL0nzSwD\neAi4yjlXCHw6OKVFr09OyeO+6yeSmhjHD19Zx/SfvMG3n13F6rLOx6mXHz7KhzsOccWE3FOeu2hk\nJh6DRW0MifT5HAuWlXHhyEyy0xIZ1DeJUdkpXR5hIyK9S6fh7pxbAnS0KegNwALn3K5Ae6XBGTIz\nrp08iBfmn88rt1/ANZPyeGllOVc+sJSrH3yHf25tfxTL31b7++pbd8kc0zc5nkmDM1jcRmD/c9sB\nyqsbmDt10PHHiguy+GD7QY40tgThrETkbApGn/sooK+ZvWVmpWb2+fYamtk8Mysxs5Kqqq5NqIl2\n4/LS+emnJvDed2dx95VjOVTXxOcff5+XVra9y9PLK8sZl5fW7kYixQVZrCyrZv+RxhMef7a0jNTE\nWC4Zm338sZkFWTR7He+0sTG4iPRuwQj3WGAqcDkwB/hPMxvVVkPn3CPOuSLnXFFmpnYf6o70PnHc\ncv4wXr79Aibn9+VrTy3nsaUnrgO/60A9K8uqj49tb0vx6CwAlmz66I9rbUMzr67Zy1UTB5IYF3P8\n8aKhfUlNiOUtdc2IhJ1ghHsZ8HfnXJ1zbj+wBJgYhONKG9L7xPH7fz2XSwtz+O9X1vHTVzccH674\ncmDP1svb6G8/ZmxuGgNSEk7od//b6r00NPv4VKsuGYC4GA8XjhrAog0aEikSboIR7i8CF5hZrJkl\nAdOB9UE4rrQjMS6GB2+cwo3T83l48VbuemYlzV4fr6zay5T8DAb1TWr3tR6PMbMgkyWbqmjx+pc5\neK50D8Mzk5k8OOOU9jMLsthX08CGfbU9dj4iEnydruduZk8BM4EBZlYG3A3EATjnHnbOrTez14BV\ngA941DnX7rBJCY4Yj/Gja8aRnZbIvQs3sfNAPev31vD9K8Z2+trigiyeLS1jZdlh+icn8MGOg3zr\n0oI2lyWeOcrfffaPDZWMyU0L+nmISM/oNNydc5/tQpufAz8PSkXSZWbG12aNJDM1ge89vxqzjrtk\njrlg5ABiPMaiDVV4DDwGn5w8qM22WWmJjMtL462NlcwvHhHsUxCRHqKdmCLAZ8/NJy+jD2WHjpKd\nlthp+/Q+cUzN78s/NlRSfbSZ80cMICe9/dcVF2Tx4KItVNc3k54U3J2iRKRnaPmBCHHRqExumJ7f\n5fYzCjJZt7eGPYePnjC2vS0zC7LwOVjSyaJjItJ7KNyjVHGBf0hkakIscwpzOmw7aXAGfZPiNFtV\nJIwo3KPUmNxUhmcmM7do0Alj29sSE1h0bPHGqm4vZrZ81yHuf2Oz9m8VOcsU7lHKzHj1jgv5j8s7\nH10D/slPB+qaWLWn6/uwtnh93PX0Su57YxNPBmHz70UbK7nzLys05l6kCxTuUSwhNoYYz6nDH9ty\n0chMzLq3SuRzy8rYtr+OIf2T+J9X17O16sjplgrAL9/czILle1izR/u7inRG4S5d0jc5nsmDM7q8\nFEFDs5f739jMpMEZPP3lfyExLoY7n155fOJUd22prGX5Lv9SxX9fu++0jiESTRTu0mXHFh2rqm3s\ntO2f3t9FeXUD35pTQHZaIj+6Zhwrdx/m4cVbT+u9ny3dQ4zHGJObxmsKd5FOKdyly2YX5mAGP3xl\nXYf93kcaW3hw0RbOH9Gf80YMAOCKCQO5auJA/veNzazpRr89gNfneH55GcUFmXxm2mC2VB5hS+WZ\ndfGIRDqFu3RZQU4q35xTwMsry3lkybZ22/126XYO1DXxjdkFJzz+w6sL6Z8Sz51Pr6Ch2dvl9317\ncxUVNY3MnTqI2YX+JYnVNSPSMYW7dMtXZpzD5RNyuee1DSzedOqkpsP1TTyyZBuXjM1mcn7fE57L\nSIrnZ3MnsqniCPcu3NTl93ymtIy+SXFcPDqb3PQ+TBycweu9KNy3VB7h129t1Sge6VUU7tItZsbP\n505gVHYqt/9pGTtO2kD74cXbONLUcspV+zEzRmXyuY/l85u3t/H+tvZ3lDqmur6ZhWsruHpSHvGx\n/o/rnMJsVpZVU3746JmfUBD86K/ruOe1DWys0MqZ0nso3KXbkuJj+c3ni4jxGF/6fcnxbfgqaxp4\n4t3tXDMpj4Kc1HZf/91PjCG/XxJ3PbOS2oaON/9+aVU5TV7fCUskHJtR2xuu3jdV1PJWYG38N9dr\nBq/0Hgp3OS2D+yXx4A1T2La/jjv/sgKfz/Grf2yhxev4+sdHdvjapPhY7r1uIuWHj/IfL6zpsDvj\n2dIyRuekUjjwo+WGz8lMYWRWCn9fWxG08zldj769jcQ4D+dkJrNwXejrETlG4S6n7bwRA/jeJ8bw\n+roKvvfCGp76YBfXTxvMkP5t79/a2tQh/bjzklG8uKKcp0t2t9lmc0UtK3cfZu7UQaesNT+nMIf3\ntx/gYF1TUM7ldFTWNvDC8nLmTh3ENZPyWFl2mMrahpDVI9Kawl3OyC3nD+VTUwbx1Ae7iPH415fv\nqq/MHMEFIwZw90tr2djGTk/PlpYR6zGumZx3ynNzCnPwOXhjfeiulp/8506afT5uvWA4s8Zk41z3\nZvCK9CSFu5wRM+PH147jkrHZfDMwYamrYjzGfddPIiUhjvl/WkZ9U8vx51q8PhYs30Px6CwGpCSc\n8tpxeWnkZfQJWb97fVMLT763k0vGZDNsQDJjclPJy+jDG+p3l15C4S5nLDEuht98vogvXji826/N\nTE3g/s9MYmvVEf7zhbXHH1+yuYqq2sZ215o3M2YXZrNk8/7jX+ieTc+VlnG4vpkvXTT8eD2zxmSx\ndPP+bo3hF+kpCncJufNHDOD2i0fy3LIyni0tA/xdMv2S44+vO9+WOYU5NLX4WLzx7G4i4vU5Hlu6\nnYmDMyga8tFY/o+PyeZos5d3t+4/q/WItEXhLr3CHbNGMn1YP/7zhTWU7DjIG+squXrSwONj29sy\nbWg/+ifHn/W1Zhauq2DHgXrmXTj8hC96pw/vR3J8jLpmpFdQuEuvEOMxfvnZySTFx3Djo++fMra9\nvdd8fEw2izZU0thy9rpCHn17G4P69mFOYCmEYxJiY7hoVCZvrq/QbFUJOYW79BrZaYnce/0kGlt8\njM1No3BgeqevuXRcDkcaW3h3a+ezXYNh2a5DlOw8xK0XDCM25tT/fD4+JpuKmkatOS8hp3CXXmXG\nqEweuWkqP//0hC61P29Ef1ISYvn7mrPTNfPo29tIS4zluqLBbT5fPDoLj8HCEA7RFAGFu/RCswtz\nunTVDv6ukJkFmSxcV4G3h/dp3XWgntfW7OOG6UNITohts02/5HimDunLmwp3CTGFu4S9S8flcKCu\nqduB2t1++keXbiPGY3zhvKEdtps1Jpu15TXsre4dC5tJdFK4S9ibNTqbUdkp3PX0Sjbs67yv2+tz\n/PtfVjDtR290eeOQhesqePK9ncydOpic9I4nan18jP+LVo2akVBSuEvY6xMfw29vOZekhBhu+e2H\nHV4xe32Obz6zkueX78EBtzzxIbsP1nd4/PV7a7jjz8sZn5fO968Y22k952QmM7R/krpmJKQU7hIR\n8jL68PgXplHb0MItv/2QmjaWEvb5HN95bhULlu/hG7NHseAr59HY7OXm337AoXYWIKuqbeSLvysh\nNdG/zHGf+JhOazHzD9F8d8sB6kIwe1YEFO4SQQoHpvPrz01hS+URvvKHUppafMef8/kc33thNc+U\nlnHHrJF89eKRjMxO5dGbp1F26Ci3/u7DU5YNaGj2Mu/JEg7UNfLo56d1a92cWWOyafL6eHuzZqtK\naHQa7mb2uJlVmtmaTtpNM7MWM5sbvPJEuufCkZn89FMTeGfLAb6zYBXOOZxzfP+lNTz1wW6+Wjzi\nhPXmzx3Wj/uvn8Ty3Yf52lPLj4+4cc7x7edWsXzXYe67bhLjB3Vt9M4xRUP7kpYYG9JVKyW6tT2e\n60RPAA8Av2+vgZnFAPcArwenLJHTN3fqIMoPH+XehZsYmN6HI40t/OG9XXx5xnDumj3qlLXhLxuf\ny91XjOUHL6/jBy+t5YdXF/Lgoi28uKKcb8wexWXjc7tdQ1yMh+LRWSzaUInX54jxWOcvEgmiTsPd\nObfEzIZ20ux24DlgWhBqEjljt188gvLDR3lg0RYAbr1gGN+5dPQpwX7MF84fxt6aBv5v8Taqaht5\nbe0+rpk0kPnFI067hlljsnlxRTlLt+xnxqjM0z6OyOnoypV7h8wsD7gWKKaTcDezecA8gPz8/DN9\na5F2mRk/umYcPufITkvkzktOvWI/2bfnjKaiuoEXVpQzJT+Dn35qQqev6UhxQSZ5GX348pMl/Hzu\nRK6cOPC0jyXSXdaVBY4CV+6vOOfGtfHcM8AvnHPvmdkTgXbPdnbMoqIiV1JS0u2CRXpSU4uPZ0p3\nc9m4XPolx5/x8fYfaeS2J0sp2XmIrxaP4M5LRuFRF42cATMrdc4VddYuGKNlioA/m9kOYC7wkJld\nE4Tjipx18bEebpw+JCjBDjAgJYE/feljfGbaYB5YtIV5T5ZS28YwTZFgO+NuGefcsGO3W125v3Cm\nxxWJFPGxHv7nk+MZk5vGD19ZxycfepdHby46YSPx6qPNbKmsZVPFEeqbvFw5IZesbgy9FDlZp+Fu\nZk8BM4EBZlYG3A3EATjnHu7R6kQihJlx83lDGZmVwr/9aRlXPfAOV00cyPb9dWyqqKWytvGE9j99\ndT2Xjcvl5vOGMiU/44z6/tuy+2A9AzP6aBRPBOtSn3tPUJ+7RKudB+r4tz8uY/v+OkZkpTAyK5WR\n2SmMyvbfbvb6+MN7u3imZDe1jS2Mz0vn5vOGcsWEXBLjOp8h25G6xhZ+8rf1/PH9Xcwem80DN0zp\ncLcr6X262ueucBcJgWP/3XV0RV7X2MKC5Xv4/bs72Fx5hH7J8dw4PZ+b/mUIWand77L5cMdB7np6\nJbsP1TNrdBZvrK9kTqE/4OPa2HhEeieFu0iEcM7x7tYD/PadHby5oYI4j4crJw7k1guGMXZgWqev\nb2j2ct/CTTwS2B7wF5+exLnD+vHbd7bzXy+v47JxOfzys5NPK+B3HajnmdLdjM5JY05hdpu7U0lw\nKdxFItD2/XU88c52ni4p42izl/PO6c+tFwyjuCCrzSGWa/ZUc+fTK9hUcYTPnpvP9y4fQ0qrjUYe\nW7qd/35lHZ8Yn8P9n+l6wO/YX8cDi7bw/PI9x5dsyMvow83nDeH6afmk94kLzgnLKRTuIhGsur6Z\npz7cxRPv7GBfTQPg3zA81iy6JzIAAAa/SURBVGPExXiIjfHfPlTfTP/keO6ZO4Higqw2j/Xo29v4\n0V/Xc/mEXO6/flKHV9/b99fxwD+28MKKPcR6jBum5zPvouGsLqvmsaXbeX/7QZLiY/j01EF84fxh\nDBuQ3O6x5PQo3EWiQLPXx2tr9rG58ghen48Wr6PZ62jx+Wj2OtL7xHHbjOFkJHU8bv83S7bx47+t\n58qJA7nvuomYGQfqGqmsaaSqtpGKmgbe336QF1fsOT4X4MsXDT9luOaaPdU8/s52Xl5ZTovPMW1o\nP6YO6cvkwRlMys84re8KQsE5x6H6ZqpqG6msbaCqtpGjzV6aW3y0+BxNXh/NLf5/zgYkxsfQJy7w\nEx9DYpz/J9ZjxLT6OXY/MyXhtIe6KtxFpFv+b/FW/ufVDWQkxVHb0HLKnrSJcR4+N30I82YM7zSk\nK2sb+MN7u1i8sZK15TW0tOq6mZyfwYRB6eSk9yErNYHM1ASyUhNISYjFzHDOcbCuiV0H69l1sJ7d\ngd/VR5uJ8RhmhseMGAOPGR6P4TEwDI+HwPP++/VNXmobmqltaKEm8Lu2oZlmryM+1kN8jIf4WA8J\nsf7fZnDgSBP7jzTS7O3K7H3/7+7G6G0zzuE7l43u3ouOv6fCXUS66ZmS3Xy44yBZqYlkpflDNyst\n0f87NfG0hk02NHtZW17N8l2HAz+HKK9uOKVdn7gY+qfEc6iuibqmE9fWz0pNICMpDufA5xy+wG+v\nz+HzORz++/7n/VfePudIio8lNfHYTxxpgd9xMR6avF6aWnz+H6+PxmYfPufon5LQ6o9OIpmB28nx\nMce7vOJiPMTFeIjx+P8YNbb4aGj2crTZy9Em/++GZi8tXoc3UGdLoNYWn2No/2QKclJP69+Rwl1E\neq3q+mYqAt0dlbUNVNY0UlnbyIEjjWQkxZPfL4n8fkkM6Z/EoL5JXdoBK1p0NdzPePkBEZHuSk+K\nIz0pjlHZp3f1Kp3ToFQRkQikcBcRiUAKdxGRCKRwFxGJQAp3EZEIpHAXEYlACncRkQikcBcRiUAh\nm6FqZlXAztN8+QBgfxDLCTfRfP7RfO4Q3eevc/cb4pzL7OwFIQv3M2FmJV2Zfhupovn8o/ncIbrP\nX+fevXNXt4yISARSuIuIRKBwDfdHQl1AiEXz+UfzuUN0n7/OvRvCss9dREQ6Fq5X7iIi0gGFu4hI\nBAq7cDezS81so5ltMbPvhLqenmZmj5tZpZmtafVYPzNbaGabA7/7hrLGnmJmg81skZmtM7O1ZnZH\n4PGIP38zSzSzD8xsZeDc/yvw+DAzez/w+f+LmXW883UYM7MYM1tuZq8E7kfTue8ws9VmtsLMSgKP\ndetzH1bhbmYxwIPAZcBY4LNmNja0VfW4J4BLT3rsO8CbzrmRwJuB+5GoBbjLOTcW+BgwP/DvOxrO\nvxG42Dk3EZgEXGpmHwPuAe5zzo0ADgG3hrDGnnYHsL7V/Wg6d4Bi59ykVuPbu/W5D6twB84Ftjjn\ntjnnmoA/A1eHuKYe5ZxbAhw86eGrgd8Fbv8OuOasFnWWOOf2OueWBW7X4v8PPY8oOH/ndyRwNy7w\n44CLgWcDj0fkuQOY2SDgcuDRwH0jSs69A9363IdbuOcBu1vdLws8Fm2ynXN7A7f3AdmhLOZsMLOh\nwGTgfaLk/APdEiuASmAhsBU47JxrCTSJ5M///wLfAnyB+/2JnnMH/x/y182s1MzmBR7r1udeG2SH\nOeecM7OIHs9qZinAc8DXnXM1/os4v0g+f+ecF5hkZhnA88DoEJd0VpjZFUClc67UzGaGup4QucA5\nt8fMsoCFZrah9ZNd+dyH25X7HmBwq/uDAo9FmwozywUI/K4McT09xszi8Af7H51zCwIPR835Azjn\nDgOLgH8BMszs2EVZpH7+zweuMrMd+LteLwbuJzrOHQDn3J7A70r8f9jPpZuf+3AL9w+BkYFvzeOB\nzwAvhbimUHgJuDlw+2bgxRDW0mMC/ayPAeudc/e2eiriz9/MMgNX7JhZH+AS/N85LALmBppF5Lk7\n5/6fc26Qc24o/v/G/+Gcu5EoOHcAM0s2s9Rjt4HZwBq6+bkPuxmqZvYJ/P1xMcDjzrkfh7ikHmVm\nTwEz8S/5WQHcDbwAPA3k4182+Trn3MlfuoY9M7sAeBtYzUd9r9/F3+8e0edvZhPwf2kWg/8i7Gnn\n3A/NbDj+q9l+wHLgc865xtBV2rMC3TLfcM5dES3nHjjP5wN3Y4E/Oed+bGb96cbnPuzCXUREOhdu\n3TIiItIFCncRkQikcBcRiUAKdxGRCKRwFxGJQAp3EZEIpHAXEYlA/x/2br88n4X6+gAAAABJRU5E\nrkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iUxuR9Kz02QE",
        "colab_type": "text"
      },
      "source": [
        "## Hyperparam stuff"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jhJtU4X004a2",
        "colab_type": "code",
        "outputId": "0a5f056f-b8a1-4edd-c0e3-cbff5f28caf9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 561
        }
      },
      "source": [
        "# lr_finder\n",
        "lr_list = [1e-7, 1e-6, 1e-5, 1e-4, 1e-3]\n",
        "ttrain_loader = get_loader(\"train\")\n",
        "vval_loader = get_loader(\"val\")\n",
        "val_losses = []\n",
        "train_losses = []\n",
        "FRESH = True\n",
        "SANITY = True\n",
        "SANITYSIZE = 1100\n",
        "SANITYEPOCH = 10\n",
        "if FRESH:\n",
        "    mmodel = FFNN()\n",
        "    mmodel.save()\n",
        "for lr in lr_list:\n",
        "    LEARNING_RATE=lr\n",
        "    val, trains = train(mmodel, ttrain_loader, vval_loader)\n",
        "    val_losses.extend(val)\n",
        "    train_losses.extend(trains)\n",
        "plt.plot(val_losses)\n",
        "plt.plot(train_losses)\n",
        "plt.xlabel(\"Learning rate\")\n",
        "plt.ylabel(\"Loss\")\n",
        "#plt.xticks(range(0,SANITYEPOCH*len(lr_list),10), map(str, lr_list))\n",
        "        \n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "train Dataset Size: 667411\n",
            "val Dataset Size: 675836\n",
            "Begin training\n",
            "Epoch:  1\n",
            "Learning Rate:  1e-07\n",
            "Training loss:  4.924452876934976\n",
            "Validation loss:  4.924017782644792\n",
            "Saving Model....\n",
            "Time :36.21047854423523s\n",
            "Epoch:  2\n",
            "Learning Rate:  1e-07\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-66-1093e14878a2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mlr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlr_list\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0mLEARNING_RATE\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m     \u001b[0mval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrains\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mttrain_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvval_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m     \u001b[0mval_losses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m     \u001b[0mtrain_losses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrains\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-60-4240881e2aff>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(model, train_dataloader, val_dataloader)\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Learning Rate: \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m         \u001b[0mtrain_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrun_one_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_dataloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m         \u001b[0mtrain_losses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_loss\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Training loss: \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_loss\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-57-c9087e095f9c>\u001b[0m in \u001b[0;36mrun_one_epoch\u001b[0;34m(model, loader, criterion, update, optimizer)\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mupdate\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m             \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m             \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m             \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0mplotloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    101\u001b[0m                     \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mmax_exp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias_correction2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m                     \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias_correction2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    104\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m                 \u001b[0mstep_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lr'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mbias_correction1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hezn5ZEJ-Hxm",
        "colab_type": "text"
      },
      "source": [
        "## Prediction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D_jZohZA-Jh6",
        "colab_type": "code",
        "outputId": "06824221-0454-4d45-8ecc-cfe7e5243c85",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "model = FFNN.load()\n",
        "test = get_loader(\"test\")\n",
        "results = get_predictions(model, test)\n",
        "tmp = list(zip(range(len(results)), results))\n",
        "with open('predt680v639.csv', 'w') as f:\n",
        "    writer = csv.writer(f)\n",
        "    writer.writerow(['id','label'])\n",
        "    writer.writerows(tmp)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "test Dataset Size: 223592\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8GCB-ci-3eNc",
        "colab_type": "text"
      },
      "source": [
        "# Testing MNIST\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bPtVEhOjhi1g",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mnist_train = np.load(\"../data/train.npy\")\n",
        "mnist_train_labels = np.load(\"../data/train_labels.npy\")\n",
        "mnist_dev = np.load(\"../data/dev.npy\")\n",
        "mnist_dev_labels = np.load(\"../data/dev_labels.npy\")\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8wQfGnyjh4U_",
        "colab_type": "code",
        "outputId": "fa870e29-5823-4599-ba3c-77255dcc1ddc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "print(mnist_train.shape,mnist_train_labels.shape, mnist_dev.shape,mnist_dev_labels.shape)\n",
        "\n",
        "class MNIST_dataset(Dataset):\n",
        "  def __init__(self, X, Y):\n",
        "    self.X = torch.from_numpy(X).float()\n",
        "    self.Y = torch.from_numpy(Y).long()\n",
        "  \n",
        "  def __len__(self):\n",
        "    return len(self.Y)\n",
        "\n",
        "  def __getitem__(self, index):\n",
        "    return self.X[index], self.Y[index]\n",
        "\n",
        "mnist_train_data = MNIST_dataset(mnist_train, mnist_train_labels)\n",
        "mnist_dev_data = MNIST_dataset(mnist_dev, mnist_dev_labels)\n",
        "mnist_train_dataloader = DataLoader(mnist_train_data,batch_size=BATCH_SIZE,shuffle=True)\n",
        "mnist_dev_dataloader = DataLoader(mnist_dev_data,batch_size=BATCH_SIZE,shuffle=False)\n",
        "\n",
        "model = FFNN()\n",
        "val_losses, train_losses = train(model, mnist_train_dataloader, mnist_dev_dataloader)\n",
        "plt.plot(val_losses)\n",
        "plt.plot(train_losses)\n",
        "plt.title(\"Validation Loss\");\n",
        "plt.xlabel(\"Iterations\");\n",
        "plt.ylabel(\"Loss\");\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(50000, 784) (50000,) (10000, 784) (10000,)\n",
            "Begin training\n",
            "Epoch:  1\n",
            "Learning rate: [3.9999999999999956e-07]\n",
            "Training Epoch Accuracy: 27.014%\n",
            "Training loss:  2.209235254760898\n",
            "Validation Epoch Accuracy: 64.78%\n",
            "Validation loss:  1.6154187560840776\n",
            "Saving Model....\n",
            "Time :3.879451274871826s\n",
            "Epoch:  2\n",
            "Learning rate: [2.801856579744565e-06]\n",
            "Training Epoch Accuracy: 84.382%\n",
            "Training loss:  0.5975573842539964\n",
            "Validation Epoch Accuracy: 91.45%\n",
            "Validation loss:  0.2898275336357439\n",
            "Saving Model....\n",
            "Time :3.834383726119995s\n",
            "Epoch:  3\n",
            "Learning rate: [7.6037117232856525e-06]\n",
            "Training Epoch Accuracy: 92.376%\n",
            "Training loss:  0.26428379548137143\n",
            "Validation Epoch Accuracy: 93.82%\n",
            "Validation loss:  0.21017339728345538\n",
            "Saving Model....\n",
            "Time :3.778704881668091s\n",
            "Epoch:  4\n",
            "Learning rate: [9.999999176566715e-06]\n",
            "Training Epoch Accuracy: 94.254%\n",
            "Training loss:  0.19390377388966967\n",
            "Validation Epoch Accuracy: 95.51%\n",
            "Validation loss:  0.1572481741903314\n",
            "Saving Model....\n",
            "Time :3.807025194168091s\n",
            "Epoch:  5\n",
            "Learning rate: [9.503600528520866e-06]\n",
            "Training Epoch Accuracy: 95.4%\n",
            "Training loss:  0.15484527100230117\n",
            "Validation Epoch Accuracy: 95.91%\n",
            "Validation loss:  0.1399880231233539\n",
            "Saving Model....\n",
            "Time :3.8118104934692383s\n",
            "Epoch:  6\n",
            "Learning rate: [8.115212524010932e-06]\n",
            "Training Epoch Accuracy: 96.188%\n",
            "Training loss:  0.1293935276891874\n",
            "Validation Epoch Accuracy: 96.42%\n",
            "Validation loss:  0.12560607509533311\n",
            "Saving Model....\n",
            "Time :3.8244893550872803s\n",
            "Epoch:  7\n",
            "Learning rate: [6.109822434791551e-06]\n",
            "Training Epoch Accuracy: 96.682%\n",
            "Training loss:  0.11263364653967683\n",
            "Validation Epoch Accuracy: 96.58%\n",
            "Validation loss:  0.12110496824903852\n",
            "Saving Model....\n",
            "Time :3.910836696624756s\n",
            "Epoch:  8\n",
            "Learning rate: [3.8846223625280535e-06]\n",
            "Training Epoch Accuracy: 97.038%\n",
            "Training loss:  0.10076275858504083\n",
            "Validation Epoch Accuracy: 96.65%\n",
            "Validation loss:  0.11581181718190764\n",
            "Saving Model....\n",
            "Time :3.8230111598968506s\n",
            "Epoch:  9\n",
            "Learning rate: [1.880340471820184e-06]\n",
            "Training Epoch Accuracy: 97.246%\n",
            "Training loss:  0.09399579521125692\n",
            "Validation Epoch Accuracy: 96.8%\n",
            "Validation loss:  0.11414919012955799\n",
            "Saving Model....\n",
            "Time :3.7279415130615234s\n",
            "Epoch:  10\n",
            "Learning rate: [4.939493720269017e-07]\n",
            "Training Epoch Accuracy: 97.348%\n",
            "Training loss:  0.09097330108084871\n",
            "Validation Epoch Accuracy: 96.76%\n",
            "Validation loss:  0.11401999423837965\n",
            "Saving Model....\n",
            "Time :3.7899069786071777s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de3Qcd3338fd3V3dZlmxJvq9jO5aT\nJuTixJYTEhkoPJRASihQIH3CJQUCLW1K75TneXhantNCC4cWSk97QrhDAzQBGmgg0FMgDk1CbCd2\n4jgkjh3b8iWWfNVd2t3v88eMpLWiy9rS7K40n9c5e3Z3Znbm67Wtj2Z+M98xd0dEROIrUewCRESk\nuBQEIiIxpyAQEYk5BYGISMwpCEREYk5BICIScwoCmZPMbJWZuZmVhe9/YGbvzGfZ89jWh83szunU\nK1JMCgIpSWb2QzP76DjTbzKzo+f6Q9vdb3D3L89AXS83s/Yx6/4bd3/PdNc9zrbeZWYPzvR6RcZS\nEEip+jJwi5nZmOlvB77u7uki1CQyJykIpFR9F2gE2oYnmNkC4EbgK+H715nZY2Z2xswOmtlfTrQy\nM/upmb0nfJ00s0+aWaeZ7QVeN2bZW81st5l1mdleM3tfOL0W+AGwzMy6w8cyM/tLM/tazudfb2a7\nzOxUuN1fyZn3vJn9iZntNLPTZvZNM6s61y8n3O69ZnbCzPaY2Xtz5rWa2dbwe3nBzD4VTq8ys6+Z\n2fGwtkfNbPG5blvmHgWBlCR37wO+BbwjZ/JbgKfdfUf4viec30Dww/x3zOwNeaz+vQSBsh7YALx5\nzPxj4fz5wK3A35vZVe7eA9wAHHb3eeHjcO4HzWwdcBfwQaAZuA/4nplVjPlzvAZYDVwOvCuPmsf6\nBtAOLAvr/xsz+9Vw3qeBT7v7fOBCgu8R4J1APZAiCNn3A33nsW2ZYxQEUsq+DLw55zfmd4TTAHD3\nn7r7E+6edfedBD+AX5bHet8C/IO7H3T3E8DHcme6+3+4+3Me+BnwI3L2TKbwVuA/3P3H7j4EfBKo\nBl6as8xn3P1wuO3vAVfmuW4AzCwFXAf8ubv3u/vjwJ2MhuYQsNbMmty9290fzpneCKx194y7b3P3\nM+eybZmbFARSstz9QaATeIOZXQi0Av86PN/MNpnZT8ysw8xOE/yG25THqpcBB3Pe78+daWY3mNnD\n4WGXU8Br81zv8LpH1ufu2XBby3OWOZrzuheYl+e6c7dxwt27cqbtz9nGu4F1wNPh4Z8bw+lfBe4H\nvmFmh83s78ys/By3LXOQgkBK3VcIftO9Bbjf3V/ImfevwL1Ayt3rgX8Bxg4uj+cIweGRYSuHX5hZ\nJXAPwW/yi929geDwzvB6p2rXexi4IGd9Fm7rUB515eswsNDM6nKmrRzehrs/6+43A4uAvwXuNrNa\ndx9y979y90sI9lBu5OxDbxJTCgIpdV8BXkVwXH/s6Z91BL8Z95tZK/Bbea7zW8DtZrYiHID+UM68\nCqAS6ADSZnYD8Oqc+S8AjWZWP8m6X2dmrwx/2/5jYAD47zxrG8vCQd6Rh7sfDNf3sXDa5QR7AV8L\nP3CLmTWHeyOnwvVkzewVZnaZmSWBMwSHirLnWZfMIQoCKWnu/jzBD71agt/+c/0u8FEz6wI+wuig\n6FQ+R3CIZAewHfh2zva6gNvDdZ0kCJd7c+Y/TTAWsTc882bZmHp/SbD38o8Eh7V+Hfh1dx/Ms7ax\nXkowoDvyCK+huBlYRbB38B3g/7r7f4afeQ2wy8y6CQaO3xYOvi8B7iYIgd3AzwgOF0nMmW5MIyIS\nb9ojEBGJOQWBiEjMKQhERGJOQSAiEnPn1Xa3mJqamnzVqlXFLkNEZFbZtm1bp7s3jzdv1gXBqlWr\n2Lp1a7HLEBGZVcxs/0TzdGhIRCTmFAQiIjGnIBARiTkFgYhIzCkIRERiTkEgIhJzCgIRkZiLTxC8\n8BTc/79gsLfYlYiIlJT4BMHpg/DQZ+HwY8WuRESkpMQnCFZsDJ4PPlLcOkRESkx8gqBmITS2QPuj\nxa5ERKSkxCcIAFKtwR6B7somIjIifkHQexxO7C12JSIiJSNmQbApeD74i+LWISJSQuIVBE0XQWU9\ntCsIRESGxSsIEglYsUF7BCIiOeIVBBCME7ywC/rPFLsSEZGSEM8gwOHQtmJXIiJSEuIXBMs3AKbD\nQyIiofgFQdV8WHSJBoxFRELxCwKA1EY4+Chks8WuRESk6GIaBJtg4DR0PlPsSkREii6+QQBqQCci\nQlyDYOEaqGnUgLGICHENAjNY0aoBYxERIgwCM0uZ2U/M7Ckz22VmfzDOMmZmnzGzPWa208yuiqqe\nF0ltDMYIek8UbJMiIqUoyj2CNPDH7n4JcA3wATO7ZMwyNwAt4eM24J8jrOdsw+ME7VsLtkkRkVIU\nWRC4+xF33x6+7gJ2A8vHLHYT8BUPPAw0mNnSqGo6y7L1YEkNGItI7BVkjMDMVgHrgbE/dZcDB3Pe\nt/PisMDMbjOzrWa2taOjY2aKqqiFJZdpnEBEYi/yIDCzecA9wAfd/bw6vbn7He6+wd03NDc3z1xx\nqVZo3waZ9MytU0Rklok0CMysnCAEvu7u3x5nkUNAKuf9inBaYaQ2wVAPHNtVsE2KiJSaKM8aMuDz\nwG53/9QEi90LvCM8e+ga4LS7H4mqphdJtQbPup5ARGKsLMJ1Xwe8HXjCzB4Pp30YWAng7v8C3Ae8\nFtgD9AK3RljPi9WnYN6SIAha31vQTYuIlIrIgsDdHwRsimUc+EBUNUzJLBwn0B6BiMRXPK8szpVq\nhZPPQ/exYlciIlIUCoKRBnTaKxCReFIQLL0CkhW6sExEYktBUFYJS6+E9keLXYmISFEoCCAYJzi0\nHdKDxa5ERKTgFAQQBEFmAI7uLHYlIiIFpyCA4N4EoAFjEYklBQHA/KVQv1IDxiISSwqCYalWDRiL\nSCwpCIalWuHMITjdXuxKREQKSkEwTA3oRCSmFATDFr8EyqoVBCISOwqCYclyWH61BoxFJHYUBLlS\nrcG1BEN9xa5ERKRgFAS5Uq2QTcPhx4pdiYhIwSgIcunCMhGJIQVBrtpGWHihgkBEYkVBMFZqU3DH\nMvdiVyIiUhAKgrFSG6GnA07uK3YlIiIFoSAYS3csE5GYURCM1XwxVNQpCEQkNhQEYyWSsGKDgkBE\nYkNBMJ7UJji2Cwa6il2JiEjkFATjSW0Ez8KhbcWuREQkcgqC8SzfABgc1P0JRGTuUxCMp7ohGDRW\nAzoRiQEFwURSrcGFZdlssSsREYmUgmAiqVboPw3Hny12JSIikYpNELg7+4/34Pm2jhi5sEyHh0Rk\nbotNENyz/RAv+8RP2dvZk98HGtdC9QJdTyAic15sgqB11UIAtjzTkd8HzIK21AoCEZnjYhMEKxtr\nuKCxhi3Pdub/oVQrdP4S+k5GV5iISJHFJggA2lqaeGjvcQbTeZ4JlApvVNO+NbqiRESKLGZB0Ezv\nYIbtB/L8DX/ZVWBJDRiLyJwWqyC49sJGkgljy7N5jhNUzoPFl2qcQETmtFgFwfyqctanGs5xnGBT\n0HMom4muMBGRIopVEEBweOiJQ6c50TOY3wdSrTDYDceeirYwEZEiiSwIzOwLZnbMzJ6cYP7Lzey0\nmT0ePj4SVS252tY14Q4/35PnXsHwgLHGCURkjopyj+BLwGumWGaLu18ZPj4aYS0jLl9ez/yqsvzH\nCRougNpF6kQqInNWZEHg7g8AJ6Ja//kqSya4bm0TW57tzK/dhFmwV6A9AhGZo4o9RnCtme0wsx+Y\n2aUTLWRmt5nZVjPb2tGR52/yk2hraebI6X6e6+jO7wOpTXByH3RPf9siIqWmmEGwHbjA3a8A/hH4\n7kQLuvsd7r7B3Tc0NzdPe8NtLU0APPDMOY4TtOs0UhGZe4oWBO5+xt27w9f3AeVm1lSIbacW1rC6\nqTb/cYKlV0KiXIeHRGROKloQmNkSM7PwdWtYy/FCbb+tpYmH955gIJ3H9QHlVbD0Cg0Yi8icFOXp\no3cBDwEXmVm7mb3bzN5vZu8PF3kz8KSZ7QA+A7zN875ZwPS1tTTTN5Rh2/48202kNsHh7ZDO8/oD\nEZFZoiyqFbv7zVPM/yzw2ai2P5Vr1iykLGFsebaTl16YxxGp1EZ4+J/ghSdg+dXRFygiUiDFPmuo\naOqqyrlq5YL8xwlWDF9YpgFjEZlbYhsEEIwTPHnoDMe7B6ZeuH451KcUBCIy58Q7CNYFp6I+mG+7\niRUbFQQiMufEOgguW15PfXV5/t1IU5vgTDucPhRtYSIiBRTrIEgmjOvXNvFgvu0mUhuDZ11YJiJz\nSKyDAIJxgqNn+tlzLI92E0suh7JqHR4SkTkl9kFw/XC7iXwODyXLYdl6BYGIzCmxD4IVC2pY03wO\n7SZSrXBkBwz1R1uYiEiBxD4IADa3NPPw3uP5tZtItUJ2CI48Hn1hIiIFoCAgGCfoH8qy7fk82k2s\n0B3LRGRuURAA16xppDxp+Y0TzGuGhWs0TiAic4aCAKitLDv3dhMHfwGF65EnIhIZBUFo87pmdh0+\nQ2c+7SZSrdBzDE4+H3ldIiJRUxCEhu9a9vN82k2M3LFM9ycQkdlPQRC6dFk9C2rK87t95aJLoGKe\nBoxFZE5QEISSCeO6tU1sebZj6nYTiWRwTwINGIvIHKAgyLG5pZljXQM880Ie7SZSm+CFJ2Egj2VF\nREqYgiDHcLuJvM4eSrWCZ+HQtoirEhGJloIgx7KGatYumpff9QQrNgTP6kQqIrNcXkFgZheaWWX4\n+uVmdruZNURbWnG0tTTxyN7j9A9N0W6iegE0X6xxAhGZ9fLdI7gHyJjZWuAOIAX8a2RVFdHmlmYG\n0lm25tVuYmNwCmk2G31hIiIRyTcIsu6eBn4D+Ed3/1NgaXRlFc+mNQspT1qe4wSboO8kHN8TfWEi\nIhHJNwiGzOxm4J3A98Np5dGUVFw1FWVsuGBhfuMEIxeW6fCQiMxe+QbBrcC1wF+7+z4zWw18Nbqy\niqttXRO7j5zhWNcU9xxobIGqBl1YJiKzWl5B4O5Pufvt7n6XmS0A6tz9byOurWg2tzQDebSbSCSC\ncYKDajUhIrNXvmcN/dTM5pvZQmA78Dkz+1S0pRXPJUvns7C2gi35tJtIbYKO3dB3KvrCREQikO+h\noXp3PwO8EfiKu28CXhVdWcWVSBjXr23igWc7p243kdoYPLdvjb4wEZEI5BsEZWa2FHgLo4PFc1pb\nSxOd3QM8fbRr8gWXXw2W0ICxiMxa+QbBR4H7gefc/VEzWwM8G11ZxdcWjhNMeRppZR0svlQDxiIy\na+U7WPxv7n65u/9O+H6vu78p2tKKa0l9FesWz2NLXu0mWqF9G2SnuBpZRKQE5TtYvMLMvmNmx8LH\nPWa2Iuriiq2tpZlH9p2Yut1EahMMdsGx3YUpTERkBuV7aOiLwL3AsvDxvXDanHZ9SxOD6Sy/2Hdi\n8gVHBow1TiAis0++QdDs7l9093T4+BLQHGFdJWHT6oVUJBNTjxMsWA21zWpAJyKzUr5BcNzMbjGz\nZPi4BTgeZWGloKaijA2rFkw9TmAWjBNowFhEZqF8g+C3CU4dPQocAd4MvCuimkpKW0szTx/t4tiZ\nKdpNpFrhxF7oyWNwWUSkhOR71tB+d3+9uze7+yJ3fwMwp88aGtY2cteyKX7ADzeg0+EhEZllpnOH\nsj+asSpK2CVL59NYWzH1OMGy9ZAo04CxiMw60wkCm7EqSlgiYVzf0sSDezrJZidpN1FeDUuv0B6B\niMw60wmCSZvwmNkXwmsOnpxgvpnZZ8xsj5ntNLOrplFLpNpamunsHmT30TOTL7iiFQ5th8xQYQoT\nEZkBkwaBmXWZ2ZlxHl0E1xNM5kvAayaZfwPQEj5uA/75HOouqHMaJ0j3wdEnClCViMjMmDQI3L3O\n3eeP86hz97IpPvsAMNmVWDcRdDJ1d38YaAgb25WcxfOruGhx3dTjBBowFpFZaDqHhqZrOXAw5317\nOO1FzOw2M9tqZls7OvK4l3AE2lqaeHTfSfoGJ2k3Ub8C5i/XgLGIzCrFDIK8ufsd7r7B3Tc0Nxfn\ngua2dc0MZrI8sm+K6+hWbNQegYjMKsUMgkNAKuf9inBaSWpdtZCKskQe4wSb4PRBOHO4MIWJiExT\nMYPgXuAd4dlD1wCn3f1IEeuZVHVFktZVC/MYJ9gUPGuvQERmiciCwMzuAh4CLjKzdjN7t5m938ze\nHy5yH7AX2AN8DvjdqGqZKW0tTTzzQjdHT0/SbmLJZVBWBe26ob2IzA6TnvkzHe5+8xTzHfhAVNuP\nQltLMx/7wdNsebaD39yQGn+hsorgKmM1oBORWWJWDBaXiouX1NE0r3LqcYIVG+Hw4zA0RaM6EZES\noCA4B4mE0ZZPu4nUJsgOwZEdhStOROQ8KQjOUVtLEyd6BnnqyCTtJkYuLNPhIREpfQqCc3T92qDd\nxAOTnT00bxEsWKULy0RkVlAQnKNF86u4eEkdW56ZapygNTiF1CftzSciUnQKgvOweV0zW/efoHcw\nPfFCqVbofgFOHShcYSIi50FBcB7aWpoYyjiP7J2kp54uLBORWUJBcB42rlpIZVli8nGCRZdAea3G\nCUSk5CkIzkNVeZLW1Qsnv54gWQYrrtaZQyJS8hQE52lzSzN7jnVz+FTfxAutaIWjT8JgT+EKExE5\nRwqC89S2LjiN9MHJ9gpSm8Azwe0rRURKlILgPF20uI7musrJxwlWbAiedXhIREqYguA8mY22m8hM\n1G6iZiE0rVMnUhEpaQqCadjc0syp3iF2HT498UK6sExESpyCYBquC9tNTHr2UKoV+k7A8ecKVJWI\nyLlREExDc10llyydzwPPTDJOMHJhmcYJRKQ0KQimqW1dE9sPnKR7YIJ2E03roKpeQSAiJUtBME2b\nW5rDdhPHx18gkQhuVKMBYxEpUQqCabr6ggVUlScmHydY0QrHdkP/JIPKIiJFoiCYpqryJJtWN05+\nPUGqFXBo31qwukRE8qUgmAFtLU3s7eih/WTv+AssvxowdSIVkZKkIJgBm9c1A5O0m6iaD4svVSdS\nESlJCoIZ0LJoHovnV7Jlz2TjBBuDQ0PZTOEKExHJg4JgBgTtJpr5+WTtJlKbYOAMdDxd2OJERKag\nIJghbS1NnOod4slDE5wZlGoNnjVOICIlRkEwQ64faTcxwdlDC9dATaOCQERKjoJghjTOq+Qly+fz\nwEQDxmbB4SENGItIiVEQzKC2lma275+k3cSKjXB8D/RMcBWyiEgRKAhmUFtLE+ms8/BzE/ygH25A\np3YTIlJCFAQz6OoLFlBdnpx4nGDZekiUqQGdiJQUBcEMqixLcs2ahRP3HaqogSWXacBYREqKgmCG\ntbU0s7ezh4MnJmg3kbomGDDet6WwhYmITEBBMMM2rwtOI31woquMr/9gcCrp194Eu79fwMpERMan\nIJhhFzbPY2l91cTjBHVL4NYfBIeIvvV2eOxrhS1QRGQMBcEMC9pNNPHgs5O0m6hZCO/4d1jzcvj3\nD8DPP13IEkVEzqIgiEBbSzNn+tPsbD818UKV8+Dmb8Klb4QffwR+9H/AJwgOEZEIlRW7gLnourVN\nmMGWZztZv3LBxAuWVcCb7gz2EP77M9B3Am78NCT11yIihaM9gggsrK3gsuX1E48T5Eok4bWfhJf9\neTBe8G/vhKH+6IsUEQlFGgRm9hoz+6WZ7TGzD40z/11m1mFmj4eP90RZTyG1tTSx/cApuvqHpl7Y\nDF7xYbjh7+Dp7wdnFOn+xiJSIJEFgZklgX8CbgAuAW42s0vGWfSb7n5l+LgzqnoKra2lmUzWeWii\ndhPj2fQ+eOOdcPBh+NKN0J3HHoWIyDRFuUfQCuxx973uPgh8A7gpwu2VlKtWLqCmIjnxVcYTufw3\n4eZvQOez8IVXw8n90RQoIhKKMgiWAwdz3reH08Z6k5ntNLO7zSw13orM7DYz22pmWzs6ZsdvyRVl\nCa5d05jfOMFYLf8jOL209zh84dfghadmvkARkVCxB4u/B6xy98uBHwNfHm8hd7/D3Te4+4bm5uaC\nFjgdbS1NPH+8lwPHJ2g3MZmVm+DWHwanlH7xBvUnEpHIRBkEh4Dc3/BXhNNGuPtxdx8I394JXB1h\nPQXXti4IrS17znMvZvEl8O77g9NLv3ITPPufM1idiEggyiB4FGgxs9VmVgG8Dbg3dwEzW5rz9vXA\n7gjrKbg1TbUsb6hmyzPnOE6Qa8Eq+O37oXEt3PVWeOLuGatPRAQiDAJ3TwO/B9xP8AP+W+6+y8w+\namavDxe73cx2mdkO4HbgXVHVUwzD7SZ+/lwn6Uz2/Fc0bxG86/tB59J73gOP3DFzRYpI7EU6RuDu\n97n7One/0N3/Opz2EXe/N3z9F+5+qbtf4e6vcPeno6ynGNpamunqT7OjfZrXBVTVwy33wEWvhR/8\nKfzkY2pJISIzotiDxXPedWsbw3YTM3C2U3kVvOUrcOUt8LOPw31/Atlp7GmIiKAgiFxDTQWXr2g4\n9+sJJpIsg5s+Cy+9HR69E+55N6QHZ2bdIhJLCoIC2NzSxOMHT3G6L492E/kwg1f/P3jVX8GubweD\nyIM9M7NuEYkdBUEBnFe7iXxc/0F4/Wdh70/hy6+H3hMzu34RiQUFQQGsX9lAbUVyZsYJxrrq7fCW\nr8LRJ4ILz04fmvozIiI5FAQFUJ5McO2FTTM3TjDWr9wIt9wdhMAXfg0690SzHRGZkxQEBdLW0sSB\nE73sPx7RsfzVm4NrDYb6gjA4/Fg02xGROUdBUCBtLU0APBDVXgHAsiuDq5DLa+BLvw77HohuWyIy\nZygICmT1SLuJiLunNq0N+hPVLw9ucPPUvVN/RkRiTUFQIGbG5nVNPPTccYam024iH/OXwa0/gKVX\nBLe+3DZuU1cREUBBUFBtLc10DaTZcfBU9BurWRjc0+DCX4Xv3Q4P/r1aUojIuBQEBfTSCxtJWMTj\nBLkqauFtd8FL3gT/+Zfwo/+tMBCRF1EQFNBwu4kfP/UCB4734oX4oVxWEdwHeeN74aHPwnd/FzLp\n6LcrIrNGWbELiJtXX7qYv/vhL9n8iZ/QWFvB+pUNrF+5gPUrG7hiRQO1lRH8lSQS8NpPQG0T/PRj\n0H8K3vwFKK+e+W2JyKxjBfmtdAZt2LDBt27dWuwyzpu789SRMzx24FT4OMnezuDagoTBusV1XHXB\nAtangoBY01RLImEzV8AvPgf3/Slc8FK4+a6gvbWIzHlmts3dN4w7T0FQfCd7Bnm8fTQYHj9wiq6B\n4PBNfXU5V6YaRvYcrkw1UF9dPr0NPnE3fOd9UF4LSy8Pzi4afjSuhURyBv5UIlJKFASzTDbrPNfR\nzWMHTrH9wEkeO3CKZ451jYzzrl00j/WphmDPYWUDLYvqSJ7rXsP+h2DnN+HIDnhhF2TCW0eX18CS\ny2BJTkA0XxyMNYjIrKUgmAO6+ofY2X6a7ftP8tjBYM/hZG/Q1rq2IskVqQauCscarkw10DivMv+V\nZ4ag85kgFI7sgCM74ehOGOwO5icrYNGvjAbDkitg8aVQURPBn1REoqAgmIPcnf3He0f2GB47eJLd\nR7rIZIO/z1WNNSOD0OtTC7h4aR3lyXM4SSybhRN74eiOnIDYAX0ng/mWgKaLwnAI9x6WXKYxB5ES\npSCIib7BDDvbT43sMWw/cIqOruCQT1V5gsuXj441XLWygUXzq85tA+5wun00FI7uDJ67jowus3DN\n2YeVll4RnK0kIkWlIIgpd+fQqb7RM5QOnmTXoTMMhi0umusqWd1Yy+qmWlY11bK6qYbVTfO4oLGG\nqvJzGDDuemE0FIYfp/aPzp+/POewUhgS85cFd1oTkYJQEMiIgXSGXYfPsH3/SX55tIvnj/ewr7OX\nzu6BkWXMYFl9NauaalgVBsXwI7WwJr9DTH0ng5vl5I47dD4DhP/eappGDys1tsD8pUFg1C2FqvnR\n/OFFYkxBIFPq6h/i+c5e9nZ283xnL88f72FvZw/Pd/acda/lZMJYsaA62IsYExLLGqonP3tpoDs4\nQ+nIjtGxh2O7ITvmSueKujAYlo2Gw/xlo4+6ZVDTGFwoJyJ5URDItJzsGRwJhdyA2NfZQ+9gZmS5\nimSClY3BXsSa5rODYvH8Smy8Q0HpAThzOHh0HYEzh+DM8HM4resoeObszyUrzg6IunCPInfPom4J\nJKd5zYXIHKEgkEi4Ox1dA+wLQ2Hf8dGAeP54L4Pp0Xbb1eXJkXGI3IBY1VRLY23F+CExLJuB7mNh\nMIShMRIYOdPS/WM+aDBv8dnhkLtnMTxNp8FKDCgIpOCyWefw6T6e7+xl3/Ee9nUEexPPd/Zw4EQv\n6ezovzszqK0oo7YySW1FGTWVSWoqyqitSFJbWTYybfh5XmXZyPyaynC5iiR12S5qh45R03eMit6j\n2EhoHB4NjP7TLy62qiEIhppGqF6Q36O8WoPdMqsoCKSkDGWyHDrZN7IncbJ3kJ6BDL2DaXoGM/QO\npOkeSNM7mKFnME3vQPDcM5Amm+c/10QYLjVjwmVh+RDLEidZwgmaOU5j9jgL0h3UpzupTp+maug0\nFUNnKB88RSI7NPEGkpUThETD5AFSWacAkaKYLAjUfVQKrjyZYFV4WOgV5/A5d2cgnaUnJyR6BjLh\n+/RZYdIzkH5RuPQMpjnUk+CZwXp6B+bRM7hsknBxqhmggR4arJsG66Y+fL3QemjyHhb29LKgr5sG\nupjPEeZ7F/O8m0ofe4hqVNaSDJXXk66sJ1PZQLYqCA+rXkCidiHJmgVYZR1WUUuispZE1TwSFfNI\nVNUF95eoqA3agKgflMwgBYHMGmZGVXmSqvIkjTO0ztxw6RvK0D+UoXcwQ99gZuR931CGvsEsvYPp\nkfftg1meHVk+Td9Qlv7wM+nBPsoHgz2LqqHTVGfO0GDdzKeHBuuhYaibhr7hYNlHA900WA911pt3\n3X1eQZ9V0UsVfVTRZ9X0U0m/VdNvVfQnqum3agYS1QxYVfCcqGYoUc1goprBZBVDyRoGE9WkkzUM\nJWvIJCpJJhMkEkZZwkgmjKQZZcngdVkiQeKs9xa8TxjJZLDs8PRkMhF8dng9iZx5Z71PTDi/LGEk\nEoaFf/fBMxgWPNvZ0xPha1vCLo0AAAgESURBVHKXIZweTsPOXjZ3fZy17JhtzvG9OAWBxFpuuERl\nOGyGw6V3cDRgTg1mODIcOP0DZHpPkUz3khjqIZHuJTnUSzLTS1m6l7JMD2XpvvB1L+WZvuCR7aUq\n3cf8bC8V2Q4qsv1UZPuozPZNuncyVoYE/VTSZ1X0U0U/FQxSTj8VDHg5feFzv5fTN/xMOQNeQffw\ncpTT78HzQM5nh+edPT94Dn7szi3nmhv5Lv6+l13In7/m4nOuZyoKApGI5YbNgkJvPJuBoV4Y7Bnn\n0X3W6+RQL7WDPdQOT08PBGdiDfUHz+l+SJ8Mpg/1QXoAT/djk42l5FNiooJsWRWZRCXZZCWZRCWZ\n4edEBW5JsokyshY+EmVkSY6+tjKyliRr5WQsmO6WJBMunwnnZ3KWy5IkfdYySTKErwmmZ0iSSSRJ\nE3wOErglcGzMcyLn2XC34JlE0JNrCucySrtx1cLz/p4noyAQmcsSyWCAurIuktUbBLc+zQyMCYz+\nMSEyAOm+s0JkeJlEup9EeoCykem58wcgOwSZ3uDCw/RQ8JwdCrabHQq652Yzo6/HXnNSbJaY4JEM\njzuNMy8xwbx574R1vzfjJSoIRGR6kmXBo6K22JUE3IOwyAzlBEZuaKQnmR8+j8xPj4aLZ3MeHjxn\nx07PnT/RvGzQ3XeieZ4ZXf/Yx7xFkXxlCgIRmVvMgivKdVV53tSsRUQk5hQEIiIxpyAQEYm5SIPA\nzF5jZr80sz1m9qFx5lea2TfD+Y+Y2aoo6xERkReLLAjMLAn8E3ADcAlws5ldMmaxdwMn3X0t8PfA\n30ZVj4iIjC/KPYJWYI+773X3QeAbwE1jlrkJ+HL4+m7glTbXr+UWESkxUQbBcuBgzvv2cNq4y7h7\nGjgNL24jY2a3mdlWM9va0dERUbkiIvE0KwaL3f0Od9/g7huam5uLXY6IyJwS5QVlh4BUzvsV4bTx\nlmk3szKgHjg+2Uq3bdvWaWb7z7OmJqDzPD87F+n7OJu+j1H6Ls42F76PCyaaEWUQPAq0mNlqgh/4\nbwN+a8wy9wLvBB4C3gz8l09xpxx3P+9dAjPbOtGNGeJI38fZ9H2M0ndxtrn+fUQWBO6eNrPfA+4H\nksAX3H2XmX0U2Oru9wKfB75qZnuAEwRhISIiBRRpryF3vw+4b8y0j+S87gd+M8oaRERkcrNisHgG\n3VHsAkqMvo+z6fsYpe/ibHP6+5h1N68XEZGZFbc9AhERGUNBICISc7EJgqka4MWJmaXM7Cdm9pSZ\n7TKzPyh2TcVmZkkze8zMvl/sWorNzBrM7G4ze9rMdpvZtcWuqVjM7A/D/yNPmtldZlZV7JqiEIsg\nyLMBXpykgT9290uAa4APxPz7APgDYHexiygRnwZ+6O4XA1cQ0+/FzJYDtwMb3P0lBKfBz8lT3GMR\nBOTXAC823P2Iu28PX3cR/Ecf2wcqNsxsBfA64M5i11JsZlYPbCa4xgd3H3T3U8WtqqjKgOqw80EN\ncLjI9UQiLkGQTwO8WArvAbEeeKS4lRTVPwB/BmSLXUgJWA10AF8MD5XdaWYlclf6wnL3Q8AngQPA\nEeC0u/+ouFVFIy5BIOMws3nAPcAH3f1MsespBjO7ETjm7tuKXUuJKAOuAv7Z3dcDPUAsx9TMbAHB\nkYPVwDKg1sxuKW5V0YhLEOTTAC9WzKycIAS+7u7fLnY9RXQd8Hoze57gkOGvmtnXiltSUbUD7e4+\nvId4N0EwxNGrgH3u3uHuQ8C3gZcWuaZIxCUIRhrgmVkFwYDPvUWuqWjCm/98Htjt7p8qdj3F5O5/\n4e4r3H0Vwb+L/3L3OflbXz7c/Shw0MwuCie9EniqiCUV0wHgGjOrCf/PvJI5OnAeaa+hUjFRA7wi\nl1VM1wFvB54ws8fDaR8Oe0OJ/D7w9fCXpr3ArUWupyjc/REzuxvYTnCm3WPM0VYTajEhIhJzcTk0\nJCIiE1AQiIjEnIJARCTmFAQiIjGnIBARiTkFgcSOmXWHz6vM7LdmeN0fHvP+v2dy/SJRUBBInK0C\nzikIwuZjkzkrCNx9Tl6JKnOLgkDi7ONAm5k9HvadT5rZJ8zsUTPbaWbvAzCzl5vZFjO7l/AqWzP7\nrpltC3vV3xZO+zhBp8rHzezr4bThvQ8L1/2kmT1hZm/NWfdPc/r/fz28ihUz+3h4z4idZvbJgn87\nEhuxuLJYZAIfAv7E3W8ECH+gn3b3jWZWCfzczIa7TV4FvMTd94Xvf9vdT5hZNfComd3j7h8ys99z\n9yvH2dYbgSsJ+vs3hZ95IJy3HriUoMXxz4HrzGw38BvAxe7uZtYw4396kZD2CERGvRp4R9h24xGg\nEWgJ5/0iJwQAbjezHcDDBA0NW5jc9cBd7p5x9xeAnwEbc9bd7u5Z4HGCQ1angX7g82b2RqB32n86\nkQkoCERGGfD77n5l+Fid03++Z2Qhs5cTdKa81t2vIOhBM51bGA7kvM4AZe6eJrih0t3AjcAPp7F+\nkUkpCCTOuoC6nPf3A78TtujGzNZNcFOWeuCku/ea2cUEt/scNjT8+TG2AG8NxyGaCe4C9ouJCgvv\nFVEfNgL8Q4JDSiKR0BiBxNlOIBMe4vkSwb16VwHbwwHbDuAN43zuh8D7w+P4vyQ4PDTsDmCnmW13\n9/+ZM/07wLXADsCBP3P3o2GQjKcO+PfwZukG/NH5/RFFpqbuoyIiMadDQyIiMacgEBGJOQWBiEjM\nKQhERGJOQSAiEnMKAhGRmFMQiIjE3P8HiwOkL9O7KXUAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}