train_PMF_project.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "train_PMF_project.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "iphBixbijuJN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "67e080a4-f562-49b8-f29b-c351a2d7f959"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "root_path = '/content/drive/MyDrive/CSC2547/project'\n",
        "\n",
        "import os\n",
        "os.chdir(root_path)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "T4oQhwIIV37u",
        "outputId": "a2a0db8f-3468-45e2-8be7-20f5d3be655a"
      },
      "source": [
        "from __future__ import print_function\n",
        "import numpy as np\n",
        "import pickle\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "from torch.autograd import Variable\n",
        "import torch.utils.data\n",
        "import matplotlib.pyplot as plt\n",
        " \n",
        "from model import PMF\n",
        "from evaluations import RMSE\n",
        " \n",
        "print('------------------------ Train PMF ---------------------------')\n",
        "# --------------------------------------------- HYPERPARAMETERS ----------------------------------------------------\n",
        "# Input batch size for training\n",
        "batch_size = 100000\n",
        "# Number of maximum epoches to train\n",
        "epoches = 150\n",
        "# Enables CUDA training\n",
        "no_cuda = False\n",
        "# Generate random seed\n",
        "seed = 1\n",
        "# Weight decay\n",
        "weight_decay = 0.1\n",
        "# Size of embedding features\n",
        "embedding_feature_size = 100\n",
        "# Training ratio\n",
        "ratio = 0.8\n",
        "# Learning rate\n",
        "lr = 0.0001\n",
        "# Momentum value\n",
        "momentum = 0.9\n",
        "# ------------------------------------------------------------------------------------------------------------------\n",
        " \n",
        "# Load datasets\n",
        "user = pickle.load(open('dataset/user_id_to_num.pkl', 'rb'))\n",
        "print(\"Loaded user\")\n",
        "rest = pickle.load(open('dataset/rest_id_to_num.pkl', 'rb'))\n",
        "print(\"Loaded rest\")\n",
        "data = np.load('dataset/data.npy')\n",
        "print(\"Loaded data\")\n",
        " \n",
        "# Normalize rewards to [-1, 1]\n",
        "data[:,1] = 0.5*(data[:,1] - 3)\n",
        " \n",
        "# Shuffle data\n",
        "np.random.shuffle(data)\n",
        " \n",
        "# Split data\n",
        "train_data = data[:int(ratio*data.shape[0])]\n",
        "vali_data = data[int(ratio*data.shape[0]):int((ratio+(1-ratio)/2)*data.shape[0])]\n",
        "test_data = data[int((ratio+(1-ratio)/2)*data.shape[0]):]\n",
        " \n",
        "# Extract number of users and items\n",
        "NUM_USERS = len(user)\n",
        "NUM_ITEMS = len(rest)\n",
        " \n",
        "# Get CUDA device if available\n",
        "cuda = torch.cuda.is_available()\n",
        " \n",
        "# Set device to CUDA or CPU, depending on availability and desire\n",
        "device = torch.device(\"cuda\" if cuda and no_cuda else \"cpu\")\n",
        " \n",
        "# Generate and apply seeds\n",
        "torch.manual_seed(seed=seed)\n",
        "if cuda:\n",
        "    torch.cuda.empty_cache()\n",
        "    torch.cuda.manual_seed(seed=seed)\n",
        " \n",
        "# Specify number of workers for cuda\n",
        "kwargs = {'num_workers':2, 'pin_memory':True} if cuda else {}\n",
        " \n",
        "# Construct Data Loaders\n",
        "train_data_loader = torch.utils.data.DataLoader(torch.from_numpy(train_data), batch_size=batch_size, shuffle=False, **kwargs)\n",
        "test_data_loader = torch.utils.data.DataLoader(torch.from_numpy(test_data), batch_size=batch_size, shuffle=False, **kwargs)\n",
        " \n",
        "# Initialize model\n",
        "model = PMF(n_users=NUM_USERS, n_items=NUM_ITEMS, n_factors=embedding_feature_size, no_cuda=no_cuda)\n",
        " \n",
        "# Move model to CUDA if CUDA selected\n",
        "if cuda:\n",
        "    model.cuda()\n",
        "    print(\"Model moved to CUDA\")\n",
        " \n",
        "# Set loss function\n",
        "loss_function = nn.MSELoss(reduction='sum')\n",
        " \n",
        "# Set optimizer (uncomment Adam for adam)\n",
        "optimizer = optim.SGD(model.parameters(), lr=lr, weight_decay=weight_decay, momentum=momentum)\n",
        "# optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)\n",
        " \n",
        "# Function for training one epoch\n",
        "def train(epoch, train_data_loader):\n",
        "    # Initialize\n",
        "    model.train()\n",
        "    epoch_loss = 0.0\n",
        "    optimizer.zero_grad()\n",
        " \n",
        "    # Go through batches\n",
        "    for batch_idx, ele in enumerate(train_data_loader):\n",
        "        # Zero optimizer gradient\n",
        "        optimizer.zero_grad()\n",
        " \n",
        "        # Extract user_id_nums: row, item_id_nums: col, ratings: val\n",
        "        row = ele[:, 2]\n",
        "        col = ele[:, 4]\n",
        "        val = ele[:, 1]\n",
        " \n",
        "        # Set to variables\n",
        "        row = Variable(row.long())\n",
        "        if isinstance(col, list):\n",
        "            col = tuple(Variable(c.long()) for c in col)\n",
        "        else:\n",
        "            col = Variable(col.long())\n",
        "        val = Variable(val.float())\n",
        " \n",
        "        # Move data to CUDA\n",
        "        if cuda:\n",
        "            row = row.cuda()\n",
        "            col = col.cuda()\n",
        "            val = val.cuda()\n",
        " \n",
        "        # Train\n",
        "        preds = model.forward(row, col)\n",
        "        loss = loss_function(preds, val)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        " \n",
        "        # Update epoch loss\n",
        "        epoch_loss += loss.data\n",
        " \n",
        "    epoch_loss /= train_data_loader.dataset.shape[0]\n",
        "    return epoch_loss\n",
        " \n",
        "# training model part\n",
        "print('------------------------------------------- Training Model------------------------------------------------')\n",
        "train_loss_list = []\n",
        "last_vali_rmse = None\n",
        "train_rmse_list = []\n",
        "vali_rmse_list = []\n",
        "print('parameters are: train ratio:{:f},batch_size:{:d}, epoches:{:d}, weight_decay:{:f}'.format(ratio, batch_size, epoches, weight_decay))\n",
        "print(model)\n",
        " \n",
        "# Go through epochs\n",
        "for epoch in range(1, epoches+1):\n",
        " \n",
        "    # Train epoch\n",
        "    train_epoch_loss = train(epoch, train_data_loader)\n",
        " \n",
        "    # Get epoch loss\n",
        "    train_loss_list.append(train_epoch_loss.cpu())\n",
        " \n",
        "    # Move validation data to CUDA\n",
        "    if cuda:\n",
        "        vali_row = Variable(torch.from_numpy(vali_data[:, 2]).long()).cuda()\n",
        "        vali_col = Variable(torch.from_numpy(vali_data[:, 4]).long()).cuda()\n",
        "    else:\n",
        "        vali_row = Variable(torch.from_numpy(vali_data[:, 2]).long())\n",
        "        vali_col = Variable(torch.from_numpy(vali_data[:, 4]).long())\n",
        " \n",
        "    # Get validation predictions\n",
        "    vali_preds = model.predict(vali_row, vali_col)\n",
        " \n",
        "    # Calculate train rmse loss\n",
        "    train_rmse = np.sqrt(train_epoch_loss.cpu())\n",
        " \n",
        "    # Calculate validation rmse loss\n",
        "    if cuda:\n",
        "        vali_rmse = RMSE(vali_preds.cpu().data.numpy(), vali_data[:, 1])\n",
        "    else:\n",
        "        vali_rmse = RMSE(vali_preds.data.numpy(), vali_data[:, 1])\n",
        " \n",
        "    # Add losses to rmse loss lists\n",
        "    train_rmse_list.append(train_rmse)\n",
        "    vali_rmse_list.append(vali_rmse)\n",
        " \n",
        "    print('Training epoch:{: d}, training rmse:{: .6f}, vali rmse:{:.6f}'. \\\n",
        "              format(epoch, train_rmse, vali_rmse))\n",
        " \n",
        "    # Early stop condition\n",
        "    if last_vali_rmse and last_vali_rmse < vali_rmse:\n",
        "        break\n",
        "    else:\n",
        "      last_vali_rmse = vali_rmse\n",
        " \n",
        "print('------------------------------------------- Testing Model------------------------------------------------')\n",
        " \n",
        "# Move test set to CUDA\n",
        "if cuda:\n",
        "    test_row = Variable(torch.from_numpy(test_data[:, 2]).long()).cuda()\n",
        "    test_col = Variable(torch.from_numpy(test_data[:, 4]).long()).cuda()\n",
        "else:\n",
        "    test_row = Variable(torch.from_numpy(test_data[:, 2]).long())\n",
        "    test_col = Variable(torch.from_numpy(test_data[:, 4]).long())\n",
        " \n",
        "# Get test predictions\n",
        "preds = model.predict(test_row, test_col)\n",
        " \n",
        "# Get test rmse loss\n",
        "if cuda:\n",
        "    test_rmse = RMSE(preds.cpu().data.numpy(), test_data[:, 1])\n",
        "else:\n",
        "    test_rmse = RMSE(preds.data.numpy(), test_data[:, 1])\n",
        "print('Test rmse: {:f}'.format(test_rmse))\n",
        " \n",
        "# Create plots\n",
        "plt.figure(1)\n",
        "plt.plot(range(1, len(train_rmse_list)+1), train_rmse_list, color='r', label='train rmse')\n",
        "plt.plot(range(1, len(vali_rmse_list)+1), vali_rmse_list, color='b', label='test rmse')\n",
        "plt.legend()\n",
        "plt.annotate(r'train=%f' % (train_rmse_list[-1]), xy=(len(train_rmse_list), train_rmse_list[-1]),\n",
        "             xycoords='data', xytext=(-30, 30), textcoords='offset points', fontsize=10,\n",
        "             arrowprops=dict(arrowstyle='->', connectionstyle='arc3, rad=.2'))\n",
        "plt.annotate(r'vali=%f' % (vali_rmse_list[-1]), xy=(len(vali_rmse_list), vali_rmse_list[-1]),\n",
        "             xycoords='data', xytext=(-30, 30), textcoords='offset points', fontsize=10,\n",
        "             arrowprops=dict(arrowstyle='->', connectionstyle='arc3, rad=.2'))\n",
        "plt.xlim([1, len(train_rmse_list)+10])\n",
        "plt.xlabel('iterations')\n",
        "plt.ylabel('RMSE')\n",
        "plt.title('RMSE Curve in Training Process')\n",
        "plt.show()\n",
        " \n",
        "# Save model\n",
        "path_to_trained_pmf = 'dataset/data_runs/ratio_{:f}_bs_{:d}_e_{:d}_wd_{:f}_lr_{:f}_trained_pmf.pt'.format(ratio, batch_size, len(train_rmse_list), weight_decay, lr)\n",
        "torch.save(model.state_dict(), path_to_trained_pmf)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "------------------------ Train PMF ---------------------------\n",
            "Loaded user\n",
            "Loaded rest\n",
            "Loaded data\n",
            "Model moved to CUDA\n",
            "------------------------------------------- Training Model------------------------------------------------\n",
            "parameters are: train ratio:0.800000,batch_size:100000, epoches:150, weight_decay:0.100000\n",
            "PMF(\n",
            "  (user_embeddings): Embedding(2189437, 100)\n",
            "  (item_embeddings): Embedding(50793, 100)\n",
            "  (ub): Embedding(2189437, 1)\n",
            "  (ib): Embedding(50793, 1)\n",
            ")\n",
            "Training epoch: 1, training rmse: 0.644812, vali rmse:0.631768\n",
            "Training epoch: 2, training rmse: 0.623630, vali rmse:0.621488\n",
            "Training epoch: 3, training rmse: 0.613595, vali rmse:0.615945\n",
            "Training epoch: 4, training rmse: 0.606919, vali rmse:0.612270\n",
            "Training epoch: 5, training rmse: 0.601816, vali rmse:0.609586\n",
            "Training epoch: 6, training rmse: 0.597635, vali rmse:0.607511\n",
            "Training epoch: 7, training rmse: 0.594060, vali rmse:0.605844\n",
            "Training epoch: 8, training rmse: 0.590913, vali rmse:0.604467\n",
            "Training epoch: 9, training rmse: 0.588086, vali rmse:0.603307\n",
            "Training epoch: 10, training rmse: 0.585505, vali rmse:0.602313\n",
            "Training epoch: 11, training rmse: 0.583121, vali rmse:0.601450\n",
            "Training epoch: 12, training rmse: 0.580896, vali rmse:0.600693\n",
            "Training epoch: 13, training rmse: 0.578804, vali rmse:0.600022\n",
            "Training epoch: 14, training rmse: 0.576823, vali rmse:0.599424\n",
            "Training epoch: 15, training rmse: 0.574938, vali rmse:0.598887\n",
            "Training epoch: 16, training rmse: 0.573134, vali rmse:0.598402\n",
            "Training epoch: 17, training rmse: 0.571401, vali rmse:0.597963\n",
            "Training epoch: 18, training rmse: 0.569729, vali rmse:0.597562\n",
            "Training epoch: 19, training rmse: 0.568112, vali rmse:0.597196\n",
            "Training epoch: 20, training rmse: 0.566543, vali rmse:0.596860\n",
            "Training epoch: 21, training rmse: 0.565016, vali rmse:0.596551\n",
            "Training epoch: 22, training rmse: 0.563527, vali rmse:0.596266\n",
            "Training epoch: 23, training rmse: 0.562071, vali rmse:0.596003\n",
            "Training epoch: 24, training rmse: 0.560644, vali rmse:0.595759\n",
            "Training epoch: 25, training rmse: 0.559244, vali rmse:0.595534\n",
            "Training epoch: 26, training rmse: 0.557866, vali rmse:0.595325\n",
            "Training epoch: 27, training rmse: 0.556509, vali rmse:0.595131\n",
            "Training epoch: 28, training rmse: 0.555170, vali rmse:0.594951\n",
            "Training epoch: 29, training rmse: 0.553847, vali rmse:0.594784\n",
            "Training epoch: 30, training rmse: 0.552537, vali rmse:0.594629\n",
            "Training epoch: 31, training rmse: 0.551239, vali rmse:0.594485\n",
            "Training epoch: 32, training rmse: 0.549951, vali rmse:0.594351\n",
            "Training epoch: 33, training rmse: 0.548672, vali rmse:0.594228\n",
            "Training epoch: 34, training rmse: 0.547400, vali rmse:0.594113\n",
            "Training epoch: 35, training rmse: 0.546134, vali rmse:0.594007\n",
            "Training epoch: 36, training rmse: 0.544873, vali rmse:0.593909\n",
            "Training epoch: 37, training rmse: 0.543616, vali rmse:0.593819\n",
            "Training epoch: 38, training rmse: 0.542362, vali rmse:0.593735\n",
            "Training epoch: 39, training rmse: 0.541111, vali rmse:0.593659\n",
            "Training epoch: 40, training rmse: 0.539862, vali rmse:0.593589\n",
            "Training epoch: 41, training rmse: 0.538614, vali rmse:0.593525\n",
            "Training epoch: 42, training rmse: 0.537368, vali rmse:0.593467\n",
            "Training epoch: 43, training rmse: 0.536122, vali rmse:0.593415\n",
            "Training epoch: 44, training rmse: 0.534877, vali rmse:0.593367\n",
            "Training epoch: 45, training rmse: 0.533631, vali rmse:0.593324\n",
            "Training epoch: 46, training rmse: 0.532386, vali rmse:0.593286\n",
            "Training epoch: 47, training rmse: 0.531140, vali rmse:0.593252\n",
            "Training epoch: 48, training rmse: 0.529893, vali rmse:0.593221\n",
            "Training epoch: 49, training rmse: 0.528645, vali rmse:0.593195\n",
            "Training epoch: 50, training rmse: 0.527396, vali rmse:0.593172\n",
            "Training epoch: 51, training rmse: 0.526145, vali rmse:0.593152\n",
            "Training epoch: 52, training rmse: 0.524892, vali rmse:0.593136\n",
            "Training epoch: 53, training rmse: 0.523637, vali rmse:0.593123\n",
            "Training epoch: 54, training rmse: 0.522380, vali rmse:0.593112\n",
            "Training epoch: 55, training rmse: 0.521121, vali rmse:0.593104\n",
            "Training epoch: 56, training rmse: 0.519858, vali rmse:0.593099\n",
            "Training epoch: 57, training rmse: 0.518593, vali rmse:0.593096\n",
            "Training epoch: 58, training rmse: 0.517324, vali rmse:0.593095\n",
            "Training epoch: 59, training rmse: 0.516052, vali rmse:0.593097\n",
            "------------------------------------------- Testing Model------------------------------------------------\n",
            "Test rmse: 0.592648\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3hUZfbA8e9JIPQmRQKhhA4JECA0sYIgiAKKiFhBEF0LVhTLoou66KIuYOMnisiuiogKrKAICoIUITQ19CoJICEUQWqS8/vjnZAhCSSBTCaZnM/z3Gfm3rnl3KBz5r5VVBVjjDEmvSB/B2CMMSZ/sgRhjDEmU5YgjDHGZMoShDHGmExZgjDGGJMpSxDGGGMyZQnCmHMQkZoickREggtCDPkhXhM4LEEUMiKyXUSOeb5E9ojIRBEp7fX5RBFREemZ7rh/e7b396yHiMjrIhLnOdd2ERl9luukLm+dI64GIvK5iOwTkUMi8ouIPObvLzpV/V1VS6tqck6OE5HbvO77mIikeP8tfBXD+cabHZ7/Nk567mG/iMwRkUa5fR2Tf1iCKJyuV9XSQBTQAng63ecbgTtTV0SkCHAzsMVrn6eBaKANUAa4EliZ2XW8lgczC0ZE6gI/AzuBpqpaDujjOX+ZnN6cJ16/UtWPU+8b6Abs8v5beO/r7ySYQ//yxB8G7AUmpt9BHPtuCQD2j1iIqeoeYDYuUXj7H3CpiFTwrHcFfgH2eO3TGvhKVXeps11VJ51nKP8AFqvqY6q62xPbBlW9VVUPisiVIhLnfYDnCeVqz/sXRGSqiPxXRP4EnvH8ar/Ia/8WnqeTop71u0VknYgcEJHZIlIrs8BEpLbnyamIZ32+iLwoIotE5LCIfCcilXJys55f4u+KyCwR+Qu4SkS6i8gqEflTRHaKyAvnE0NO4xWRO0Vkh4gkisjfvf+u56KqR4FPgEiv67wsIouAo0AdEblERJZ7ngiXi8glXte9SEQ+FJFdnn+DaV6fXSciq0XkoIgsFpFmXp89JSLxnnvZICKdPNvbiEiM5+/3h4i8kZN/E5M5SxCFmIiE4X7dbk730XFgOnCLZ/1OIP2X/1LgMRG5X0SaiohcQChXA1Mv4HiAnp5zlAdGAUuA3l6f3wpMVdVT4orPngFuBCoDC4FPc3CtW4EBQBUgBHjiPOK9FXgZ94T0E/AX7u9cHugO/E1EeuVSDJnuKyJNgHeA24BQoBxQPTvBiyuWvA1Y5bX5DmCw554OAzOBsUBF4A1gpohU9Oz7H6AkEOGJ69+e87YAJgD3eo77P2CGiBQTkYbAg0BrVS0DXANs95xvDDBGVcsCdYEp2bkPc26WIAqnaSJyGFeksxd4PpN9JgF3ikh54ApgWrrPRwKv4r4kYoB4Ebkrk+sc9FruOUs8FYHd53kvqZao6jRVTVHVY7hft/3AFXngkt0nnn3vA0aq6jpVTQL+CUSd7SkiEx+q6kbPdaaQ8QksO6ar6iJPvMdVdb6q/upZ/wWXsK7IpRjOtu9NwP9U9SdVPQkMB7IanO0JETmI+1FRGujv9dlEVY31/E27AJtU9T+qmqSqnwLrgetFJBT3w+Q+VT2gqqdU9UfPOQYD/6eqP6tqsqp+BJwA2gHJQDGgiYgU9Ty1phZ7ngLqiUglVT2iqkuzuA+TDZYgCqdenl9gVwKNgAxFJKr6E+7X9bPA154vF+/Pk1X1bVXtgPvV+zIwQUQap7tOea9l/FniScT9gr0QO9OtfwG093wZXQ6k4J4UAGoBY1ITF7AfELL565kzi9qO4r4oLyheEWkrIvNEJEFEDuGS2LmKrnISw9n2reYdh6fYKDGLuF/z/FtWVdUeXl/QcOY9VQN2pDt2B+5vXAPYr6oHMjl/LeBx7x8Wnv2rqepm4BHgBWCviEwWkWqe4wYCDYD1nuKs67K4D5MNliAKMc+vtonAa2fZ5b/A42QsXkp/nmOq+jZwAGhyHqHM5czioPT+whVHAKcrdSunDyNdTAeA74C+uCKWyZo2dPFO4N50yauEqi4+j9jPV/pf6p8AM4Aankr6cbik5Uu7cZXNAIhICdzT3PnyvqdduC97bzWBeNzf/yLP02l6O4GX0/3blPQ8gaCqn6jqpZ5zK+4pFlXdpKr9cMVVrwJTRaTUBdyLwRKEgdFAZxFpnslnY4HOwIL0H4jII+Iqj0uISBFP8VIZziyTzq7ngUtEZJSIVPWcv564SufyuFZVxT0VuUWB53BFDVn5BFeufxNpxUvgvnyfFpEIz7XKiUif84g7N5XB/ao+LiJtcEnN16biinwuEZEQ3C/z3EpKs4AGInKr57+PvrgfD197GiJ8A7wjIhVEpKiIXO45bjxwn+eJSkSklOffvYyINBSRjiJSDFdPdgz3ZIiI3C4ilVU1BTjoOVdKLt1LoWUJopBT1QTcE8LwTD7br6rfe/3y9nYUeB1XfLEPeADorapbvfb5n5zZD+Krs8SwBWgP1AZiPUUsX+DqNg6r6iHgfuB93C/Qv4C4zM6VzgygPrBHVdd4Xe8r3K/MyeJaPf2GKxP3p/uBEZ66oeHkQSWrqsYCDwGTcU8TR3B1Uidy4dyJwHW4J9BE4EngOlXd59nlDly9wXrPNR/xHBcD3AO8hXsi3UxaPUcx4BXcf297cE8LqU20u+L+2zmCq7C+JX2xqMk5yfz/fWNMYeNpmXQQqK+q2/wdj/E/e4IwphATketFpKSnvP414FfSmo6aQs4ShDGFW09chfIuXHHcLWcpUjSFkBUxGWOMyZQ9QRhjjMmU3wc1yy2VKlXS2rVr+zsMY4wpUFasWLFPVdP3KwICKEHUrl2bmJgYf4dhjDEFioik7/F+mhUxGWOMyZQlCGOMMZmyBGGMMSZTAVMHYYwJDKdOnSIuLo7jx4/7O5SAUrx4ccLCwihatGi2j7EEYYzJV+Li4ihTpgy1a9fmwuahMqlUlcTEROLi4ggPD8/2cVbEZIzJV44fP07FihUtOeQiEaFixYo5fiqzBGGMyXcsOeS+8/mbBk6CiI+Hffuy3s8YY0y2BE6C2LMHYmP9HYUxpoA7ePAg77zzznkde+2113Lw4MGsdywgAidBAGzY4O8IjDEF3LkSRFJS0jmPnTVrFuXLZzaTatayOrc/BE4rJhHYuNHfURhjCrhhw4axZcsWoqKi6Ny5M927d+fvf/87FSpUYP369WzcuJFevXqxc+dOjh8/zsMPP8zgwYOBtCF/jhw5Qrdu3bj00ktZvHgx1atXZ/r06ZQoUeKMa/Xv35/ixYuzatUqOnTowP79+ylRogSrVq1i7969TJgwgUmTJrFkyRLatm3LxIkTSU5OZuDAgcTExCAi3H333Tz66KNs2bKFBx54gISEBEqWLMn48eNp1KjRBf0tAidBFC9uTxDGBJpHHoHVq3P3nFFRMHr0WT9+5ZVX+O2331jtue78+fNZuXIlv/322+kmohMmTOCiiy7i2LFjtG7dmt69e1OxYsUzzrNp0yY+/fRTxo8fz80338wXX3zB7bffnuF6cXFxLF68mODgYPr378+BAwdYsmQJM2bMoEePHixatIj333+f1q1bs3r1apKTk4mPj+e3334DOF2kNXjwYMaNG0f9+vX5+eefuf/++/nhhx8u6E/l0yImEekqIhtEZLOIDDvLPjeLyFoRiRWRT9J9VlZE4kTkrSwvZgnCGOMjbdq0OaP/wNixY2nevDnt2rVj586dbNq0KcMx4eHhREVFAdCqVSu2b9+e6bn79OlDcHDw6fXrr78eEaFp06ZcfPHFNG3alKCgICIiIti+fTt16tRh69atPPTQQ3z77beULVuWI0eOsHjxYvr06UNUVBT33nsvu3fvvuD79tkThIgEA28DnXETzC8XkRmqutZrn/q4Scc7qOoBEamS7jQvAguydcFixWDrVjh5EkJCcuUejDF+do5f+nmpVKlSp9/Pnz+fuXPnsmTJEkqWLMmVV16Zaf+CYsWKnX4fHBzMsWPHsjy393FBQUFnnCMoKIikpCQqVKjAmjVrmD17NuPGjWPKlCmMHj2a8uXLn37qyS2+fIJoA2xW1a2qehKYjJve0Ns9wNuqegBAVfemfiAirYCLge+ydbXixSE5GbbZXOvGmPNXpkwZDh8+fNbPDx06RIUKFShZsiTr169n6dKleRgd7Nu3j5SUFHr37s1LL73EypUrKVu2LOHh4Xz++eeA6zm9Zs2aC76WLxNEdWCn13qcZ5u3BkADEVkkIktFpCuAiAQBrwNPZPtqxYu7VytmMsZcgIoVK9KhQwciIyMZOnRohs+7du1KUlISjRs3ZtiwYbRr1y5P44uPj+fKK68kKiqK22+/nZEjRwLw8ccf88EHH9C8eXMiIiKYPn36BV/LZ3NSi8hNQFdVHeRZvwNoq6oPeu3zNXAKuBkIwxUnNQVuB0qq6r9EpD8Q7X2c1/GDgcEAtcPCWm2Li4N//Qsy+Uc1xhQM69ato3Hjxv4OIyBl9rcVkRWqGp3Z/r5sxRQP1PBaD/Ns8xYH/Kyqp4BtIrIRqA+0By4TkfuB0kCIiBxR1TMqulX1PeA9gOjoaOXkSXuCMMaYXOLLIqblQH0RCReREOAWYEa6faYBVwKISCVckdNWVb1NVWuqam1cMdOk9MkhUw0bWoIwxphc4rMEoapJwIPAbGAdMEVVY0VkhIj08Ow2G0gUkbXAPGCoqiae90UbNLDOcsYYk0t82lFOVWcBs9JtG+71XoHHPMvZzjERmJitCzZsCB98AAcPwnl2dzfGGOME1lhMDRu6VytmMsaYC2YJwhhjTKYCK0HUqQPBwVYPYYw5bxcy3DfA6NGjOXr0aC5G5D+BlSCKFnVJwp4gjDHnKa8SRHJy8nlfI68EVoIAa+pqjLkg3sN9p/akHjVqFK1bt6ZZs2Y8//zzAPz11190796d5s2bExkZyWeffcbYsWPZtWsXV111FVdddVWGc9euXZunnnqKli1b8vnnn1O7dm2efvppoqKiiI6OZuXKlVxzzTXUrVuXcePGAbB7924uv/xyoqKiiIyMZOHChQB89913tG/fnpYtW9KnTx+OHDmS63+LwBnuO1XDhjB3LqSkQFDg5T9jChM/jPadYbjv7777jk2bNrFs2TJUlR49erBgwQISEhKoVq0aM2fOBNwYTeXKleONN95g3rx5VKpUKdPzV6xYkZUrVwIuGdWsWZPVq1fz6KOP0r9/fxYtWsTx48eJjIzkvvvu45NPPuGaa67h2WefJTk5maNHj7Jv3z5eeukl5s6dS6lSpXj11Vd54403GD58eKbXPF8BkyDi4+HoUSjZsCEcPw47d0KtWv4OyxhTwH333Xd89913tGjRAoAjR46wadMmLrvsMh5//HGeeuoprrvuOi677LJsna9v375nrPfo4bqFNW3alCNHjlCmTBnKlClDsWLFOHjwIK1bt+buu+/m1KlT9OrVi6ioKH788UfWrl1Lhw4dADh58iTt27fPxbt2AiZB7NnjHhx6NGjgNmzYYAnCmAIuP4z2rao8/fTT3HvvvRk+W7lyJbNmzeK5556jU6dO2foFn9PhvS+//HIWLFjAzJkz6d+/P4899hgVKlSgc+fOfPrppxd4d+cWMGUwwcEwbRrW1NUYc0HSD/d9zTXXMGHChNNl/PHx8ezdu5ddu3ZRsmRJbr/9doYOHXq62Cir4cJzaseOHVx88cXcc889DBo0iJUrV9KuXTsWLVrE5s2bAVcfstEHrTcD5gmiXDmYMQOS/u9iipQtawnCGHNevIf77tatG6NGjWLdunWni3BKly7Nf//7XzZv3szQoUMJCgqiaNGivPvuu4Cb+rNr165Uq1aNefPmXXA88+fPZ9SoURQtWpTSpUszadIkKleuzMSJE+nXrx8nTpwA4KWXXqJBaglKLvHZcN95rW7daN26NYYff4TLn2jjMsacOf4OyxiTQzbct+/kdLjvgCliKlfOzTQ6bRo2aJ8xxuSCgEkQQUFw9dUuQWiDhvD7765ZkzHGmPMSMAkCoFcvNyX1ryXauA2bNvk3IGPMeQmUou/85Hz+pgGVIK6/HkRg2rbmboNVVBtT4BQvXpzExERLErlIVUlMTKR48eI5Oi5gWjEBVK0K7dvD9CVVGA5WD2FMARQWFkZcXBwJCQn+DiWgFC9enLCwsBwdE1AJAlwx05NPBvF7aFtq2hOEMQVO0aJFCQ8P93cYhgArYgLo2dO9Ti9zuxUxGWPMBQi4BNGgATRuDNP+6uwShJVjGmPMeQm4BAGumOnH3fXZ/2cw7Nrl73CMMaZACtgEkZwSxEy6w6xZ/g7HGGMKpIBMENHRUK2aMq3UbfDll/4OxxhjCiSfJggR6SoiG0Rks4gMO8s+N4vIWhGJFZFPPNuiRGSJZ9svItI3s2PPJigIevYUvj3Rkb/mLoGDB3PjdowxplDxWYIQkWDgbaAb0AToJyJN0u1TH3ga6KCqEcAjno+OAnd6tnUFRotI+Zxc/4474GhSCKOSHgHPjE/GGGOyz5dPEG2Azaq6VVVPApOBnun2uQd4W1UPAKjqXs/rRlXd5Hm/C9gLVM7Jxdu3h743K6/yFL9/vPACb8UYYwofXyaI6sBOr/U4zzZvDYAGIrJIRJaKSNf0JxGRNkAIsCWTzwaLSIyIxGTW6/JfowSCg3nyu042cJ8xxuSQvyupiwD1gSuBfsB476IkEQkF/gMMUNWU9Aer6nuqGq2q0ZUrZ3zAqFkTnro1js+S+7Bw9Apf3YMxxgQkXyaIeKCG13qYZ5u3OGCGqp5S1W3ARlzCQETKAjOBZ1V16fkG8eSbNQiTeB5+vQbJyed7FmOMKXx8mSCWA/VFJFxEQoBbgBnp9pmGe3pARCrhipy2evb/CpikqlMvJIiS5Yoy6rIZrNpfmw/fT7qQUxljTKHiswShqknAg8BsYB0wRVVjRWSEiPTw7DYbSBSRtcA8YKiqJgI3A5cD/UVktWeJOt9Y+j5WnQ78xDNPpViLV2MMpUuXBmDXrl3cdNNN2T5uxYoVNG3alHr16jFkyJBMhySfP38+5cqVIyoqiqioKEaMGHH6szFjxhAZGUlERASjR48+vf3vf/87zZo1Iyoqii5durDLMwKEqjJkyBDq1atHs2bNWLly5eljnnrqKSIjI4mMjOSzzz47vf2yyy47fe1q1arRq1ev7P9h0lPVgFhatWqlZ3X0qK4ofokKyfrYY2ffzRhTOJQqVeq8jmvdurUuWbJEU1JStGvXrjpr1qwM+8ybN0+7d++eYfuvv/6qERER+tdff+mpU6e0U6dOumnTJlVVPXTo0On9xowZo/fee6+qqs6cOVO7du2qKSkpumTJEm3Tpo2qqn799dd69dVX66lTp/TIkSMaHR19xjlS3XjjjfrRRx+d856AGD3L96q/K6nzRokStLy+OgNLfMLYscrcuf4OyBiTm4YNG8bbb799ev2FF17gpZdeolOnTrRs2ZKmTZsyffr0DMdt376dyMjIbF1j9+7d/Pnnn7Rr1w4R4c4772TatGnZjnHdunW0bduWkiVLUqRIEa644gq+9Iz0ULZs2dP7/fXXX4gIANOnT+fOO+9ERGjXrh0HDx5k9+7drF27lssvv5wiRYpQqlQpmjVrxrfffnvG9f78809++OGHC3qCKBwJAuCGG3jt2AM0qnmU3r3h11/9HZAxJrf07duXKVOmnF6fMmUKd911F1999RUrV65k3rx5PP744+ecpW7Dhg2ni2bSLwcPHiQ+Pv6MCXfCwsKIj0/f7sZZsmQJzZs3p1u3bsTGxgIQGRnJwoULSUxM5OjRo8yaNYudO9N6Ajz77LPUqFGDjz/++HSxVHx8PDVqpLX1Sb1m8+bN+fbbbzl69Cj79u1j3rx5Z5wLYNq0aXTq1OmM5JNTATdh0Fl17065kOPM6vQGbb/+O927w9KlUK2avwMzxlyoFi1asHfvXnbt2kVCQgIVKlSgatWqPProoyxYsICgoCDi4+P5448/qFq1aqbnaNiwIatXr77gWFq2bMmOHTsoXbo0s2bNolevXmzatInGjRvz1FNP0aVLF0qVKkVUVBTBwcGnj3v55Zd5+eWXGTlyJG+99Rb/+Mc/znqNLl26sHz5ci655BIqV65M+/btzzgXwKeffsqgQYMu6F4KzxNE2bJw9dXUmDOBmV8rBw5A9+5w+LC/AzPG5IY+ffowdepUPvvsM/r27cvHH39MQkICK1asYPXq1Vx88cUcP378rMdn9QRRvXp14uLiTu8fFxdH9erp+/664qLUSvBrr72WU6dOsW/fPgAGDhzIihUrWLBgARUqVKBBgwYZjr/tttv44osvAKhevfoZTwbe13z22WdZvXo1c+bMQVXPONe+fftYtmwZ3bt3z8mfMIPCkyAAbrkFtm+nxZ5v+PxzV8x0881w6pS/AzPGXKi+ffsyefJkpk6dSp8+fTh06BBVqlShaNGizJs3jx07dpzz+NQniMyW8uXLExoaStmyZVm6dCmqyqRJk+jZM/3oQbBnz57TRVnLli0jJSWFihUrArB3714Afv/9d7788ktuvfVWADZt2nT6+OnTp9OoUSMAevTowaRJk1BVli5dSrly5QgNDSU5OZnExEQAfvnlF3755Re6dOly+hxTp07luuuuo3jx4uf753TOVntd0JZztmJKdfKkas2aqpdcopqSouPHq4Lq3XerJiVlfbgxJn+LjIzUK6+8UlVVExIStF27dhoZGan9+/fXRo0a6bZt21Q1rRXTtm3bNCIiItvnX758uUZERGidOnX0gQce0JSUFFVVfffdd/Xdd99VVdU333xTmzRpos2aNdO2bdvqokWLTh9/6aWXauPGjbVZs2Y6d+7c09tvvPFGjYiI0KZNm+p1112ncXFxqqqakpKi999/v9apU0cjIyN1+fLlqqp67Ngxbdy4sTZu3Fjbtm2rq1atOiPOK664Qr/55pts3RPnaMUkGiBTckZHR2tMTEzWO779Njz4IMybB1deyfDh8OKLcP318PHHUKaM72M1xpj8QkRWqGp0Zp8VriImgLvvhosvhpdfBmDECHjrLTfxXIcOkMVTqDHGFBqFL0GUKAGPPw5z58KyZQA88AB88w38/ju0aQOLF/s5RmOMyQcKX4IAuO8+qFDh9FMEQOfOrtlr2bJw1VUwcSIESOmbMcacl8KZIMqUgYcfhhkzzugx16gR/PyzK2oaMAB69gSvVm3GGJOpQKnLTa9wJgiAhx6C0qXhn/88Y/NFF8GcOfD6664UKiIC3nsPUjLMRmGMMTBjxgwefvhhf4fhE4U3QVx0EfztbzBlCni1QQYIDobHHnMPF9HRcO+90LEjbNzop1iNMfnW+PHjad26tb/D8InCmyDAZYGiReGVVzL9uG5d9xTx/vuwerV7mrjvPit2MsY4CQkJLFy4kBtuuMHfofhE4U4QVavCPffARx/BL79kuosIDBwI69e7J4kJE6BePZdbMpkG2xhTiHzzzTd06tTp9NAagaZwJwiAF15wLZoGD+Zcc5JWrer6S2zcCP36wZgxEB4OTz3lmscaYwqfH374gauvvtrfYfiMJYiKFeHf/3bNl8aNy3L32rXhww8hNhauuw5ee80liptuggULrGmsMYWFqvLDDz/QsWNHf4fiM5YgAG67zXWEePppOMv47uk1agSTJ8PWrTB0qBu544orICoK3n0X9u/3cczGGL/asmULycnJmY7IGigsQYCraHj3XTes65AhOTq0Vi1Xx71zJ4wf77bdf78rkrrhBvjySzhxwgcxG2P86ocffqBTp06nZ38LRJYgUtWtC8OHu2/0GTNyfHjJkjBokGvttHKl62axdCn07u2SxYABMH06HD3qg9iNMXlu8eLFXHrppf4Ow6csQXh7/HHXlvXBB+HIkfM6hQi0aOE62u3cCbNnu5Fiv/oKevWCSpXc64cfwp49uRy/MSbPbNu2jfr16/s7DJ+yBOEtJMR1m965E5577oJPV6QIdOkCkya5JrFz57omsytXukFlQ0OhWTPXZPabb+Cvv3LhHowxeWLbtm3Url3b32H4lE8ThIh0FZENIrJZRIadZZ+bRWStiMSKyCde2+8SkU2e5S5fxnmGSy5xlQhjxsDMmbl22qJFoVMnePNNN6T4ypWu7qJKFXjnHbj2Wtfa9tJLYdgw+PprOHAg1y5vjMlFJ0+eZM+ePYSFhfk7FJ/y2YRBIhIMbAQ6A3HAcqCfqq712qc+MAXoqKoHRKSKqu4VkYuAGCAaUGAF0EpVz/qVme0Jg7Lj2DFo3951cFi1ytVE+9CxY/DTT+4JY+FCiIlJmwY1MhLatnXDkLdp40rAihb1aTjGmCxs3bqVjh07sn37dn+HcsHONWFQER9etw2wWVW3eoKYDPQE1nrtcw/wduoXv6ru9Wy/Bpijqvs9x84BugKf+jDeNCVKwNSp0KoV9OnjvrWLFfPp5Tp3dgu4iuzly91lf/rJ1V988IH7rHhxV8fRsiU0b+6a1UZGunMYY/LG77//Ts2aNf0dhs/5MkFUB3Z6rccBbdPt0wBARBYBwcALqvrtWY6tnv4CIjIYGAzk/j9WvXpuUogbb3SVBG+/nbvnP4eSJV2fiiuucOuqrr/FsmUucSxf7uo1Dh92nwcFQcOG7unCe6lf3542jPGFpKQkQkJC/B2Gz/kyQWT3+vWBK4EwYIGINM3uwar6HvAeuCKmXI/uhhvgiSdcd+kOHeDWW3P9Etkh4lrh1q3rhvkAN/z49u2uWe3q1bBmjSsN++KLtN7cRYq4Yxo2hAYN0l7r1nUV5EHWRMGY8xIUFERKIZgDwJcJIh6o4bUe5tnmLQ74WVVPAdtEZCMuYcTjkob3sfN9Fum5/POfrkPD4MGuPKdJE7+EkV5QENSp45Ybb0zbfuyYG1gwNhbWroUNG9wye/aZHfaKF3fHpiae2rXdEh7uXsuWzeMbMqYAERFSUlI4evQowcHBFPNhEbQ/+bKSugiukroT7gt/OXCrqsZ67dMVV3F9l4hUAlYBUaRVTLf07LoSV0l91gEscrWSOr1du1zBf/nysGiR68xQwCQnuzr3jRthy5Yzl61bM3bgK18eatQ4cwkLg+rV0xZLIqaw+YpkaJAAACAASURBVPnnn9m/fz8lS5Zk+PDhNGjQgEsuuYQBAwb4O7Tz5pdKalVNEpEHgdm4+oUJqhorIiOAGFWd4fmsi4isBZKBoaqa6An6RVxSARhxruTgc9WqubKbzp3dCH3ffw+lSvktnPMRHOyeDsLDM36mCvv2uSKr1GXHDtcdZOdOV/exb1/G40qXdn+a0FDXWzw0NO19lSpw8cVuqVzZ6kJMYChRogR33nkn7777LqdOneLLL7/k+eef93dYPuOzJ4i85tMniFTTprmxM7p1c++L+LsKJ+8cPeoepOLjz1x27z5zOVtnvwoVXKLwXipVcoPppr6mLhdd5J5gCtGf1xQgI0eOZNq0aezevZuaNWvy008/+TukC3KuJwhLEDk1bpybqvTuu91UcwE8UNf5OHwY/vgD9u51r6lLQkLGJTERkpLOfq5y5VyyqFDBLeXLp70vV86te7+WK+eKvcqVgzJlLMEY30hKSuKSSy4hJiaGMWPG8NBDD/k7pAvir34Qgem++9xP6RdfdOUrL77o74jylTJl3FKvXtb7qsKff7pEsW+fe92//8wlMREOHnS9ynftSnt//HjW5y9ZMi2esmXT3pcp44rHSpc+832pUmmv3kvJkmmvISH2m6CwK1KkCP/9739p0qQJvXr18nc4PmUJ4nz84x/u2+qll1yB+wMP+DuiAkkk7Zd/nTo5O/bECTh0yC0HD7rXP//M+Hr4cNrr4cNuPvEjR9xy+HDOx78KCnKJInUpUSLt1XspXvzM995LsWIZX9MvISEZX0NCXF2ONU/2vwYNGnDy5EmCAvwfwxLE+RBxRU0JCW7kV7AkkceKFXMV4VWqXNh5UlJckvBejhxJe3/06JmfHTvmtqVuP3YsbTl82BWtHTvmnnCOH0/77FxFaTlVpEhasvBOHOdaihRJe03/PrMlODjja1ZLUFDG90FB534fFOT+d0r/3vv1bNvOtcC533u/ZrUtVcbS+CBCQlxDjEBlCeJ8FSkCU6ZA374uSSQlwcMP+zsqk0NBQWnFTr6UnOyeelITx/Hjbj11SV0/eTJtW+r7kyfPXE6ccGN1pa6nvj91Km1JXU9KcudOfZ/6mrqkricnn7k9dVsh6At2QTp0cMPhBCpLEBeiWDGXJPr1g0cecf+3PfGEv6My+VBwcFqxVEGimpY8kpPPvqSkZHyfknL298nJ7typ21Ovo5q23fv1bNsyW1LjPtt779estqWX/qkikJ8ewBLEhQsJcZNT33abm5w6KcmN121MABBJK3YyhY/9s+eGokXhk0/c69NPu0LnF16w5i7GmALNEkRuKVLEDbFarBiMGOGay4wbZ12IjTEFliWI3BQc7CZuCAtz/SN27YLPP3eN640xpoAJ7Ea8/iDiniDGj4c5c9ykDnv2+DsqY4zJMUsQvjJoEPzvf26s7XbtYN06f0dkjDE5YgnCl7p1gx9/dA3R27Z1CcMYYwqIcyYIEeno9T483Wc3ZjzCZNCqlZsjtGFD6NHD1U1Y7yNjTAGQ1RPEa17vv0j32XO5HEvgqlEDFiyAO++E4cPdkOGpE0obY0w+lVWCkLO8z2zdnEuJEjBxIowe7Yqa2rVz07sZY0w+lVWC0LO8z2zdZEXEjdf03XdukoRWrVwHO2OMyYey6gdRR0Rm4J4WUt/jWc9k8kqTLR07wqpVcOutboiOH36AsWML3kA9xpiAllWC6On1/rV0n6VfNzlRowbMmwfPPw8jR8LSpW7gvyZN/B2ZMcYAWRQxqeqP3guwGPgTWOdZNxeiSBF4+WX49ls3kUDr1q6DXYBMA2uMKdiyauY6TkQiPO/LAWuAScAqEemXB/EVDl26wJo10L49DB7smsNa72tjjJ9lVUl9marGet4PADaqalOgFfCkTyMrbEJDXeX16NEwdy40bQpffeXvqIwxhVhWCeKk1/vOwDQAVc3Wz1sR6SoiG0Rks4hkmCRBRPqLSIKIrPYsg7w++5eIxIrIOhEZK1IIxs4OCnKtnFasgJo14cYbYcAAN8GyMcbksawSxEERuU5EWgAdgG8BRKQIUOJcB4pIMPA20A1oAvQTkcxqYD9T1SjP8r7n2Es812sGRAKtgSuyf1sFXJMmsGQJPPecG0I8IgK+/trfURljCpmsEsS9wIPAh8AjXk8OnYCZWRzbBtisqltV9SQwmTNbRZ2LAsWBEKAYUBT4I5vHBoaQEDcsx9KlUKECXH+9axK7b5+/IzPGFBJZtWLaqKpdPb/uJ3ptn62qj2dx7urATq/1OM+29HqLyC8iMlVEanjOvwSYB+z2LLNVNcNwqCIyWERiRCQmISEhi3AKqNatXZHT88+nNYP97DNr6WSM8bmsWjGNPdeSC9f/H1BbVZsBc4CPPNetBzQGwnBJpaOIXJb+YFV9T1WjVTW6cuXKuRBOPhUS4qYwXbkSatWCW26B666Dbdv8HZkxJoBlVcR0H3ApsAuIAVakW84lHqjhtR7m2Xaaqiaq6gnP6vu41lEANwBLVfWIqh4BvgHaZ3G9wNe0qaubeP11N4x4RITrZHfyZNbHGmNMDmWVIEKB94BrgDtwdQHTVfUjVf0oi2OXA/VFJFxEQoBbgBneO4hIqNdqDyC1GOl34AoRKSIiRXEV1DbjDrjOdY895iYg6tYNnnkGoqLcaLHGGJOLsqqDSFTVcap6Fa4fRHlgrYjckdWJVTUJV8E9G/flPkVVY0VkhIj08Ow2xNOUdQ0wBOjv2T4V2AL8iuuct0ZVbbYdbzVqwBdfuJFhjx51U5vefrubB9sYY3KBaDYqO0WkJdAP1xdiBfC6qq71cWw5Eh0drTExMf4Owz+OHoV//hNGjYKiReHvf4dHHoFixfwdmTEmnxORFaoandlnWVVSjxCRFcBjwI9AtKoOzG/JodArWRJeegnWroVOnWDYMFdfMWuWvyMzxhRgWdVBPIcrVmoOjARWepqk/ioiv/g8OpMzdevC9OnwzTdu7onu3aFrV/jtN39HZowpgLIa7tvmfCiIunaFX3+Ft95yne2aN3eDAP7jH1Clir+jM8YUEFlVUu/IbMF1gLs0b0I05yUkxLV22rwZHnwQ3n8f6tWDV16BY8f8HZ0xpgDIqg6irIg8LSJviUgXcR4CtgI3502I5oJUrAhjxrhipiuvhKefhvr1YcIESE72d3TGmHwsqzqI/wANcc1NB+GGv7gJ6KWq2R1XyeQHDRvCjBkwfz6EhcHAgdCsmWsma8N2GGMykVWCqKOq/VX1/3DNXJsA16jqat+HZnziiitcb+ypU+HUKTc50eWXw8KF/o7MGJPPZJUgTqW+UdVkIE5Vj/s2JONzItC7N8TGwjvvwJYtLkl07eoGBjTGGLJOEM1F5E/PchholvpeRP7MiwCNDxUtCn/7m6vI/te/YPlyiI5OSx7GmEItq1ZMwapa1rOUUdUiXu/L5lWQxsdKloShQ93osM8/D3PmuI52/fq5MZ+MMYVSVk8QpjApW9YNK751Kzz1lKvAjohwExVt2ODv6IwxecwShMmoUiU3jPi2bfDEEzBtmpuo6I47YP16f0dnjMkjliDM2VWu7Oomtm1zne6+/NIliltuseE7jCkELEGYrFWp4kaK3b7dFT3NnOnqKG680c1yZ4wJSJYgTPZVruyKnnbsgOHD4YcfoFUrN3GR9aMwJuBYgjA5d9FFbuC/HTvcPBQrVrh+FJdd5kaStZ7ZxgQESxDm/JUr58Z22r4dxo51CePaa6FFC/jkE0hK8neExpgLYAnCXLiSJeGhh1yHuwkT4MQJ1zS2fn035PjRo/6O0BhzHixBmNwTEgIDBrhe2NOmQWioSxy1arkiqX37/B2hMSYHLEGY3BcUBD17wqJFrvK6XTvXAa9mTbj/fvekYYzJ9yxBGN8RgUsvdT2yY2Ph1lvhgw+gQQM33tPixVahbUw+5tMEISJdRWSDiGwWkWGZfN5fRBJEZLVnGeT1WU0R+U5E1onIWhGp7ctYjY81aeJmtduxA555BubNgw4doH17mDLFKrSNyYd8liBEJBh4G+iGm0ein4g0yWTXz1Q1yrO877V9EjBKVRsDbYC9vorV5KGqVeGll2DnTnj7bUhMhL59oW5deP11OHTI3xEaYzx8+QTRBtisqltV9SQwGcjWLHSeRFJEVecAqOoRVbWmMIGkVClXH7FhA0yfDuHhbtynsDAYMgQ2bfJ3hMYUer5MENWBnV7rcZ5t6fUWkV9EZKqI1PBsawAcFJEvRWSViIzyPJGcQUQGi0iMiMQkJCTk/h0Y3wsKcrPazZ/vOtzdeCOMG+emSL3+evj+e6unMMZP/F1J/T+gtqo2A+YAH3m2FwEuA54AWgN1gP7pD1bV91Q1WlWjK1eunDcRG99p2RI++gh+/90N5fHzz3D11W7cp/fes/4UxuQxXyaIeKCG13qYZ9tpqpqoqic8q+8DrTzv44DVnuKpJGAa0NKHsZr8pGpV1yz299/hww/dzHf33uuKn5580vXcNsb4nC8TxHKgvoiEi0gIcAsww3sHEQn1Wu0BrPM6tryIpD4WdATW+jBWkx8VLw79+7sRYxcudE8Tb7zhKrRvuMGKn4zxMZ8lCM8v/weB2bgv/imqGisiI0Skh2e3ISISKyJrgCF4ipFUNRlXvPS9iPwKCDDeV7GafC61P8WUKW5uiqeegp9+cgkjIgLeeQcOH/Z3lMYEHNEA+QUWHR2tMTEx/g7D5JXjx13CePNNiImBMmXgrrtcy6jGjf0dnTEFhoisUNXozD7zdyW1MeeneHG4805YtgyWLnVDe7z3nuuQ16kTfPGFdb4z5gJZgjAFmwi0bQv/+Y/rfPfPf7qxnm66CWrXhhEjYNcuf0dpTIFkCcIEjipV3PwUW7e6zncREfD882402T593PAeAVKkakxesARhAk9wsOt8N3s2bNwIDz/sWjx17OiKoMaMgYMH/R2lMfmeJQgT2OrXh9deg/h4mDgRypaFRx6BatVg4EBYvtzfERqTb1mCMIVDiRKuldPPP7shPW6/HSZPhjZtIDoaxo+HI0f8HaUx+YolCFP4tGzpWjzt2uWayZ44AYMHu6eKBx6AX37xd4TG5AuWIEzhVa4cPPigSwiLFkGvXm5Co+bN3TwVEyfa+E+mULMEYYwIXHIJTJrk6ireeAMOHHDza1ev7oYfj431d5TG5DlLEMZ4q1gRHn0U1q1zQ5B36wb/938QGemG+5g0CY4d83eUxuQJSxDGZEYErrgCPvnEPVW89hokJLiK7mrV3FPFb7/5O0pjfMoShDFZqVQJHn8c1q93TxXXXuueKpo2dUVTH34If/3l7yiNyXWWIIzJrtSnio8/dk8Vr7/u6iruvts9Vdx/P6xe7e8ojck1liCMOR+VKsFjj8HatbBggRsscMIEaNECWrd2/SpsCHJTwFmCMOZCiMBll7nK69273TAex4+7fhWhoXDPPW7EWRsDyhRAliCMyS0VKrjK619+gSVLoG9fV8ndti1ERcFbb7kiKWMKCEsQxuQ2EWjXznW6270b3n3Xzav90EOuruKOO+DHH+2pwuR7liCM8aWyZeG++9ysdytXus53M2bAlVdCo0bwr3/BH3/4O0pjMmUJwpi80qKFmz979243jEeVKm5+7bAw6N0bvvkGkpP9HaUxp1mCMCavlSzpOtwtXOhaQT38sGsJde21EB4OL7wAO3b4O0pjLEEY41eNG6fNVzFlipvQaMQIlyi6doWpU+HkSX9HaQopSxDG5AchIW5a1G+/hW3bYPhw93TRp48rgnriCTc+lDF5yKcJQkS6isgGEdksIsMy+by/iCSIyGrPMijd52VFJE5E3vJlnMbkK7VquWKmbdtg1iy4/HLXv6JJE+jQwYb2MHnGZwlCRIKBt4FuQBOgn4g0yWTXz1Q1yrO8n+6zF4EFvorRmHwtONiNJjt1atqAgfv3u6E9QkPh3nutE57xKV8+QbQBNqvqVlU9CUwGemb3YBFpBVwMfOej+IwpOKpUcQMGrl3rKrd794b//Md1wmveHMaOdcnDmFzkywRRHdjptR7n2ZZebxH5RUSmikgNABEJAl4HnjjXBURksIjEiEhMQkJCbsVtTP4l4ual+PBD11x23DgoVsy1hKpWDfr1g++/h5QUf0eapw4ePMg777yT4+OuvfZaDh48eMHXHzlyJPXq1aNhw4bMnj0703369+9PeHg4UVFRREVFsdozsOP69etp3749xYoV47XXXjvjmIMHD3LTTTfRqFEjGjduzJIlSwDYv38/nTt3pn79+nTu3JkDnh768+fPp1y5cqevMWLECAB27tzJVVddRZMmTYiIiGDMmDHZuzFV9ckC3AS877V+B/BWun0qAsU87+8FfvC8fxB40vO+f/rjMltatWqlxhRaq1erPvSQaoUKqqAaHq764ouqO3f6O7I8sW3bNo2IiMiw/dSpUz6/dmxsrDZr1kyPHz+uW7du1Tp16mhSUlKG/e666y79/PPPM2z/448/dNmyZfrMM8/oqFGjzvjszjvv1PHjx6uq6okTJ/TAgQOqqjp06FAdOXKkqqqOHDlSn3zySVVVnTdvnnbv3j3DNXbt2qUrVqxQVdU///xT69evr7GxsaqqCsToWb5XffkEEQ/U8FoP82w7TVUTVfWEZ/V9oJXnfXvgQRHZDrwG3Ckir/gwVmMKttRipl273PhPderA3//uKry7d4cvvwzo5rLDhg1jy5YtREVF0bp1ay677DJ69OhBkyau2rNXr160atWKiIgI3nvvvdPH1a5dm3379rF9+3YaN27MPffcQ0REBF26dOFYNmcOnD59OrfccgvFihUjPDycevXqsWzZsmzHXqVKFVq3bk3RokXP2H7o0CEWLFjAwIEDAQgJCaF8+fKnr3nXXXcBcNdddzFt2rRzXiM0NJSWLVsCUKZMGRo3bkx8fPw5jwHfFjEtB+qLSLiIhAC3ADO8dxCRUK/VHsA6AFW9TVVrqmptXDHTJFXN0ArKGJNO8eKumGnuXNiyBZ55BtascXUWYWEwdKib+CjAvPLKK9StW5fVq1czatQoVq5cyZgxY9i4cSMAEyZMYMWKFcTExDB27FgSExMznGPTpk088MADxMbGUr58eb744gsARo0adbrIxnsZMmQIAPHx8dSokfZbOCws7Kxfvs8++yzNmjXj0Ucf5cSJE5nuk2rbtm1UrlyZAQMG0KJFCwYNGsRfntZrf/zxB6Gh7uuzatWq/OE1XMuSJUto3rw53bp1IzaTudS3b9/OqlWraNu27TmvDz5MEKqahCsqmo374p+iqrEiMkJEenh2GyIisSKyBhiCK04yxuSGOnXgxRddr+yZM13dxejRrnPepZe64T4CtLlsmzZtCA8PP70+duxYmjdvTrt27di5cyebNm3KcExq/QBAq1at2L59OwBDhw5l9erVGZaxY8fmKKaRI0eyfv16li9fzv79+3n11VfPuX9SUhIrV67kb3/7G6tWraJUqVK88krGghQRQUQAaNmyJTt27GDNmjU89NBD9OrV64x9jxw5Qu/evRk9ejRly5bNMmaf9oNQ1Vmq2kBV66rqy55tw1V1huf906oaoarNVfUqVc3w00ZVJ6rqg76M05iAFhzshvH48kuIi3MDBO7b5wYODA11c1cEWHPZUqVKnX4/f/585s6dy5IlS1izZg0tWrTg+PHjGY4pVqzY6ffBwcEkJSUBWT9BVK9enZ0709rjxMXFUb16xvY4oaGhiAjFihVjwIABWRZDhYWFERYWdvqX/k033cTKlSsBuPjii9m9ezcAu3fvpkqVKgCULVuW0qVLA64C/tSpU+zbtw+AU6dO0bt3b2677TZuvPHGc147lfWkNqYwufhiV8y0bp1rLnvjjfDf/7rmss2auScMzxdKQVKmTBkOn2UGv0OHDlGhQgVKlizJ+vXrWbp0aY7OndUTRI8ePZg8eTInTpxg27ZtbNq0iTZt2mQ4T+oXuqoybdo0IiMjz3ndqlWrUqNGDTZs2ADA999/f7pOpUePHnz00UcAfPTRR/Ts6XoQ7NmzJ7UBEMuWLSMlJYWKFSuiqgwcOJDGjRvz2GOPZf/mz1Z7XdAWa8VkzHk6eFB13DjV1q1dC6iQENWbb1adPVs1Odnf0WVbv379NCIiQqOjo89oyXP8+HHt2rWrNmrUSHv27KlXXHGFzps3T1VVa9WqpQkJCRlaQY0aNUqff/75bF/7pZde0jp16miDBg101qxZp7d369ZN4+PjVVX1qquu0sjISI2IiNDbbrtNDx8+rKqqu3fv1urVq2uZMmW0XLlyWr16dT106JCqqq5atUpbtWqlTZs21Z49e+r+/ftVVXXfvn3asWNHrVevnnbq1EkTExNVVfXNN9/UJk2aaLNmzbRt27a6aNEiVVVduHChAtq0aVNt3ry5Nm/eXGfOnKmq527FJBogj5XR0dEaExPj7zCMKdh+/dVNdPSf/7iOd7VquZ7bAwZAjRpZH28KHBFZoarRmX1mRUzGmDRNm7pipvh4mDwZ6teH5593icJGly10LEEYYzIqXtzNqT1nDmzdCs89B7GxbnTZ6tXhscfcugloliCMMecWHu7mqNi+3Y0ue8UV8NZbEBkJ7du7IqkjR/wdpfEBSxDGmOzJbHTZQ4dg0CCoWtW9LlkSUM1lc+Knn37Kcd+I/M4ShDEm5ypXdqPLxsbC4sWuOGryZLjkEvdk8cYbUMgG0Bw+fDgXXXSRv8PIVZYgjDHnTyStmGn3bhg/HsqWdcmjenVXZzF7NiQn+ztSn1q1ahUbN26kb9++/g4lV1mCMMbkjjJl0oqZfvsNHnwQ5s1zrZ/q1HGz5O3Y4e8ofWLcuHHcf//9GQbcK+isH4QxxndOnIDp090Txpw5bluXLjBwIPTs6ebiLuBOnTpFaGgoMTEx1K5d29/h5Jj1gzDG+EexYnDzza6Yads2GD7czYp3882uCCp1lrwCbO7cuTRo0KBAJoesWIIwxuSNWrVcMdO2bfDNN3DllfDmmxAR4Sq3J0wokM1lJ0+eTL9+/fwdhk9YgjDG5K3gYFcv8fnnrrns66/DwYOu2Ck0FO65B37+uUA0lz1+/DgzZsygT58+/g7FJyxBGGP8p3LltF7ZP/3kWj198gm0a+dGlx0zBjKZ3Ce/+OGHH2jevDlVq1b1dyg+YQnCGON/ItChgytm2r0b3nsPSpaERx6BatXcLHnffw8pKf6O9AxLly7l0ksv9XcYPmMJwhiTv5Qtm1bMtGYN3Hefq+S++mqoVw9eftkVTeUDy5cvp3Xr1v4Ow2csQRhj8q/UYqZdu1zRU3i4GziwZk24/nqYMQM8M7/lNVW1BGGMMX5XvHhaMdPmzTBsGKxY4fpS1KwJzzwDW7bkaUjbt2+nWLFiVKtWLU+vm5csQRhjCpa6dV0x0++/uyeI6Gh49VVX/NSxI3z6KWQy53RuW7FiBdHRmfYvCxiWIIwxBVORImnFTL//7pLG9u1w662uYvvhh92QHz4SHx9PrVq1fHb+/MAShDGm4Kte3RUzbd4Mc+fCNdfAuHFuhrx27XwyZ0ViYmLAjd6ank8ThIh0FZENIrJZRIZl8nl/EUkQkdWeZZBne5SILBGRWBH5RUQCa4hEY4xvBAVBp06umCk+Hv79bzh82A0iGBoKgwfDsmW50gkvMTGRihUr5kLQ+ZfPEoSIBANvA92AJkA/EWmSya6fqWqUZ3nfs+0ocKeqRgBdgdEiUt5XsRpjAlClSq4fxW+/uTkr+vSBjz+Gtm0hKsrNinfgQI5P+8EHH3Ds2DH2799P+fLlueOOO0jJZ/0zcosvnyDaAJtVdauqngQmAz2zc6CqblTVTZ73u4C9QGWfRWqMCVypc1akdsJ7911Xf/HQQ66u4o47YOHCbD9VfPvtt/z73/8mMTGRjRs3sm3bNoKCArO03pd3VR3Y6bUe59mWXm9PMdJUEamR/kMRaQOEABnasInIYBGJEZGYhEI2e5Ux5jyULes63q1Y4Zb+/d1w5JdfDk2auHGhsvguefXVV3njjTc4dOgQ//vf/xgyZEjexO4H/k57/wNqq2ozYA7wkfeHIhIK/AcYoKoZnuFU9T1VjVbV6MqV7QHDGJMDLVu6p4ndu93TRYUK8MQTrsK7b19X2Z1J0VGdOnUYNGgQGzZsID4+nhtuuMEPwecNXyaIeMD7iSDMs+00VU1U1ROe1feBVqmfiUhZYCbwrKou9WGcxpjCrFQpGDDA1VP89hvcf7+b3KhzZ6hfH0aOdEnEy7PPPsvx48fp3r17wM0i582XCWI5UF9EwkUkBLgFmOG9g+cJIVUPYJ1newjwFTBJVaf6MEZjjEkTEQGjR7uhPT7+OK2Xdo0acMMNbh6L5GTKlCnDnDlzGDNmjL8j9imfJQhVTQIeBGbjvvinqGqsiIwQkR6e3YZ4mrKuAYYA/T3bbwYuB/p7NYGN8lWsxhhzhuLFXYe7efNg40Y3893ixXDttW5+7REjuCw8nHLlyvk7Up+yOamNMSY7Tp50vbbfe88VQQUFwS23uCeNAuxcc1IXyetgjDGmQAoJgZtucsvWra53dnCwv6PyKUsQxhiTU3XquLGfApy/m7kaY4zJpyxBGGOMyZQlCGOMMZmyBGGMMSZTliCMMcZkyhKEMcaYTFmCMMYYkylLEMYYYzIVMENtiEgCsMPfceRQJWCfv4O4QHYP+UMg3AMExn0UtHuopaqZzpcQMAmiIBKRmLONgVJQ2D3kD4FwDxAY9xEI95DKipiMMcZkyhKEMcaYTFmC8K/3/B1ALrB7yB8C4R4gMO4jEO4BsDoIY4wxZ2FPEMYYYzJlCcIYY0ymLEHkERGZICJ7ReQ3r20XicgcEdnkea3gzxjPRURqiMg8EVnrmUf8Yc/2AnMPACJSXESWicgaz338w7M9XER+FpHNIvKZiIT4O9asiEiwiKwSka896wXqHkRkakQPdAAABVxJREFUu4j86plzPsazraD991ReRKaKyHoRWSci7QvaPZyLJYi8MxHomm7bMOB7Va0PfO9Zz6+SgMdVtQnQDnhARJpQsO4B4ATQUVWbA1FAVxFpB7wK/FtV6wEHgIF+jDG7HgbWea0XxHu4SlWjvPoNFLT/nsYA36pqI6A57t+joN3D2amqLXm0ALWB37zWNwChnvehwAZ/x5iDe5kOdC7g91ASWAm0xfV8LeLZ3h6Y7e/4sog9DPfl0xH4GpACeA/bgUrpthWY/56AcsA2PI19CuI9ZLXYE4R/Xayquz3v9wAX+zOY7BKR2kAL4GcK4D14imZWA3uBOcAW4KCqJnl2iQOq+yu+bBoNPAmkeNYrUvDuQYHvRGSFiAz2bCtI/z2FAwnAh56ivvdFpBQF6x7OyRJEPqHu50a+b3MsIqWBL4BHVPVP788Kyj2oarKqRuF+hbcBGvk5pBwRkeuAvaq6wt+xXKBLVf+/vfsJsaoM4zj+/dFUSg1K4SKwmIJKKEoHXIQmA4UQSJRERkVSmwoqMCLMTRAEA0bQKoiEWkyGWGMuwlr0bzDSAc3JGtpU0BiOgSWpFGVPi/e5dRjP3FuNzp2Dv8/mnnPPn/u+w2Gee973nueJfuA2ypDlqurGBlxPPUA/8HJELANOMGU4qQF9aMsBorsmJV0GkK9HutyetiSdTwkOQxHxdr7dqD5URcTPwIeU4ZiFknpy02LgUNca1tkK4HZJ3wFvUoaZXqJZfSAiDuXrEWCYEqybdD1NABMRsSfXt1MCRpP60JYDRHftBNbn8nrKuP6cJEnAFmA8Il6sbGpMHwAkLZK0MJfnU+ZRximB4q7cbU73IyKeiYjFEdEH3AN8EBH30aA+SLpIUm9rGVgNHKRB11NEHAa+l3RtvnUL8BUN6kMnfpJ6lkjaCgxQUgFPAs8CO4BtwBWUVOV3R8TRbrWxHUkrgRHgC/4Z995EmYdoRB8AJN0AvA6cR/mCtC0inpN0FeXb+CXAfuD+iPitey39dyQNAE9FxJom9SHbOpyrPcAbEfG8pEtp1vW0FHgVuAD4BniQvK5oSB/acYAwM7NaHmIyM7NaDhBmZlbLAcLMzGo5QJiZWS0HCDMzq+UAYZYkfZqvfZLuPcPn3lT3WWZzmX/majZF9dmC/3BMTyUPUt324xFx8Zlon9ls8R2EWZJ0PBcHgZuzTsGGTO63WdKopDFJD+f+A5JGJO2kPEGLpB2ZfO7LVgI6SYPA/DzfUPWzVGyWdDBrI6yrnPujSq2BoXyaHUmDKnU5xiS9MJt/Izu39HTexeycs5HKHUT+oz8WEcslXQjslvR+7tsPXB8R3+b6QxFxNNN4jEp6KyI2SnosEwROtZZSl+JGylP2o5I+yW3LgOuAH4DdwApJ48CdwJKIiFbaELOzwXcQZp2tBh7IFOF7KKm1r85teyvBAeAJSQeAz4DLK/tNZyWwNTPMTgIfA8sr556IiD+Bzyn1RI4BvwJbJK0FTs64d2bTcIAw60zA41Eqny2NiCsjonUHceLvncrcxa3ATVEq1u0H5s3gc6t5lE5RigH9Qcl6uh1YA+yawfnN2nKAMDvdL0BvZf094NFMd46kazID6VQLgJ8i4qSkJZTSrC2/t46fYgRYl/Mci4BVwN7pGpb1OBZExLvABsrQlNlZ4TkIs9ONAadyqOg1Sq2FPmBfThT/CNxRc9wu4JGcJ/iaMszU8gowJmlfpuZuGabUozhAKSzzdEQczgBTpxd4R9I8yp3Nk/+vi2ad+WeuZmZWy0NMZmZWywHCzMxqOUCYmVktBwgzM6vlAGFmZrUcIMzMrJYDhJmZ1foLT6ew+88zPRoAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4eGeePM34U76",
        "outputId": "1a32f94b-9573-4109-b453-af35878be0e7"
      },
      "source": [
        "idx =  1002457\n",
        "print(model.predict(torch.tensor([data[idx][2]]).to(device), torch.tensor([data[idx][4]]).to(device)).cpu().data[0] * 2 + 3)\n",
        "\n",
        "print(data[idx][1] * 2 + 3)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(3.9568)\n",
            "5\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}