multi-task-learning-example-PyTorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "import pylab\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "\n",
    "def gen_data(N):\n",
    "    X = np.random.randn(N, 1)\n",
    "    w1 = 2.\n",
    "    b1 = 8.\n",
    "    sigma1 = 1e1  # ground truth\n",
    "    Y1 = X.dot(w1) + b1 + sigma1 * np.random.randn(N, 1)\n",
    "    w2 = 3\n",
    "    b2 = 3.\n",
    "    sigma2 = 1e0  # ground truth\n",
    "    Y2 = X.dot(w2) + b2 + sigma2 * np.random.randn(N, 1)\n",
    "    return X, Y1, Y2\n",
    "\n",
    "\n",
    "class TrainData(Dataset):\n",
    "\n",
    "    def __init__(self, feature_num, X, Y1, Y2):\n",
    "\n",
    "        self.feature_num = feature_num\n",
    "\n",
    "        self.X = torch.tensor(X, dtype=torch.float32)\n",
    "        self.Y1 = torch.tensor(Y1, dtype=torch.float32)\n",
    "        self.Y2 = torch.tensor(Y2, dtype=torch.float32)\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.feature_num\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.X[idx,:], self.Y1[idx,:], self.Y2[idx,:]\n",
    "\n",
    "\n",
    "class MultiTaskLossWrapper(nn.Module):\n",
    "    def __init__(self, task_num, model):\n",
    "        super(MultiTaskLossWrapper, self).__init__()\n",
    "        self.model = model\n",
    "        self.task_num = task_num\n",
    "        self.log_vars = nn.Parameter(torch.zeros((task_num)))\n",
    "\n",
    "    def forward(self, input, targets):\n",
    "\n",
    "        outputs = self.model(input)\n",
    "\n",
    "        precision1 = torch.exp(-self.log_vars[0])\n",
    "        loss = torch.sum(precision1 * (targets[0] - outputs[0]) ** 2. + self.log_vars[0], -1)\n",
    "\n",
    "        precision2 = torch.exp(-self.log_vars[1])\n",
    "        loss += torch.sum(precision2 * (targets[1] - outputs[1]) ** 2. + self.log_vars[1], -1)\n",
    "\n",
    "        loss = torch.mean(loss)\n",
    "\n",
    "        return loss, self.log_vars.data.tolist()\n",
    "\n",
    "\n",
    "class MTLModel(torch.nn.Module):\n",
    "    def __init__(self, n_hidden, n_output):\n",
    "        super(MTLModel, self).__init__()\n",
    "\n",
    "        self.net1 = nn.Sequential(nn.Linear(1, n_hidden), nn.ReLU(), nn.Linear(n_hidden, n_output))\n",
    "        self.net2 = nn.Sequential(nn.Linear(1, n_hidden), nn.ReLU(), nn.Linear(n_hidden, n_output))\n",
    "\n",
    "    def forward(self, x):\n",
    "        return [self.net1(x), self.net2(x)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANUAAAB0CAYAAAASAHfIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFAFJREFUeJztnX+QVNWVxz9nmgZ7JGFAdNURxU0R3FVZCEStglpLCaLGH6wpUbOlplzXWLoqao1gYilm3RKl4o/aLWuXTfyRXUvBVUdItFAhtRvZwsiIomAwJjHC4CYSHRRnhJ7ps3+8ftOvu9+v7n7v9evhfqooZl53v3tfz/u+c+65554rqorBYIiOtmZ3wGAYaRhRGQwRY0RlMESMEZXBEDFGVAZDxBhRGQwRY0RlMESMEZXBEDFGVAZDxIxqdgecTJw4USdPntzsbhgMrvT09OxW1UOD3pcqUU2ePJlNmzY1uxuppHtzL8vXbmdX3wBHduTomj+VBTM6m92tAwoR+X2Y96VKVAZ3ujf3cuszbzGQHwKgt2+AW595C6AuYY0Ugab1OsyYqgVYvnb7sKBsBvJDLF+7veZz2QLt7RtAKQm0e3NvRL1NhjRfhxFVC7Crb6Cm435EKdBmkubrMKJqAY7syNV03I8oBdpM0nwdRlQtQNf8qeSymbJjuWyGrvlTaz5XlAJtJmm+DiOqFmDBjE7uvuBEOjtyCNDZkePuC070HZR3b+5l9rL1HLvkZ8xetn54rHHace4R4cmHNP9mrIUoHzRRY6J/LcKCGZ1lIrJFUxn56t7cy51rtvJJf374vc5o4c9/9ZHr+f/3Nx/Tvbk3FdGzMNj9TGP0T9K0nH7WrFk60uepoggDV4bYwXpKf2tmJ0/39FYN4G06O3LsKkbLvF7fsOT0yEPVaQ1914qI9KjqrKD3GUuVIFHNN3lFvp54dQdDPg9J+6bu9RnkxzEnFuX5WgEzpkqQqMLAXhEuP0EBw1ZCfF6POlSd5tB3XBhRJUhUYWCvCFdGvORSGsQvmNHJ355ydJWw7Ne9+uJl3YJIc+g7LoyoEiSqMLBX5OuSkydVHQfoyGXLooV3LTiR+y+a7hpN9OqLQF3ZCmkOfceFGVMlSNf8qa4BhlrDwH6Rr1nHTAgVFKiMJjr7uGjlG1XHtdhereOgqK45SuIOnBhRJUiUYWAvUXgdj4J6XLa0hb6TCJwYUSVMnDd9FPgFEOp12dJ0zX6Bk9SISkQmAT8BDgcKwApVfVBEJgArgcnA+8BCVf2k0fYM8eJnjbxctrjcqTjOm0TgJApLNQjcrKqvi8iXgB4ReQn4DrBOVZeJyBJgCbA4gvZGLGmYJPWaxxrfnnXtSxzuVPfmXpau3krfgHtWiH3eer4vr+uLMnDScPRPVT9U1deLP38GvAN0AucDjxXf9hiwoNG2RjJJrA/yygd04hVZvOPc413PWes8VFAf7O/BKSi389b7fSWRMxjpmEpEJgMzgFeBP1PVD8ESnogcFmVbI424ff2wFqWWwEL35l7f7Ix6+uD2Pbid1+v7unPNVt++JxE4iUxUIjIWeBpYpKqfis9EZMXnrgKuAjj66KOj6k7qqXRdark566EW0YYJLNgC8cLNnQrTh6DrHZfL+r7vk/78cDKx34MjTrc6kslfEcliCepxVX2mePgPInJE8fUjgD+6fVZVV6jqLFWddeihgYVqRgRurotf6lAURD1A97MoXu5UmD4EXe/n+wfp3twb+ntpRkpUw6ISyyT9GHhHVe9zvLQauLz48+XAc422NVJwuyEVPFOHnIQZF7kRJrOhlnP7idFrrZdXHxSG2+uaP5Vsm7eXkx9Slq/d7jo2qqevcRCFpZoNXAqcLiJvFP+dDSwD5onIr4F5xd8NeP+RlVL+nttCRDcLd+PKN7it29sNszntuEN9RVvrwN9LIJ0dOU/Xyk8ITldt7EH+o5JdfQOuCzc7iq5h2L7GRcNjKlV9heqHrM3cRs8/EvEbQw2pliW/OvGycI9v/IBZx0zwvJm7N/fydE+vyzoq5caVb7B87XY+3zdYU6CknvQjZ5DA7frt9vr6qyN/TmyRuC3cTENKlEmobQJd86eSzXi7OF7jAD8L5zdu8Br/DOQLw1bJLYTt12Y9S/ztz21YcrrnU3hX3wAd7e4WB/xFUm+fosakKTWBBTM6qyY3K3G7meuNEjYyplDwXGbfSBTNbxL2832Drp8RvMdrUfQpKoylahJ7fAQF5YN3m6AFhl40OqboeurNyItU+o3x/L6bZgsmDEZUTSLMjV4ZLAhaYOhFLZEyN/IFDRWWDhs9dBvjCfCtmZ2+a7paZQ2WEVWTCHujV46v/BYYelE51hjfnq0KWwdN1Qe5kLVED70CLnalpzSXHwuDEVWTcBtUexHFPMuCGZ10zZ/KkR05PunPl9Wz6MhlXS2gkyArUUsOYNAkcFoCDvViAhVNpHJQPXvZ+sAM6nqzwis/V3D4XvsGC8w6ZgIA/7nxg6rPZtsk0ErUkrERJlM81oDDllWw7gewZyeMOwrm3g7TFkZ2emOpUkQYt6fe6kR+aUX25+9acCIPXDSd8Y6Qdkcuy/IL/yrUkoqwx5vq3m1ZBWuuhz07ALX+X3O9dTwijKWqkzjWPoXJoK43hy/s6/VaiFomg2PJFN+yCl5YDAMfFxufAGfdU22B1v0A8hXfRX7AOh6RtTKiqoM46xwE3dT1LrLzm+MK8/kgahVKJO7dsBu3o/q1gY/hmb+HDzbCOY6U1D073c/ldbwOjKjqIIk6B14EWQQvC9o1fypdT71JvlCdrCRYD4bZy9Y3ZDFCCaUoBN2zkz2MZaigjG/7nC9yh9N+lsNaBI17tqyC566Fof3+7W16GI4+pfTZcUe5izA3PvyFBmBEVQfNLBDpZxH8LCjgGTe3ZVarxQ3tApdZFAEUATr4bHhU3z7wIYPPXWfdkB9shE0/Ln1+z45qq/PC4mBB2VfndO3m3g7d10ChYoJ5/16rnxG4gC23QUEa6jh4RensAv/NwqtfGZHAktBOwlyHV/Lq3RecyILMhioRhSY3oTQucmPW31nCWjou/DkRWNpX+vWeY93bGDcJbnzb+ywjcYOCpIrdBwk3jQUiwbs0s5+gzmt7hVtGreJI2U0fY1GF8QN74f5JvPaV61i0bYrr97B87XbmDf03S8f8hPHsHT5f/3NjIKMOK1LjQ9tPUFBy52ph3FEVbXgU9YpoXNVSokpiLNO9ubds7NHbN0DXU28CJeFGEb2KY7uaIJvgFNAunci6wnQuzPwP7WIJYAJ7Sy7inh2c0HMbM/NX0sucqgfYrE9f4ofZf2WUFMraOJh94F1iIgKK7lyQRbPJ5iyXz4nXuKpSfHXSUu7fsUt+5nrTCPC7Zd+MpA/T73zRNXu8I5fljTvOiKQNX9epTmF5uX4257W9wrLsj4YFBNYEsM8iWwBU4WMdy52Dl7G6MMdyDc/ezdAzV5Gp1QoFoICEEovABSuqx0ZtWfjaZfDrF30ndl9b/W+c0HMbOcd3MaCjeXvmXXz9vO96tzoS3b+oarb5WQmv5Rh+yzRqJQ6LGxQkuWXUqjJBQbCgAETgENnLg9mHmDn0Lq9/+lVY80jkgoKikTzrHvdAgpNxR5WEUkdmxKJtU5iZv7Jotf/ELj2EewcX0rNtChvOa/w6WkpUUYxlohiXNeq6xRE9tB84lS7e+sJ0Tm97g07ZXfe5wRLXZZmXOX/Uq9WTp1EhmZIonBO5Tpzu3LSFdUXrdvUN0MscVu+fU958RNHblhJVFGOZICsxvj1btl+ujZ26E4Uoo7C4trBnfvoSi7Or+AW76R8zhoPZh10d7ijZzaVtLwdmoIdFBMbxWURnc0GLfxenWGLI04u7Sm1LiQoan4kPshJ3nHs8Xf/1JvmhknuTzchwhdawrpufNWvU4v7mke9y7vsrOR+FLMMiGsu+qvdGJajIzpebUIy+ubiP4yZVH6vTGvkRd/S25UTVKEFPqSBrGMZ1C7Jmnm1kNsD9VrbBH5jI3fsvZNOX55VeK879/DklISWNApLNVbuAow+Gwf1lY6GqsmvZnDVm+mCjFRp3CsstShcTcVepbanoXxQ0GnkLM/EbanK4MgHU5abs19E8NfTXnJt5lfHyWeRWpx6GEDIXrBh2yfpzh3Nv/iIe23sSl4/9JbdkV9I+8H+WqzblDO9IXMzLL+IgbPQvdlGJyJnAg0AG+JGqetb/S0JU0FigIUwmQaFvB7t0IvcOLmR1YU5Z8KCtY5J1s216BGvnIX9Um2eVKlGF/xj6Bpfd9TQQz9RAvSSRaZMKUYlIBngXq5jmTuA14BJV3eb2/qREVTeOZFBX92zN9WVukf3VKuHC13HiVgE38DOOW2OINh4fOp0VY6+tzSK7EMfEdxLiTss81UnAe6r622KnnsTaYsdVVKnGXtyWt+qeH85HPDj6IZAnIXOP6zod28KkwdDsI8NBFAibNtSvo1mSv5LVhVLYOZsRljsG8/VMDcSRatbMVQNuxC2qTsCZD7ITONn5htTv+uG3ZgeK63auouYctwTp19Hcmr+SBy+eAc9eXQpdV1DAShrvzx3BPw5cyOpCKcdufHuWO849fjgbfvna7Z5XbAd93CxSkhPfSddQt4lbVG4P6bK/haquAFaA5f7F3B+LoEHyT2+Cnkc9b75q0ikoZ4pRz5fnwTTLJRt87jpGDX1R9r5PGMtyuYInvjgF+aL8iuxN39yWl1Rih6a9LJLX56KY+HY73gzirlGxE3BOPhwF7Iq5TX+CahT89CZrLU9oQZWIO5CqWt1GZZOKldO3szCRG/LXMHP/Cl7KnFqag5m2kFHn/zP9uSMoIOwsTOR7cj0nD/47T3xxius5nTUw/GpdOKseeVmkjEfUpREBpK2kWdyW6jVgiogcC/QCFwPfjrlNf4JqFPQ8WveptZgmXk+0zhbL54xhrFRP4u7TDF15K9nTzln7ov1wPjzsVA7+/ToO0938USay42td9E46Z9jt6nQLBExbSPu0hcPumfWU938i2JbEy6IIDAcn/HZYtDdgiHLiNYndEWshVlGp6qCI/AOwFiuk/rCqbo2zzUCCahTUYaFsPtSJvDw0nUszL5dF+4LC4oPaxk35q8uywCuLmGz5iyX0FNc29bTPG74JLZeqlAWaey3D3ZOoa5GhH7YlCXK1gnZYzIhw9wUnxl40x7asIzFQgao+DzwfdzuhCVpLI5mQwipfvTTAGO7JW/NSPYWvlmVArytMZ24xqbUyvO6Msg0/saedTvfQ7NKNd1COrklT2XBe+Q0ye9n6ugf9QXvrOslmSnX/glJ8gs47pBpLTb+kFrCGofXSlBqdiZ97e9V8UlmKzMzvlNdHcGPcJOv9jn4s+ejc4fDz6kJ5BnS2TXhg9Cj6+vNlWQd2NsKafSeVuWlBN0i521ZNmEF/TYEBh2dYbxqXjV8l3kZIU1i9tUTlmCsCSkEGCC+soHU4dmGRnkdRHaoaIw3oaN7+ynV8vSLRc9Oy9eBRH6K8GOU3gTsBaAeWFv85CSqYGeS2hRn0B5Usc5IvKDevKq1+9rM0fueNM3iQprB6a1Wo9Qsy1MK0hVaBj6V91v+VgjznPrjjY+Yc9Cw35K9hZ2EiBbUiZYvzV7Jo25SqU3pFoH64MLi6ayV+N0iQexX2xvXqrxdDqr7blfqdF6yV03GmL6Vpp5DWslQJFEJ0UstitigjUH7BAL8nr2ukzwOv/vq5lWHcqWZF4tJUjKe1RBVzwY5Kap1UjGoA7neDeN309ZRH8+pvo5O0zdjNME1h9dYSVVCQIWJOO+5Q110wTjvu0JrOU2sCadANEucT2W7j5lVvupY2S/PGa2nYmhRaTVQNFPuoB3sTsrDH3ag31Ot1gyTxRE5CvCOZ1hIVxLK82osoIkpxhHqTeCKnyZ1qNVpPVAkSRaJmmkK9tZIWd6rVaK2QesJEkaiZplCvIRmMqHyIYu/ZtGVQG+LHuH8BNOoCmbHJgYcRVQKYscmBhXH/DIaIMaIyGCLGiMpgiBgjKoMhYoyoDIaIMdG/GkjDJt6G9GNEFZI01UAwpBvj/oUkaIm7wWBjRBWSVk6MNSRLQ6ISkeUi8isR2SIiz4pIh+O1W0XkPRHZLiLzG+9qczGJsYawNGqpXgJOUNVpWFvm3AogIn+JVY32eOBM4KHitjoti0mMNYSlIVGp6ouqOlj8dSNWrXSwtst5UlX3qervgPewttVpWaLIWDccGEQZ/bsCWFn8uRNLZDY7i8eqcG6lA+wVkaRG/hOB3fV++H3gb26NrC9J0dA1tyhRXvMxYd4UKCoReRk43OWl76vqc8X3fB8YBB63P+byftcK+M6tdJJERDaF2RVvJGGuORkCRaWq3/B7XUQuB84B5mppr9P0baFjMCREo9G/M4HFwHmq2u94aTVwsYiMKW6jMwX4ZSNtGQytQqNjqn8BxgAviVVwfKOqXq2qW0VkFdbevoPAtaoN7FETD4m7nCnAXHMCxLo7vcFwIGIyKgyGiDmgReWXETLSEJEzi9kt74nIkmb3J05EZJKI/FxE3hGRrSJyQ6LtH8jun4icAawvbqN6D4CqLm5ytyKnmM3yLjAPKzL7GnCJqm5rasdiQkSOAI5Q1ddF5EtAD7Agqes9oC2VT0bISOMk4D1V/a2q7geexMp6GZGo6oeq+nrx58+Ad/BIPoiDA1pUFVwBvNDsTsREJ+Dcg8gzw2WkISKTgRnAq0m1OeIXKdaZETLSCJ3hMpIQkbHA08AiVf00qXZHvKjqzAgZaRxwGS4iksUS1OOq+kyibY/c+yiYYkbIfcCpqhp+06kWQ0RGYQUq5gK9WIGKb6vq1qZ2LCbEykR4DPhYVRcl3v4BLqr3sDJC/lQ8tFFVr25il2JDRM4GHgAywMOq+k9N7lJsiMgc4BfAW0ChePh7qvp8Iu0fyKIyGOLARP8MhogxojIYIsaIymCIGCMqgyFijKgMhogxojIYIsaIymCIGCMqgyFi/h/hpw5Dn3kuUQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 216x108 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "np.random.seed(0)\n",
    "\n",
    "feature_num = 100\n",
    "nb_epoch = 2000\n",
    "batch_size = 20\n",
    "hidden_dim = 1024\n",
    "\n",
    "X, Y1, Y2 = gen_data(feature_num)\n",
    "pylab.figure(figsize=(3, 1.5))\n",
    "pylab.scatter(X[:, 0], Y1[:, 0])\n",
    "pylab.scatter(X[:, 0], Y2[:, 0])\n",
    "pylab.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = TrainData(feature_num, X, Y1, Y2)\n",
    "train_data_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size)\n",
    "\n",
    "model = MTLModel(hidden_dim, 1)\n",
    "\n",
    "mtl = MultiTaskLossWrapper(2, model)\n",
    "mtl\n",
    "\n",
    "# https://github.com/keras-team/keras/blob/master/keras/optimizers.py\n",
    "# k.epsilon() = keras.backend.epsilon()\n",
    "optimizer = torch.optim.Adam(mtl.parameters(), lr=0.001, eps=1e-07)\n",
    "\n",
    "loss_list = []\n",
    "for t in range(nb_epoch):\n",
    "    cumulative_loss = 0\n",
    "    \n",
    "    for X, Y1, Y2 in train_data_loader:\n",
    "\n",
    "        loss, log_vars = mtl(X, [Y1, Y2])\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        cumulative_loss += loss.item()\n",
    "\n",
    "    loss_list.append(cumulative_loss/batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHTZJREFUeJzt3XuYXHWd5/H3ty7d1Z3uTnenO0knIRegEwggtzYGiaAoiKBcRtdFXYkjI+rqKDuzM+LjszO6O7OPrqsMXlY3KmN0QYIoJDrrDBAhCCuRDkkgIZBOAiHpXLpz6UuSvlXXb/+o00mRdHVVX6pOnerP63nqOad+fSrnm1PVn/r179zMOYeIiARfyO8CRERkYijQRUSKhAJdRKRIKNBFRIqEAl1EpEgo0EVEioQCXUSkSCjQRUSKhAJdRKRIRPK5srq6Ojd//vx8rlJEJPA2bNhwyDlXn2m5vAb6/PnzaW5uzucqRUQCz8x2Z7OchlxERIqEAl1EpEgo0EVEioQCXUSkSCjQRUSKRFZHuZjZ60A3MAjEnXNNZlYLrALmA68DH3bOHc1NmSIiksloeujvcs5d4pxr8p7fDax1zjUCa73nIiLik/EMudwMrPTmVwK3jL+c4f36hb3cvz6rwzBFRCatbAPdAY+Z2QYzu9Nrm+Gc2w/gTacP90Izu9PMms2sub29fUxF/mbzPlY9v2dMrxURmSyyPVP0SufcPjObDjxuZq9kuwLn3ApgBUBTU9OY7khtZiR0M2sRkRFl1UN3zu3zpm3AI8AS4KCZNQB407ZcFWmA8lxEZGQZA93MpphZ5dA8cB2wBVgDLPcWWw6szlWRZqZAFxHJIJshlxnAI2Y2tPwDzrl/NbPngYfM7A7gDeDf5apIMzTkIiKSQcZAd87tAi4epv0w8O5cFHU6y8dKREQCLhBnioY05CIiklEgAl1DLiIimQUm0BXnIiIjC0igG049dBGREQUj0NFx6CIimQQi0ENmGnIREckgEIGunaIiIpkFI9DRkIuISCaBCPTkkIsSXURkJIEIdAwSCb+LEBEpbIEIdNPJ/yIiGQUi0EOGjkMXEckgEIGePMrF7ypERApbIAJdO0VFRDILRKCrhy4iklkgAh10+VwRkUwCEeghA11vUURkZIEIdA25iIhkFoxAR5fPFRHJJBCBHtINLkREMgpEoJsZCY25iIiMKBCBDuqhi4hkEohAD+mmoiIiGQUi0HWDCxGRzAIR6NopKiKSWSAC3czUQxcRySAYgY5uQScikkkwAt1MQy4iIhkEJNB1gwsRkUyCEehoyEVEJJNABHpIQy4iIhkFItB1HLqISGZZB7qZhc1so5n91nu+wMzWm1mLma0ys5JcFakhFxGRzEbTQ/8isC3l+TeAe5xzjcBR4I6JLCyVmQHaMSoiMpKsAt3M5gA3Aj/2nhtwDfCwt8hK4JZcFJhcX3KqPBcRSS/bHvo/AX8LJLzn04AO51zce74XmD3cC83sTjNrNrPm9vb2sRU51EMf06tFRCaHjIFuZu8H2pxzG1Kbh1l02Lx1zq1wzjU555rq6+vHVOTQyrRjVEQkvUgWy1wJ3GRmNwAxoIpkj73azCJeL30OsC9XRWrIRUQks4w9dOfcl51zc5xz84HbgN875z4GPAl8yFtsObA6V0We3CmqQRcRkbTGcxz6l4C/MrMdJMfUfzIxJZ1JPXQRkcyyGXI5yTn3FPCUN78LWDLxJZ3JGDpsMR9rExEJpkCcKRoa6qFryEVEJK1ABPrQkEtCeS4iklYgAj2kM0VFRDIKRKAPUQ9dRCS9QAS6nTzMxd86REQKWSACXTtFRUQyC0Sgnzr139cyREQKWjACXTtFRUQyCkSghzSELiKSUSACfehAdF1tUUQkvUAEekjXchERySgQgR72euiD2isqIpJWIAI9FFKgi4hkEohAD2sMXUQko2AEunroIiIZBSLQh4Zc1EMXEUkvEIF+aqeoz4WIiBSwYAS6V6WGXERE0gtEoIe0U1REJKNABLp2ioqIZBaIQD95HLp66CIiaQUi0E8eh64euohIWoEI9IjXQ48r0EVE0gpEoJ88Dl2BLiKSViACPawxdBGRjAIR6CFdbVFEJKNABHpYp/6LiGQUjEDXqf8iIhkFItBDOvVfRCSjQAS6hlxERDILRqBrp6iISEYZA93MYmb2JzPbbGZbzexrXvsCM1tvZi1mtsrMSnJWpHroIiIZZdND7wOucc5dDFwCXG9mS4FvAPc45xqBo8AduSpSPXQRkcwyBrpLOuY9jXoPB1wDPOy1rwRuyUmF6GqLIiLZyGoM3czCZrYJaAMeB3YCHc65uLfIXmB2bkrUkIuISDayCnTn3KBz7hJgDrAEOH+4xYZ7rZndaWbNZtbc3t4+piJ1cS4RkcxGdZSLc64DeApYClSbWcT70RxgX5rXrHDONTnnmurr68dUZGkkWWbfgM4sEhFJJ5ujXOrNrNqbLwPeA2wDngQ+5C22HFidqyJj0TAAvfHBXK1CRCTwIpkXoQFYaWZhkl8ADznnfmtmLwMPmtk/ABuBn+SqyKEeeq966CIiaWUMdOfci8Clw7TvIjmennNmRmkkRO+AeugiIukE4kxRgLKSsAJdRGQEgQn0WESBLiIykuAEejREj8bQRUTSClCgq4cuIjKSwAR6VSxKZ8+A32WIiBSswAR6fWUph7r7/C5DRKRgBSrQ2xXoIiJpBSrQu/vi9PRrHF1EZDiBCnSAtu5enysRESlMgQn0ebXlALx26LjPlYiIFKbABPrCGZUAtBw8lmFJEZHJKTCBXjOlhLqKElrauv0uRUSkIAUm0AEap1eyXT10EZFhBSrQF86oYEfbMZxuRScicoZABfq5Myo51hdnf6eOdBEROV2gAv38mckdo1taO32uRESk8AQq0C+cPZVo2NjwxlG/SxERKTiBCvRYNMyFs6ey4XUFuojI6QIV6ABLFtSyeW8Hx/vifpciIlJQAhfoVzXWMzDoeG7XYb9LEREpKIEL9Kb5NZSXhPn9K21+lyIiUlACF+ilkTDvaKzj96+06Xh0EZEUgQt0gGsXz2R/Zy9bWrv8LkVEpGAEMtCvOW86IYP/u2W/36WIiBSMQAZ67ZQS3rVoOo+80KphFxERTyADHeCGixo40NXLpj0dfpciIlIQAhvo114wg9JIiF+9sNfvUkRECkJgA70qFuXGixp4dOM+nWQkIkKAAx3go2+by7G+OL/ZvM/vUkREfBfoQL98Xg0LZ1TwwJ/e8LsUERHfBTrQzYyPvW0eL+7t5KW9uqSuiExugQ50gFsunU0sGuL+9bv9LkVExFcZA93MzjKzJ81sm5ltNbMveu21Zva4mbV405rcl3umqWVRbrp4Fqs37aOzZ8CPEkRECkI2PfQ48NfOufOBpcDnzGwxcDew1jnXCKz1nvvi9ivm0zMwyC+b9/hVgoiI7zIGunNuv3PuBW++G9gGzAZuBlZ6i60EbslVkZlcOHsqS8+u5cd/eI3+eMKvMkREfDWqMXQzmw9cCqwHZjjn9kMy9IHpE13caHz2nedyoKuXRze1+lmGiIhvsg50M6sAfgXc5ZzL+jKHZnanmTWbWXN7e/tYaszKVY11LG6o4ofrdpJI6PouIjL5ZBXoZhYlGeb3O+d+7TUfNLMG7+cNwLB3nHDOrXDONTnnmurr6yei5nQ18tl3nsOu9uM89vKBnK1HRKRQZXOUiwE/AbY5576d8qM1wHJvfjmweuLLG50bLmpg3rRyfvDUTl2FUUQmnWx66FcCHweuMbNN3uMG4OvAtWbWAlzrPfdVOGR8+qpz2Ly3kz/u1D1HRWRyiWRawDn3DGBpfvzuiS1n/P7sstnc88R2frBuJ28/t87vckRE8ibwZ4qeLhYNc8eyBfyh5RCbda10EZlEii7QAT72trlMLYvy3d/v8LsUEZG8KcpAr4xF+eSVC3hi20Fe3KteuohMDkUZ6ACfXDafmvIo3/y3V/0uRUQkL4o20CtjUf7jO8/lDy2HeHp77k5oEhEpFEUb6AAfv2IeDVNj/JfVWxgY1DVeRKS4FXWgx6Jh/uGWC9l9+ASrnteVGEWkuBV1oANcc9503jq/hnvXtnBMN5MWkSJW9IFuZnz5hvM5dKyP/6kdpCJSxIo+0AEum1vD7UvnsfKPr7Nh91G/yxERyYlJEegAf3P9eTRUxbj7Vy8yqMvrikgRmjSBXlEa4e4bzqel7Rj//OxrfpcjIjLhJk2gA7z/ogbetaiebz22ndaOHr/LERGZUJMq0EMh47/dciEAX12zVddMF5GiMqkCHWBOTTl3vaeRx18+yCMbdf9RESkeky7QAf7iHWezZH4tf7d6K28cPuF3OSIiE2JSBno4ZNxz2yWYwRdXbdRlAUSkKEzKQAeYXV3Gf7/1Ija+0cF317b4XY6IyLhN2kAH+MDFs/jgZXP43pM7WL9L9yAVkWCb1IEO8LWbL2BubTlfeHAj7d19fpcjIjJmkz7QK0ojfP9jl9FxYoDPP/CCxtNFJLAmfaADXDBrKl//4EWsf+0IX12z1e9yRETGJOJ3AYXi1kvn8MqBbv73ul3UVZTyn65d6HdJIiKjokBP8bfvPY/2rj7uXdvCBbOquO6CmX6XJCKSNQ25pAiHjH+89SIuPquav/zFRl1qV0QCRYF+mrKSMPctb2Lm1Bh3rHyeHW3H/C5JRCQrCvRhTKso5eeffBthMz76o+cU6iISCAr0NOZOK+cXdy4l4eC2FX/klQNdfpckIjIiBfoIFs6oZNWnlxIOGR9Z8RxbWjv9LklEJC0Fegbn1Ffw0KevoLwkwkd/9Byb9nT4XZKIyLAU6FmYN20Kqz69lOryEv7Dj9fT/PoRv0sSETmDAj1Lc2rKWfXppUyvLOX2+/7E09vb/S5JRORNMga6md1nZm1mtiWlrdbMHjezFm9ak9syC0PD1DIevHMps6vL+MQ//4n71+/2uyQRkZOy6aH/FLj+tLa7gbXOuUZgrfd8UpheFePhz76dK86Zxlce2cK9T7SQSOjepCLiv4yB7px7Gjh90PhmYKU3vxK4ZYLrKmhTy6Lc94m3cuuls7nnie186mfNdPYM+F2WiExyYx1Dn+Gc2w/gTadPXEnBUBoJ8+0PX8xXP7CYddvbueX7z/LqgW6/yxKRSSznO0XN7E4zazaz5vb24tqRaGZ84soFPPCppXT3xvnA957hl817/C5LRCapsQb6QTNrAPCmbekWdM6tcM41Oeea6uvrx7i6wrZkQS3/8oVlnF03hb95+EX++qHNGoIRkbwba6CvAZZ788uB1RNTTnDNqIqx5vPL+MzV5/DrjXu56XvP6CQkEcmrbA5b/AXwR2CRme01szuArwPXmlkLcK33fNIriYS4+33n8fBnrmAgnuDW//Usf796C8f74n6XJiKTgDmXv0PumpqaXHNzc97W56fOngG+/dir/Oy53TRUxfivN1/IexbP8LssEQkgM9vgnGvKtJzOFM2RqWVRvnbzhTz8mSuojEX5i58185mfb6Ctq9fv0kSkSCnQc+zyebX89gvL+NL15/Hkq21c86113PtEi4ZhRGTCKdDzIBoO8dl3nsO/3nUVy86t454ntnP1N59k5f97nf54wu/yRKRIKNDzaEHdFH748ct59HNX0ji9kr9fs5V3f/spHt3YqssHiMi4KdB9cMlZ1Tzwqbex8pNLqCyNcteqTdz43Wd48pU28rmTWkSKiwLdJ2bG1Qvr+e1fLuPe2y7heF+cP//p89z4nWdYs3kf8UENxYjI6OiwxQLRH0/w6KZWfrhuJ7vajzO3tpzPX3MuN108i1g07Hd5IuKjbA9bVKAXmETC8djLB/nO2hZe3t9FRWmE5W+fx0eWzGVOTbnf5YmIDxToAeec49kdh1nxh108vb2dkMF7L5jJR5bMZdm5dYRC5neJIpIn2QZ6JB/FyOiZGcsa61jWWMeeIyf4P+t3s+r5PfxuywFmV5fx7996Fh+6fA6zqsv8LlVECoR66AHSFx/ksa0HefD5N3h2x2EAlsyv5QOXzOLGixqonVLic4Uikgsacilyuw8fZ82mfazevI8dbceIhJI9+psunsV7Fs+gKhb1u0QRmSAK9EnCOce2/d2s2byP32zeR2tHD5GQcdm8Gq5eWM/VC+tZ3FClMXeRAFOgT0KJhGPjnqOs3dbGuu3tbN3XBUBdRQlXNdZz9aJ6lp1bx7SKUp8rFZHRUKALbd29/GH7IZ5uaefp7e0cPTGAGbxl9tRk731RPRfPqSYS1vllIoVMgS5vMphwbGntZN32dtZtb2fjG0dJOKiKRVjWWMfVC+u5dG4N59RXENbwjEhBUaDLiDpPDPDMjkOs254cnjnY1QdATXmUy+fVcvm8GpYsqOWCWVU6U1XEZzoOXUY0tTzKjW9p4Ma3NOCcY0fbMV7c28mzOw+xeU8HT2w7CEA4ZDROr+DSuTU0zavhgtlVnFtfoWEakQKkHroM60BnL5v3dvDS3k5eau3khTeO0t2bvClHLBpicUMVF8yayqKZlTROr2DRzEqqy3UcvEguqIcu4zJzaoyZU2fy3gtmAskx+B1tx3h5fycv7e1iS2snj2xs5VjKnZfqK0tZOKOCxumVNM6oYOGMShZOr2RquY6JF8kHBbpkJRwyFs2sZNHMSm69NNmWSDh2HznB64eO09LWzfaDx2g52M1DzXs40T948rWVsQjnN1SxaEYl86aVM2/aFBbUJadRDd2ITBgFuoxZKGQsqJvCgropvOu86SfbEwnHvs4eWg4eY/vBbl7e38WeIyfO6NGHDGZWxZhdU8as6jJmV3vTmuT87OoyppTqIyqSLf22yIQLhYw5NeXMqSl/U9A75+g4McBrh4+z+/BxdrUfp7Wjh9ajPbzwxlH+5cX9xE+7Fd/UsujJoJ9TU8as6hizq8u9aRl1FaU6C1bEo0CXvDEzaqaUUDOlhMvm1pzx88GEo727j9aOE7R29NJ6tId9HT20dvSw9+gJ1u86THdKDx8gGjZmVMWoqyilujwZ/tOmlFBXWUp9RSn1laXUVZRSW1FCZWkEM4W/FC8FuhSMcMi8nbExLp83/DJdvQO0Hk326vd39tDa0cuBzh4OdPWyr6OHTXs66OwZYLiDt6Jho6a8hNopZz5qykuoLo9SURphSmmEyliEqliUqliUilhEJ1tJICjQJVCqYlGqGqKc31CVdpnBhOPw8T4OdffT1t3L4WP9HDnez5ET/Rw55k2P97N1XxdHjvfT2TOQcb1TSsJUxqJUxiLeI0pV2annVak/K33zzypjUcpLwkRCpr8QJKcU6FJ0wiFjemWM6ZUxFpM++IcMDCbo7Bmg48QAx/viHO+L09Ubp6t3gO7eON1nTOMcPdHPG0dO0N07QFdvnP545pt6l0RClJeEKYuGiXmPsmjIm4aJlYSJRcKUlYROW8abeu2lXltZ6s9LQpSEQ0TDyXXoi2NyUqDLpBcNh6irSI61j1VffJDu3jhdPadCf+gLoKt3gON9g5zoj9M7MEjPwCA9Awl6BwaTz/sH6eodoKd/kN6BBD1D7QODww4dZWIGpZFkuA9No+EQJd58SdgoiZx6HgmFiIYtOR82oqEQ0YidbI94r4+GkvORkBEKGdGwEQ6dWj5kyechG3okv1xDXlvYawuFhpbjzNeESC6X+prQqeXMIBJKfmkN/V+NU//W6TvIBxNuUg2XKdBFJkBpJExpRXhcXwqnc87RF0/Q54V8atD39g/SGx+kp//UF8CJ/jh9Awn6BxP0xRP0xxMMDJ6aDgwm/72htt6BBMd64/QPOuKDp5aJJxLEBx39g8lpPJFsD5KhDE84KAmHcDgioeQ5D9Hwm4e+kl8KyS8EgL54glg05H0ZhAh7XyhGcse+WfI1J1/Pm78wTv/jaOjL6L7lb2XutNze6F2BLlKgzOzksMtU/D3b1jlHPOGIDzoGnWNwKPgT7uQXwWDC4Vzy54kEJFyyLeGcN59sSyQcCYe3nEtZLrvXxAcTb/rrxblTr004YGhKMtj7vS+jwURyWCz1y8k5h/cSHMnXlUZC9MUThM2IJxI4l+zpn1wu5c+m07/mTr+UiiP5pZJIOEqjuT+JToEuIhmZmTcs43clMhKddy0iUiTGFehmdr2ZvWpmO8zs7okqSkRERm/MgW5mYeD7wPuAxcBHzGzxRBUmIiKjM54e+hJgh3Nul3OuH3gQuHliyhIRkdEaT6DPBvakPN/rtb2Jmd1pZs1m1tze3j6O1YmIyEjGE+jDHa1/xsGqzrkVzrkm51xTfX39OFYnIiIjGU+g7wXOSnk+B9g3vnJERGSsxhPozwONZrbAzEqA24A1E1OWiIiM1rhuEm1mNwD/BISB+5xz/5hh+XZg9xhXVwccGuNrc0l1jY7qGh3VNTqFWheMr7Z5zrmMY9bjCvR8MrPmbO56nW+qa3RU1+iortEp1LogP7XpTFERkSKhQBcRKRJBCvQVfheQhuoaHdU1OqprdAq1LshDbYEZQxcRkZEFqYcuIiIjCESg+3VVRzM7y8yeNLNtZrbVzL7otX/VzFrNbJP3uCHlNV/26nzVzN6b4/peN7OXvBqavbZaM3vczFq8aY3Xbmb2Ha+2F83sshzVtChlu2wysy4zu8uPbWZm95lZm5ltSWkb9fYxs+Xe8i1mtjxHdX3TzF7x1v2ImVV77fPNrCdlu/0w5TWXe+//Dq/2cd1rLU1do37fJvr3NU1dq1Jqet3MNnnt+dxe6fLBv8+Yc66gHySPcd8JnA2UAJuBxXladwNwmTdfCWwneWXJrwL/eZjlF3v1lQILvLrDOazvdaDutLb/Adztzd8NfMObvwH4HclLNiwF1ufpvTsAzPNjmwFXAZcBW8a6fYBaYJc3rfHma3JQ13VAxJv/Rkpd81OXO+3f+RNwhVfz74D35aCuUb1vufh9Ha6u037+LeDvfNhe6fLBt89YEHrovl3V0Tm33zn3gjffDWxjmAuQpbgZeNA51+ecew3YQbL+fLoZWOnNrwRuSWn/mUt6Dqg2s4Yc1/JuYKdzbqSTyXK2zZxzTwNHhlnfaLbPe4HHnXNHnHNHgceB6ye6LufcY865uPf0OZKX0kjLq63KOfdHl0yFn6X8XyasrhGke98m/Pd1pLq8XvaHgV+M9G/kaHulywffPmNBCPSsruqYa2Y2H7gUWO81fd77s+m+oT+pyH+tDnjMzDaY2Z1e2wzn3H5IfuCA6T7VBsnLQaT+ohXCNhvt9vFju32SZE9uyAIz22hm68zsHV7bbK+WfNQ1mvct39vrHcBB51xLSlvet9dp+eDbZywIgZ7VVR1zWoBZBfAr4C7nXBfwA+Ac4BJgP8k/+SD/tV7pnLuM5E1GPmdmV42wbF5rs+T1fW4Cfuk1Fco2SyddHfnebl8B4sD9XtN+YK5z7lLgr4AHzKwqj3WN9n3L9/v5Ed7cacj79homH9IumqaGCastCIHu61UdzSxK8s263zn3awDn3EHn3KBzLgH8iFNDBHmt1Tm3z5u2AY94dRwcGkrxpm1+1EbyS+YF59xBr8aC2GaMfvvkrT5vZ9j7gY95wwJ4QxqHvfkNJMenF3p1pQ7L5KSuMbxv+dxeEeDPgFUp9eZ1ew2XD/j4GQtCoPt2VUdvfO4nwDbn3LdT2lPHnm8Fhva+rwFuM7NSM1sANJLcEZOL2qaYWeXQPMmdalu8Gob2ki8HVqfUdru3p30p0Dn0Z2GOvKnnVAjbLGV9o9k+/wZcZ2Y13nDDdV7bhDKz64EvATc5506ktNdb8naPmNnZJLfPLq+2bjNb6n1Ob0/5v0xkXaN93/L5+/oe4BXn3MmhlHxur3T5gJ+fsfHs5c3Xg+Te4e0kv22/ksf1LiP5p8+LwCbvcQPwc+Alr30N0JDymq94db7KOPeiZ6jtbJJHEGwGtg5tF2AasBZo8aa1XruRvAfsTq/2phzWVg4cBqamtOV9m5H8QtkPDJDsBd0xlu1Dckx7h/f48xzVtYPkOOrQ5+yH3rIf9N7fzcALwAdS/p0mkgG7E/ge3omCE1zXqN+3if59Ha4ur/2nwGdOWzaf2ytdPvj2GdOZoiIiRSIIQy4iIpIFBbqISJFQoIuIFAkFuohIkVCgi4gUCQW6iEiRUKCLiBQJBbqISJH4/3NL7bBtbXYvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pylab.plot(loss_list)\n",
    "pylab.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4.2984442710876465, -0.2037072628736496]\n"
     ]
    }
   ],
   "source": [
    "print(log_vars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[8.578183137529612, 0.9031617364804738]\n"
     ]
    }
   ],
   "source": [
    "# Found standard deviations (ground truth is 10 and 1):\n",
    "print([math.exp(log_var) ** 0.5 for log_var in log_vars])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}