experiments/convergence/convergence_mnist_64workers_1000ms_seed1337_largeffn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import random\n",
    "import threading\n",
    "from functools import partial\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torchvision import datasets, transforms\n",
    "from tqdm import tqdm\n",
    "\n",
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from dmoe_emulator import EmulatedDMoE, get_non_expert_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 1337\n",
    "torch.manual_seed(seed)\n",
    "np.random.seed(seed)\n",
    "random.seed(seed)\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "layer_dim = 1024\n",
    "num_blocks = 4\n",
    "\n",
    "batch_size = 4\n",
    "num_trainers = 64\n",
    "\n",
    "delay_ms = 1000\n",
    "\n",
    "eval_interval = 1024\n",
    "total_steps = eval_interval * 20\n",
    "update_every_steps = 10\n",
    "\n",
    "in_features = 28 ** 2\n",
    "num_classes = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/20480 [00:00<?, ?it/s]"
     ]
    }
   ],
   "source": [
    "class FeedforwardBlock(nn.Module):\n",
    "    def __init__(self, hid_dim):\n",
    "        super().__init__()\n",
    "        self.layers = nn.Sequential(\n",
    "            nn.Linear(hid_dim, 4 * hid_dim), nn.LayerNorm(4 * hid_dim), nn.ReLU(inplace=True),\n",
    "            nn.Linear(4 * hid_dim, 4 * hid_dim), nn.LayerNorm(4 * hid_dim), nn.ReLU(inplace=True),\n",
    "            nn.Linear(4 * hid_dim, hid_dim),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return x + self.layers(x)\n",
    "    \n",
    "Optimizer=partial(torch.optim.Adam, lr=1e-3, amsgrad=True)\n",
    "emulate_latency = lambda: time.sleep(delay_ms / 1000. * np.random.weibull(1))\n",
    "\n",
    "model = nn.Sequential(\n",
    "    nn.Linear(in_features, layer_dim),\n",
    "    *(FeedforwardBlock(layer_dim) for _ in range(num_blocks)),\n",
    "    nn.LayerNorm(layer_dim), nn.Linear(layer_dim, num_classes)\n",
    ").to(device)\n",
    "\n",
    "# create optimizer for non-expert params only\n",
    "non_expert_params = get_non_expert_params(model)\n",
    "opt = Optimizer(non_expert_params)\n",
    "\n",
    "# create initial gradients\n",
    "model(torch.zeros(1, in_features, device=device)).sum().backward()\n",
    "opt.zero_grad()\n",
    "\n",
    "\n",
    "lock_model = threading.Lock()\n",
    "need_to_eval = threading.Event()\n",
    "train_history, val_history = [], []\n",
    "progress = tqdm(total=total_steps)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAr8AAAF1CAYAAADhgoKhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdeXyU1bkH8N+TPWQDkhh2wqqyVyKyCFKXiuLVtrZu11o3qF2s3W5rW1utdbtab+vWKlVxB1G7YEFQkWCAAGELJEDIQvZ935NZzv1jJpBtktnfd2Z+38/HNpl5532fLMz8cuY554hSCkREREREgSBI6wKIiIiIiLyF4ZeIiIiIAgbDLxEREREFDIZfIiIiIgoYDL9EREREFDAYfomIiIgoYDD8ku6JyMsi8jsnH5sqIve6uyYiokAgIitFpFTrOojcKUTrAsi/iUghgHuVUp87ew6l1H3uq4iIiIgCGUd+SVMiwj/AiIjIYXz9IGcx/JLHiMjbACYB+FhEWkXklyKSLCJKRO4RkWIAX1iP/UBEKkWkSUS+FJHZvc7zhog8Zv14pYiUisjPRaRaRCpE5C476wkSkYdEpMj62LdEJM56X4SIvCMidSLSKCIZIpJkve9OESkQkRYROSMi/+3mbxURkceIyK9E5MN+tz0nIs9bP75LRE5an+MKROR7Dpz7OREpEZFmETkkIst73RcsIr8RkXzruQ+JyETrfbNF5DMRqReRKhH5jfX2s8/31s/7tF2ISKH16zkGoE1EQkTkwV7XOCEi3+hX45peX98JEblIRP5HRD7qd9zzIvKcvV87+S6GX/IYpdR3ABQD+C+lVLRS6uled18G4EIAV1s//wTADADnATgM4N0hTj0GQByA8QDuAfCSiIyyo6Q7rf99FcBUANEAXrTe913rOScCiAdwH4AOEYkC8DyAa5RSMQCWAjhqx7WIiPRiI4BrRSQGsIRSADcBeM96fzWA6wDEArgLwJ9F5CI7z50BYAGA0dbzfSAiEdb7fgbgVgDXWs99N4B2ax2fA9gGYByA6QB2OPD13ApgNYCRSikjgHwAy2F5Dv8DgHdEZKz1a/02gEcA3GGt4XoAdQDeAbBKREZajwsBcAuAtxyog3wUwy9p5RGlVJtSqgMAlFKvK6ValFJdsDxRze8ZlR2EAcCjSimDUmorgFYA59txzf8G8H9KqQKlVCuAXwO4xfqkZ4Al9E5XSpmUUoeUUs3Wx5kBzBGRSKVUhVIq29kvmojI25RSRbAMKvSMiF4OoF0ptc96/xalVL6y2AXgU1jCpD3nfkcpVaeUMiqlngUQjnPPx/cCeEgplWM9d6ZSqg6WoF2plHpWKdVpfe7f78CX9LxSqqTX68cHSqlypZRZKfU+gFwAi3rV8LRSKsNaQ55SqkgpVQHgSwDfth63CkCtUuqQA3WQj2L4Ja2U9HxgfWvsKevbVs0ACq13Jdh4bJ31r/0e7bCM4g5nHICiXp8XwTLpMwnA2wC2A9goIuUi8rSIhCql2gDcDMtIcIWIbBGRC+y4FhGRnrwHy4gpANyGc6O+EJFrRGSftQWhEZaRWlvPv32IyC+sLQVN1sfG9XrsRFhGZfuzdbu9Snp/IiJ3iMhRa8taI4A5dtQAAG8CuN368e2wvA5QAGD4JU9Tdtx+G4AbAFwJyxNnsvV2cXMt5QAm9/p8EgAjgCrrKPIflFKzYGltuA6Wt8mglNqulLoKwFgApwD83c11ERF52gcAVorIBFhGgN8DABEJB/ARgD8BSFJKjQSwFXY8/1r7e38JSwvFKOtjm3o9tgTAtEEeWgJL69lg2gCM6PX5mEGOOfv6ISKTYXlO/hGAeGsNWXbUAAD/AjBPRObA8pw/VLsd+RGGX/K0Kth+kusRA6ALlj6sEQCe8FAtGwD8VESmiEi09TrvK6WMIvJVEZlr7YVrhqUNwiwiSSJyg7X3twuWFguzh+ojIvIIpVQNgFQA6wGcUUqdtN4VBkurQg0Ao4hcA+Brdp42BpYBhBoAISLye1j6anu8CuCPIjJDLOaJSDyA/wAYKyI/EZFwEYkRkUusjzkKS3/yaBEZA+Anw9QQBUsYrgEsk/dgGfntXcMvRGShtYbp1sAMpVQngA9h+UPggFKq2M6vm3wcwy952pMAHrK+HfULG8e8BUsLQhmAEwD2eaiW12F5W+tLAGcAdAK433rfGFieBJsBnASwy3psECyTNsoB1MMyUe/7HqqPiMiT3oPlHbazLQ9KqRYAPwawCUADLO/EbbbzfNthmbR2Gpbn8E70bUn4P+t5P4XlufU1AJHWa14F4L8AVMLSo/tV62PeBpAJS/vbpwDeH6oApdQJAM8CSIdlsGUugD297v8AwOPWr7kFltHe0b1O8ab1MWx5CCCilK13pYmIiIj8l4hMgqWdbUyvSc7k5zjyS0RERAFHRHre2dvI4BtYhg2/IvK6WDYEyOp122jr4tS51v+3Z41VIiIiIs1Z53E0w9J+8bDG5ZCX2TPy+wYs69/19iCAHUqpGbAsTP2gm+siIiIi8gjrOvPRSqnZSqmS4R9B/sSunl8RSQbwH6XUHOvnOQBWKqUqrLuopCql7NlkgIiIiIhIM872/CZZd0cBLDM1k9xUDxERERGRx4S4egKllBIRm8PHIrIWwFoAiIqKWnjBBY5tjtXaZcSZ2janaps73tbuuEREjjl06FCtUipR6zq8KSEhQSUnJ2tdBgCgra0NUVFRWpcBQF+1APqqR0+1APqqR0+1AP5fz1DP2c6G3yoRGdur7aHa1oFKqXUA1gFASkqKOnjwoEMX2p1bi9tfc2TL73MOPrXaqccREfUnIkXDH+VfkpOT4ehztqekpqZi5cqVWpcBQF+1APqqR0+1APqqR0+1AP5fz1DP2c62PWwG8F3rx98F8G8nz0NERERE5DX2LHW2AZadU84XkVIRuQfAUwCuEpFcWHaLecqzZRIRERERuW7Ytgel1K027rrCzbUQEREREXkUd3gjIiIiooDB8EtEREREAYPhl4iIiIgCBsMvEREREQUMhl8iIiIiChgMv0REREQUMHQffkW0roCIiIiI/IXuwy8RERERkbvoPvwq5fxjDSaz+wohIiIiIq/pMprw+YkqNLZ3u/W8ug+/rnhhR67WJRARERGRnXoC78/eP4qUP36Oe986iO3ZlW69xrDbG/uyiqZOrUsgIiIioiF0GU04Um3E5veP4rMTVWjpMiI2IgSr5ozBtfPGYtm0BLdez6/DLxERERHpT5fRhLTTtdh6vOJs4I2LrO4TeMNCPNOgwPBLRERERB43eOANxao5YzARNbjvG5d7LPD2xvBLREREREMqqW/Hf45VnF1MQHBuOVrptS7t2dsgvT4Gcipb+gTea+aOwbVzx2KpdYQ3NTXVK8EXYPglIiIiIhsKa9vw0s48/ONIGUxm55fg6h14l01PQGiwdmsuMPwSERERUR8FNa14cWce/n20HCFBgjuWTMbaFVNxXkwElFLoicE9S9IqqD7L0/a+HQDCQ4IRHKSPncsYfomIiIgIAJBb1YIXd+bh48xyhIUE4a6lyVh7mSX0nqOPEOss3Ydfbm9MRERE5FmnKpvxwhd52Hq8ApGhwVizYirWLJ+KhOhwrUtzO92HXyIiIiLyjBPlzXjhi1x8klWJ6PAQ/GDlNNxz6VSMjgrTujSPYfglIiIiCjCFTSaseesgPjtRhZjwEPz48um4+9IpGDnCf0NvD78Ov2yZICIiIk9SSqG1y4iYiFCtSxlAKYVOgxnNnQY0dRjQ3GFAXVs3NmWUYMepTsRGGPHTK2fizmXJiIvUX/2e4tfhl4iIiMhTWruMuO/tQ9idV4tpiVFYOi0BS6bFY/HUeI+1DbR1GZFd3oy86lY0dnSjucPYJ9w2dxrR0mH9vNMAg2ng8mQjR4TimzNC8ch/fxWxOgztnqb78KucX1KOiIiIyCMa2rpx5/oDyCpvxl3LklFQ04aPDpfi7X1FAIALxsRg8dR4SxieEu/UNdq7jThR3ozjZU04XtqE42VNyKtp7ZONwoKDEBsZitjIEMRGhCIuMhQTR0UiLjLUcnuE5b4468cxESGYmRSDjPTdARl8AR8Iv0RERER6UtHUge+8dgDF9e145faFuHJWEgDAYDLjWGkT9hXUIT2/DhszivHG3kKIAJNignBV6wksmRaPRVNGD2iT6Og24URFM7LKmnCstAnHyxqRV92Knn0lEmPCMW98HFbPG4u54+NwwdhYxEeFITwkqM8OazQ8hl8iIiIiO52pbcPtr+5HU4cBb929CIunnhvVDQ0OwsLJo7Bw8ij88KvT0WU0IbOkCen5dfjkcB7e2leEV3efQZAAc8fHYdGU0WhsN+B4WRNyq1vP7qCWEB2GuePjsGrOWMwbH4e5E+KQFBthqyRyEMMvERERkR2yy5vw3dcPQClg49rFmDM+bsjjw0OCsWjKaCyaMhrzQ8qweNlyHC5qQLp1ZHj9nkLERYZi7oQ4XDUrCXOtQXdMbARHcz2I4ZeIiIhoGAfO1OOeNzIQExGCt++9BNMSox0+R0RoMJZOT8DS6QkALG0SIUHCoOtlDL9EREREQ/jiVBW+/85hjB8VibfvuQTjR0a65byhwUFuOQ85Rvfhl38MERERkVb+fbQMP9+UiQvGxuDNuxYh3g+3+w00ug+/rhAwORMREZFz3kovxMObs7EoeTRe/W6KLjeyIMf5dfglIiIicpRSCi9+kYdnPzuNKy9Mwou3fQURocFal0VuwvBLREREZGU2Kzy25SRe33MG37xoPJ6+cR5C2JvrVxh+iYiIiAAYTWb86qPj+OhwKe5alozfrZ6FoCC2UPobhl8iIiIKeJ0GE+7fcASfnajCz66aifsvn84lyPwUwy8REREFtC6jCXetz0B6QR0evWE27liSrHVJ5EEMv0RERBTQPs2uQnpBHf73xrm4+eJJWpdDHqb7Dm6ltK6AiIiI/Nm27EokRIfjWwsnal0KeYHuwy8RERGRp3QaTNh5qhpfm52EYE5uCwi6D79Gs1nrEoiIiMhPpeXWor3bhGvmjNG6FPIS3YffF77Ic/qxnKRJREREQ9mWVYnYiBAsnhqvdSnkJboPv+WNHVqXQERERH7IYDLj85NVuHJWEkK5kUXA4E+aiIiIAtK+gjo0dRhwzZyxWpdCXsTwS0RERAFpW1YlRoQFY/mMBK1LIS9i+CUiIqKAYzIrbM+uwlfPPw8RocFal0NexPBLREREAedwcQNqW7twNVd5CDh+HX652gMRERENZltWJcKCg3D5BedpXQp5mV+HXyIiIqL+lFLYllWJ5TMSEB0eonU55GW6D7/c3piIiIjcKausGWWNHWx5CFC6D79ERERE7rQtuwLBQYKrLkzSuhTSgO7Dr2t9u2z6JSIionOUUvgkqxKLp47GqKgwrcshDeg+/LLtgYhoeCKySkRyRCRPRB4c5P5JIrJTRI6IyDERuVaLOom0llfdioKaNqyazZaHQKX/8AumXyKioYhIMICXAFwDYBaAW0VkVr/DHgKwSSn1FQC3APird6sk0odtWZUQAa5m+A1Yug+/REQ0rEUA8pRSBUqpbgAbAdzQ7xgFINb6cRyAci/WR6Qbn2RV4qJJo3BebITWpZBGGH6JiHzfeAAlvT4vtd7W2yMAbheRUgBbAdzvndKI9KO4rh0nKprZ8hDgdL+4HXt+iYjc4lYAbyilnhWRJQDeFpE5Silz74NEZC2AtQCQlJSE1NRU71c6iNbWVtZig57q0VMtwMB6PjljAACMbC1EamqxprVoLZDr0X34JSKiYZUBmNjr8wnW23q7B8AqAFBKpYtIBIAEANW9D1JKrQOwDgBSUlLUypUrPVSyY1JTU8FaBqenevRUCzCwnudP7MHscRH49rXLNa9Fa4Fcj0ttDyLyUxHJFpEsEdlgfTIlIiLvygAwQ0SmiEgYLBPaNvc7phjAFQAgIhcCiABQ49UqiTRU1dyJw8WNbHkg58OviIwH8GMAKUqpOQCCYXnCJSIiL1JKGQH8CMB2ACdhWdUhW0QeFZHrrYf9HMAaEckEsAHAnUqxsYwCx6fZlQCAVdzVLeC52vYQAiBSRAwARoCzh4mINKGU2grLRLbet/2+18cnACzzdl1EerEtuxLTEqMwIylG61JIY06P/CqlygD8CZa30ioANCmlPnVXYe4gAuw8VY3UnOrhDyYiIiK/1NDWjX0F9Rz1JQCutT2MgmUdySkAxgGIEpHbBzlurYgcFJGDNTWOt5e5tr0xcNcbGbhzfYZrJyEiIiKf9dnJKpjMCqtmj9W6FNIBVya8XQngjFKqRillAPAPAEv7H6SUWqeUSlFKpSQmJrpwOSIiIiLHbc+qxPiRkZgzPnb4g8nvuRJ+iwEsFpERIiKwzCI+6Z6yzuF0DCIiInJWS6cBabm1WDVnDMTVt5PJL7jS87sfwIcADgM4bj3XOjfVRUREROSynTk16DaZ2e9LZ7m02oNS6mEAD7uplsGv4cmTExERkV/bnlWJxJhwLJw0SutSSCdc2uTCG9j2QEREFDi6jWZ8dqIKnQaT6+cyKezMqcbXZiUhKIgtD2TB7Y2JiIhIFwpqWvHAxqM4XtaEZdPj8fc7UjAizPmoklVrQnu3iS0P1IfuR36JiIjIvymlsCmjBKuf342ShnasWT4F6fl1uOO1A2juNDh93kNVJsRFhmLx1Hg3Vku+zq9HfvkGBxERkb41tRvw638ew9bjlVgyNR5/vnkBxsRFYMHEUXhg4xHc/up+vHX3IowcEebQebuNZhypNuKaeRMQGsyxPjqHvw1ERESkiX0FdbjmuS/xaXYVfrXqArxz7yUYExcBAFg9byxevn0hTlW04JZ1+1Db2uXwuduNYMsDDaD78OvoLzsRERHpm8FkxjPbT+HWv+9DeGgwPvr+Unx/5TQE95uUduWsJLx2ZwoK69pw0yvpqGzqtPsa27IrER4MLJ+R4O7yycfpPvwSERGR/yiqa8O3Xk7HSzvz8e2FE/Cf+y/F/IkjbR6/fEYi3rr7ElQ3d+GmV9JRUt8+7DVMZoVPsysxPzEYEaHB7iyf/IBfh99TlS1al0BERESwTGr76FAprn0uDWdqWvHSbRfh6W/NR1T48NOPFk0ZjXfuvQSN7d24+ZV0nKltG/L4Q0UNqG3txsIkv57aRE7y6/Bb1tChdQlEREQBr7nTgAc2HsXPP8jE7PFx+OQnK7B63liHzrFg4khsWLsYnUYzbnolHaerbA9wbcuqRFhIEOYlctSXBvLr8EtERETaOlhYj2v+koYtxyvwi6/NxIY1izF+ZKRT55o9Lg6bvrcYAuDmV9KRVdY04BilFLZnV2LFjAREhnDdJxqI4ZeIiIjczmgy48+fncZNr6QjOEjwwX1L8KPLZwyY1Oao6efFYNP3lmBEWAhu/fs+HC5u6HP/8bImlDV24OrZXOWBBsfwS0RERG6llMIP3zuM53bk4usLxmPLjy/FRZNGue38yQlR2HTfEoyOCsN3Xt2PfQV1Z+/bllWJ4CDBlRcmue165F8YfomIiMitPjxUiu3ZVfjlqvPxfzcvQExEqNuvMX5kJDZ9bwnGjozEnesPYNfpGiilsC2rEounjsaoKMc2xaDA4dfhV9jqQ0RE5FXljR149OMTWJQ8GvetmObRayXFRuD9tYsxNSEaa948iL/tykdBbRtWzXFsMh0FFr8Ov0REROQ9Sin86qNjMJoVnvn2PAS52N9rj/jocGxYsxizxsXi6W05EAGunsWWB7LNr8MvB36JiIi8Z1epEWm5tfjNtRdgcnyU164bNyIU79x7CS6bmYirZ43BebERXrs2+R6u/kxEREQuK6lvx8ZT3Vg6LR7/fclkr18/OjwEb969CEopr1+bfItfj/zy15+IiMjzzGZLuwMA/O+N3ml3sEU44YeG4dfhl4iIiIDG9m4YTWaPnf+d/UXYm1+HWy4Iw8TRIzx2HSJ3YPglIiLyY0aTGVf/5Uvc+Le9aGzvdvv5i+ra8OTWU1gxMxGXTWA3Jekfwy8REZEfyyxtQlVzFzJLm3DzK/tQ3dLptnObzQr/88ExhAQL/vfGuWw5IJ/A8EtEROTH0nJrIAK8dNtFKK5vx82v7ENZY4dbzr1+byEOFNbj99fNwti4SLeck8jTGH6JiIj8WFpuLeaNj8PqeWPxzr2LUNvahZteTkdhbZtL582vacXT207higvOw7cWTnBTtUSe59fhl2++EBFRIGvuNOBoSSOWz0gEACycPBob1ixGh8GEb7+SjpzKFqfOazIr/OKDTESEBuOJb7LdgXyLX4dfIiKiQJaeXweTWWH5jISzt80ZH4f31y6GALh5XTqOlTY6fN5X0wpwpLgRf7h+NpK4oQT5GIZfIiIiP5WWW4MRYcH4yqRRfW6fkRSDD+9biujwENz29/3IKKy3+5y5VS149rPTuHp2Em5YMM7dJRN5HMMvERGRn0rLrcWSqfEICxn4cj8pfgQ+uG8JzosNx3de24+03Jphz2c0mfHzDzIRFRaMx77OdgfyTX4dfrnDGxERBariunYU1bX3aXnob2xcJDZ9bwmmJETjnjcOYnt25ZDnfHlXPo6VNuGxr89FYky4u0sm8gq/Dr9ERESBKi3PMpJ7qXWymy0J0eHYuGYxZo2LxQ/ePYx/HSkb9LiTFc14bkcuVs8bi9Xzxrq9XiJvYfglIiLyQ2mnazEuLgLTEqOGPTZuRCjeufcSXJw8Cj/ddBTv7S/uc3+30Yyfb8pEXGQo/njDHE+VTOQVDL9ERER+xmgyY29+LZbPSLS7Lzc6PARv3LUIK2cm4jf/PI5X0wrO3vfSzjycqGjGY1+fi9FRYZ4qm8gruAk3ERGRnzlW1oTmTiOWz7Td7zuYiNBgvPKdFPzk/SN4bMtJtHYZceWFSXhpZx6+vmAcVs0Z46GKibzHr8NvRZP79i8nIiLyFWmnayECLJvmWPgFgLCQIDx/y1cQGXocf/k8F6/tPoPRUWF45PrZHqiUyPv8OvwSEREForTcGswdH4dRTrYohAQH4ZlvzUN0eDDe2leE525ZgJEj2O5A/oHhl4iIyI+0dBpwpKQR91021aXzBAUJ/nDDHPzkyplOh2giPfK5CW/2zFodTFO7wc2VEBER6U/PlsaXTh96iTN7MfiSv/G58Ds53rnwe+9bGW6uhIiISH/ScmsxIiwYF00eqXUpRLrkc+HXWacqW7QugYiIyON259Vi8dR4hIcEa10KkS4FTPglIiLydyX17ThT2zbklsZEgY7hl4iIyE+k5dYCAMMv0RAYfomIiPxEWm4NxsZFYFpitNalEOkWwy8REZEfMJkV9uTVYvmMBLu3NCYKRAy/REREfuBYaSOaO424dIZ7ljgj8lcMv0RERH4gLdeypfGl09nvSzSUgAm/fAOIiIj82e7cWswZF4fR3JSCaEgBE36JiIj8VUunAYeLG7jKA5EdGH6JiIh83L6CehjNCpcy/BINi+GXiIjIx6Xl1iAyNBgLJ4/SuhQi3WP4JSIi8nG7c2uxeOpobmlMZAeGXyIiIh9W025GQW0blzgjshPDLxERkQ/LrjMBAFaw35fILgETfrnbDRER+aPsOhPGxEZg+nnc0pjIHgEUfrWugIiIyL1MZoUTdSZcyi2NiewWMOGXiIjI3xwva0KbAVzfl8gBDL9EREQ+Ku10DQBuaUzkCIZfIiIiH5WWV4vJsUGIjw7XuhQin8HwS0TkB0RklYjkiEieiDxo45ibROSEiGSLyHverpHcq7XLiMNFDZgdz7V9iRwR4sqDRWQkgFcBzAGgANytlEp3R2FERGQfEQkG8BKAqwCUAsgQkc1KqRO9jpkB4NcAlimlGkTkPG2qJXfZl18Ho1lhTgLDL5EjXB35fQ7ANqXUBQDmAzjpeklEROSgRQDylFIFSqluABsB3NDvmDUAXlJKNQCAUqrayzWSm6Xl1iAiNAgzRvFNXCJHOP0vRkTiAKwA8BoAKKW6lVKN7irM3bgADBH5sfEASnp9Xmq9rbeZAGaKyB4R2Sciq7xWHXlEWl4tLpkSj9AgvsIROcKVtocpAGoArBeR+QAOAXhAKdXW+yARWQtgLQBMmjTJhcsREZELQgDMALASwAQAX4rI3P6DFr2fs5OSkpCamurlMgfX2trKWnqp7TCjoKYDl8Qb0NrapXk9PfTwvelNT/XoqRYgsOtxJfyGALgIwP1Kqf0i8hyABwH8rvdBSql1ANYBQEpKinLhekRENLgyABN7fT7BeltvpQD2K6UMAM6IyGlYwnBG74P6P2evXLnSUzU7JDU1FazlnI0HigEcx13XLEb5yUOa19NDD9+b3vRUj55qAQK7HlcahUoBlCql9ls//xCWMExERN6VAWCGiEwRkTAAtwDY3O+Yf8Ey6gsRSYClDaLAm0WS+6Tl1SIpNhwzuKUxkcOcDr9KqUoAJSJyvvWmKwCcGOIhRETkAUopI4AfAdgOy8TjTUqpbBF5VESutx62HUCdiJwAsBPA/yil6rSpmFxhMivsyavFpdMTuaUxkRNcWuoMwP0A3rWONBQAuMv1koiIyFFKqa0Atva77fe9PlYAfmb9j3xYVlkTGtsNWDGTu7oROcOl8KuUOgogxU21EBER0TDSci1bGi/jlsZETgmYxQH51hAREfmDtNxazBobiwRuaUzklIAJv0RERL6utcuIw8UNWM6WByKnBUz45bgvERH5uv0FdTCYFFbMSNS6FCKfFTDhl4iIyNel5dYiIjQICyeP0roUIp/F8EtEROQj0nJrsGhKPCJCg7UuhchnMfwSERH5gLLGDuTXtGHFDPb7ErmC4ZeIiMgHpOZUAwCWs9+XyCUMv0RERDqnlMKbewtxflIMZiZxS2MiVwRM+OUyv0RE5Kt2na7B6apWrFkxlevWE7koYMIvERGRr/p7WgGSYsNx/fxxWpdC5PMYfomIiHQsq6wJe/LqcOfSKQgL4cs2kasC5l+Ryay0LoGIiMhhr6YVICosGLddMknrUoj8QsCE34Z2g9YlEBEROaS8sQMfH6vALYsmIS4yVOtyiPxCwIRfIiIiX7N+zxkAwF3LkrUthMiPMPwSERHpUHOnARsOlGB7oFIAACAASURBVGD13LGYMGqE1uUQ+Q2GXyIiIh3asL8YrV1GrFk+VetSiPwKwy8REZHOdBvNWL+nEEumxmPuhDityyHyKwy/REREOvOfY+WobO7E2hUc9SVyN4ZfIiIiHVFKYd2XBZhxXjQum5modTlEfofhl4iISEd259XiVGUL1qyYiqAgbmVM5G4Mv0RERDqy7ssCJMaE44YF3MqYyBMYfomIiHTiZEUz0nJrcefSZISHBGtdDpFfYvglIiLSib+nFWBEWDD+m1sZE3kMwy8REZEOVDR1YPPRctyUMhEjR4RpXQ6R32L4JSIi0oE39hTCrBTuuXSK1qUQ+TWGXyIiIo21dBrw3v5iXDN3LCaO5lbGRJ7E8EtERKSx9zNK0NJlxPe4qQWRxzH8EhERachgMuP13WdwyZTRmDdhpNblEPk9hl8iIiINbT1egfImbmVM5C0Mv0RERBrp2cp4WmIUvnr+eVqXQxQQGH7JpuZOA1o6DVqXQUTkt9Lz65Bd3ow1y7mVMZG3hGhdAOnXvEc+BQAUPrVa40qIiPzTurQCJESH4etfGa91KUQBgyO/REREGsipbEFqTg2+uyQZEaHcypjIWxh+iYiINPBqWgEiQ4Nx++LJWpdCFFAYfomIiLysurkT/zpahptSJmBUFLcyJvImhl8iIiIvW7+3ECazwt3cypjI6xh+iYiIvKi1y4h39xVh1ZwxmBwfpXU5RAGH4deNalq68MjmbBhMZq1LISIindqUUYLmTiPWLOemFkRaYPh1o4c3Z+GNvYX44lS11qUQEZFOvbOvCCmTR+Erk0ZpXQpRQAqo8NtlNHn0/EaTAgAo5dHLEBGRj6pt7UJBbRuunj1G61KIAlZAhd9ntuVoXYJuldS343f/yoLJzOTu7zoNJlz3QhoOFTVoXQpRwDlW2ggAmD9xpMaVEAWugAq/5U0dHj2/L8fG+zccwdv7ipBpfWIm/5VT2YKssmb84eNsrUshCjhHS5oQJMCc8bFal0IUsAIq/HqL+OD27D3B3QdLJyLyGZkljZiZFIMRYSFal0IUsBh+iYiIvEAphczSRixgywORphh+iYiIvKC4vh2N7Qb2+xJpLKDCr3j4TX2u8qC9jm4THv53Ftq6jFqXQkTUx9ES62S3CQy/RFoKqPDrLeyb1c76vWfwZnoRXtmVr3UpRER9ZJY0ISI0CDOTorUuhSigBVT4VT69HgPZw2xdqs3EYXgi0pnM0kbMHR+HkOCAeukl0h2f+xf4zYvGa12CTT3bGhfVtWtcCRER6YnBZEZWWRNbHoh0wOfCb1xkqNOP9XTP78HCegDA41tPevQ6RETkW3IqW9BlNHOyG5EO+Fz4JdKDoro23PH6AbR3c2IdEQ2vZ7Iblzkj0h7DL5ETntx6Cl+ersGunBqtSyEiH5BZ0ojRUWGYMCpS61KIAl5ghV8uwxAwON+NiPQks7QR8yfEQXxxC1AiPxNY4dfDmLe0xxcWItKb1i4jcqtbsWDiKK1LISIw/HpMz8oPREQU2I6XNkEpYP7EOK1LISIw/HpMcT2XOyMiIkvLA8Cd3Yj0wuXwKyLBInJERP7jjoI8qWcpMhoo0zoT2dcpNvsSkc5kljRicvwIjIoK07oUIoJ7Rn4fAOATC9tWNXd59PzMXfrB1l8i0ovMkkaO+hLpiEvhV0QmAFgN4FX3lOPbuH2y7ypv7EBrF9fs7e+dfUU4VNSgdRlEPqu6uRPlTZ3c3IJIR1wd+f0LgF8CsDm7S0TWishBETlYU+P6mqiXTIl3+Rye0mnw/UlutuL7wcJ6mMy+E+4dHYVf+tQXuOHF3Z4pxoc99K8s3Pi3vVqXQeSzMkubAAALONmNSDecDr8ich2AaqXUoaGOU0qtU0qlKKVSEhMTnb3cWWEhnKPnjMLaNhwpdm4ELz2/Dt96OR0v78p3c1Xu58pSZ/k1bW6sJLD8+h/HkfzgFq3LINKdzJJGBAcJZo9j+CXSC1eS5DIA14tIIYCNAC4XkXfcUhW53co/peIbfx1+BK8nOuZVt569raKpY8BtRL1tOFCsdQkBT0RWiUiOiOSJyINDHHejiCgRSfFmfYEqs7QRF4yJQURosNalEJGV0+FXKfVrpdQEpVQygFsAfKGUut1tlfk4d8y3+tWHx3D5n1LdcCbH3fxK+rDHvJVeiB9vOOL5YsgjOEHTf4hIMICXAFwDYBaAW0Vk1iDHxcAySXm/dysMTGazskx2Y78vka6wh0DH3j9YgoJabd6K7zSYhj3m9//OxubMci9Uo1++mB+5EoZfWgQgTylVoJTqhuXduBsGOe6PAP4XQKc3iwtUhXVtaO40YgFXeiDSFbeEX6VUqlLqOneci/RHixFCk1nhi1NVul23lwGSdGY8gJJen5dabztLRC4CMFEpxeZsLzlqXT+dI79E+hKidQHkGdUtnVj0+A5sWLMYS6bpd4UMW/6eVoCnPjmFl29fiFVzxjj8eH1GZiJtiEgQgP8DcKcdx64FsBYAkpKSkJqa6tHa7NXa2upztWw50YXwYKDs5EFUnPLcX8y++L3xFj3Vo6dagMCuh+HXTx0qtKzs8ObeQpfDrxajnCXW7aFrWj27MQmRnygDMLHX5xOst/WIATAHQKp1RZQxADaLyPVKqYO9T6SUWgdgHQCkpKSolStXerBs+6WmpsLXavlz9h4smBSEy7+6RBf1eIOeagH0VY+eagECux72/NKwnv30tNYlOIxdCRRgMgDMEJEpIhIGyyTkzT13KqWalFIJSqlk6yTlfQAGBF9yny6jCSfLm7FgElseiPSG4ddDXFlvVm/KGju8fk22LRDZTyllBPAjANth2W5+k1IqW0QeFZHrta0uMJ2qaEG3yczJbkQ6xLYHspsWcd5//oTQRqfBhJAgQUgw/871d0qprQC29rvt9zaOXemNmgJZZiknuxHpFV8R7XDX+gNY+MfPtC7DbsV17ThU5NxublppbO/Giqd34mRFs9al+JULfrcNt/2dS7oSedvRkkYkxoRjbFyE1qUQUT8Mv3bYmVODurZuhx6j5RJdK57ZiVd3n3Hqsd/4615UNg2+BKgnv6K03FoU17fjxZ15lmsNcbG03Bq71iEmiwOF9VqXQBRwMksaMX/CSL9qgSPyFwy/Q7hlXTpusmOns8FsOljq5moAg8mMfxwu9Xiw3nGqyqPnH1K/L63/60ZedQu+89oB/PafWY6chojIa5o7DcivacOCiXFal0JEg2DP7xD2FTg/YlbT4v4lul78Ig/P7chFeEgwls9MQGxEqNuvMRRPjl/0hFw1TGxt7jQCAPJrWj1YDRGR846XNgFgvy+RXnHk10M88U5XtTVQ//C9w5j3yKfYlFEyzCN8h3BqGxH5iZ6d3eaNZ/gl0iOGXw/xTJTrOyqaerraI1fR0rmODs81LnycWY7LntkJs9n1a+h092Ui0lBmSSOmJkQhboR3350jIvuw7cFD3D3y29RhwIYD3hnptRXoPJnzzrY99O/59cCfEb/88Bg6DCZ0Gk0YEebcPwHOYSEiWzJLG7F0WoLWZRCRDRz59RFX//nLAbc5O+pYWNuG5Ae3IDVHPyPHtrJkXvXgvb3Dfel6z6bvZxRjb36t1mUQkZtVNnWiqrkL8ydwshuRXjH8eoi73w6vbB58+TFnHC62rAH876PlNo/pNpkH3OaNQNkz4a21y7KU2et7+i7ZZm8Neu9G+NVHx7n+LpEfOlpieX7lZDci/WL49WGe7Dc1mLwbH/u3Pbi6nNv+gjoU1bW5WBURkWOOljQhNFhw4dhYrUshIhsYfj1ELz2hgy0dps9JWra/YVVOjHofLm7EZc+kDnmMs9+H01UtaGgzAAC6TdxsQ2/WfZmPX314TOsyKEBlljTiwrGxiAgN1roUIrIh4MJvt9GMLqPnA4svLN1lq0JHM2FlUyeq3dSWofr9PwBc8sQOt5zbXb725y+RXlAHAPjp+5lo7TJqXBH19sTWU3j/oP8sA0i+w2RWOF7WhAVseSDStYBb7WHhY5+ho9uEvCeu9eh19DLy298drx/AxFGRbj3n4ict4bTwqdVOn8Ph75cbhq/d9TNq6jAgOjzg/ikRUT8FNa1o7TJi/gSGXyI9C7iR35ZOI4xuWN/VV315ugbv7i8e8hg95HZbNYg1sXrzJ2g2K3x2osrj20q7wzv7inCoyL6dCYfbTY+IHNOzuQUnuxHpW8CFX09pbO/u87k3Rn7tCS/bs6vQafCdvtTh8qWtb+tP3z+Kfx8tc3s9APDu/iKseesgPjrsmfO700P/ysKNf0sf8hhfaMkh8kWZpY2ICQ/B1IQorUshoiEw/LrJx8cq+nze1uV44Kxv68YXp6rQ1GFA8oNb3FUaPjpcOvgdTmSg6uZOFNQMvvauK86Vonr9r/3+eaQMD2w86tBj7B3ILW+y9DPbmninlGVk+ER5s0PXJyL/klnShHkT4xAUxD8wifSM4ddDNmfaXkPXlrvWH8Ddbxy0e/OJ7dlVqG/rHv5AB9nMhAIsemIHLn92l9uvKXptkrbTmrcO4trn0wbcXtnU6ZZtlIlI3zoNJpysaGa/L5EPYPi1QYv+zoJay7q0JgfC0s82OTba6RIPfkuOlzVZLmG9xnBR2B0/HnvztrPXKqlvx+Ind+D5L3KdO4GD8qpbvHIdIhroREUzjGbFfl8iH8Dwqyc9wc+BQdDmDoPTlztU1ICO7oHtGZvd2Du763QNyho7hj3m+R2WgDjYUme9OTpA3G0cuFOdsxy9ds+ufLtzvbON8c5TNXYdx4luRO6XaZ3sxmXOiPSP4deGb/x1r0PH2wpZNS1dDl/bkQlJh4sbHT5/j6K6djz4j4GbAWQUNgx6fIcTE+e++/oBrPrLl8PU0XcnNqXUgAmEzvpbar7N+xwd0bV1vDfeJGjtMuLL0/aFW3t5e+JbZVMncio5Ok3+KbOkEWNiI5AUG6F1KUQ0DIZfG3qWrLHXi4O8tf3Szjxc/Pjn2GlnD29Phuo/wni6yrXA8Nt/Ztm8z5Ew8klWpd3HKqVwssIyAayls+8mEP/1wm4seXLwjSuUUli/pxB78ursvtZQGgYJ0T3fX6NJobLJPZtzeNovNmXijtcPoLShXetSnLb4yR24epg/hOx1uLjBIxMviZyVWdqE+RPjtC6DiOzg8+E3NFgfE6Ua2ge2HzyzPQfAubfDnJVRaN+6rXry2u4zuOa5vhPAenbWO17WhAoboVMBdv+x4KyekdpHPs7G4id3oMnO1hFH2x7cOSKcZw16g7WpBKJv/nWvRyZeEjmjsb0bZ2rb2O9L5CN8Mvw++c25WpfgVlllTWjvNp6dZOfrKx8Alq+pt7TcGpz/0DYcdFOQd0ff6henLCG7zcPbE3vrx+kHvzZEPimz1PJ8t4ArPRD5BJ8Mv7cumqR1CW7T3GnAdS/sxo83nFu1wdtLRLZ3ezb8AcDuPMukr+HW4s2pbBkyjA7Xp+qJAOgPE8SWPfUFvvPafq3LIPJLmSWNEAHmTGDbA5Ev8Mnw62uGCmw9u68dLWk8G7GCHExwrk4Ou3N9hkuPH4ytuDjcyg8VTZ12TeKzt6WgpqULyQ9uQXr+wB5iX9iu2F3KGjuQ5oZVJ7LKmjy6mYdSCnMe3o539xd57BpE7pZZ0ohpidGIjQjVuhQisgPDr9YGyV+ODl4uePQzpy8vIjhwxnd6ih0d2e3pl357X6HNcwwWgfNrWpHy2Oc2d3Wzhz+G6+te2D3oZh6uMprM2JtXC6NZobXLiIf/ne3Q4z87UYU39xa6vS6i4SilkFnayCXOiHwIw69OiHhnySx3sXeSmLf1/x4O9i09u5HGEEn6rb2FqG3twifHz21b7ezSYN5eUswXPf9FHm57dT/2Fzj3h9iatw7i4c2OBWYidyhr7EBtazcnuxH5EIZfL2g32O5h7Qlnda3n1gPun8kMbtyoob8OJ/t9f/Te4SHvtyfI7zhZBaPJua/NHWv0uqs/2Jt/tHjyUn/anoMXdnhnN7r+8q2rWdS0+sbSc0Q9Mks42Y3I1zD8esEruwqG3aDArM5NrNqYUdLnvkc+PuGx2grrnFs3tmiYx9kT0u558yBe3Jnn1PXtNVS+HSy0Jj+4BY/0H0HUwYi8N8aOX9yZh2c/O+2FKxH5j8zSRoSFBOH8MTFal0JEdgrY8Lvuy/yzk80ctTmzHN95bT9++8/jdj/mjtcPIHeQzSoGC2CpOY7v5DXcRDK9Km1wb932/Ex7Rnx7Wjf69+a+0a931GQeOv3aWg3C25m5dxtHTUsXNvX7I8qX9P7e+ervNgWGoyWNmD0uFmEhAftySuRzAvZf6xNbT+E5J9/i/fGGI0jLrcW7+4sdelxz59AtBp0G59sbfr7JsoTY1l49qlrSarKX2cZ1XSnn1d1nAFiWpXOKG4Ztc6stbQHHSpuGOdLivncO4ZcfDdy6ujcFhX8fLTu7+Ygzyho7cNzOmpzxj8OlWPbUFz41KZMCh9FkxvHSJsxnywORTwnY8AvA4a1tzWbl1jVx69u6UdPSNfyBdjpT24YfvDt0L67e9N8Mw5Zuo9mp/mB3rvv70aFS/O5fA7eKthWsPZH/DxUNDIHVzZ2Y/putOF56bok4e36vssqa8cDGo3h6W47T9Sx76gv814u7nX58D1s/pkNFDQCAHCe2+H7431nYlqWPPwbJP+XVtKLDYOJKD0Q+JqDD7z+PlAEA1u85g2o7lrR6M70Qs36/3enr3fNm3/V0L/rjZ24JDj203vr2zb2FSH5wC1o6DXa/5X+q0r5QM/OhT7D6+XPfq57zHypqwL6CgWv4DsWeUDrYIdUtXXh7n+Prz3q6X3fX6RoYzQr/Olru1OO3ZVXaNVLv6B+LWnszvQj3veNbfwySb+nZup4rPRD5loAOvwBQVNeGP3x8Aoue2AGlFH6+KRM7TlYNeuxg/amOjEY2tntuebAOgxmPb/XcxLj+iusHTnjrWWe12o2j2b3lVLUMGMm98W97ccu6fcM+dqhd2gbLfT0jp65sNe0rO8OVNXZg08Hh+4O/+dc9XqjGwh/XSCb9Kaprw5t7C5GeXweDE+8sHS1pQmxECJLjR3igOiLylBCtCxiKN14A3ztwrm+322TGR4dL8dHh0kGP3X9m4Ajj41tPYvJo+5/4zGaFBY9+ijuXJjtc61B6RiC0VFDbdu6T/j86N/0oX9lVMOT9/X9letbY3Z5dhc2Z5bh+/ji7WiE+yap0tsQBXNkooz97/0kM9sfJUPJrzv3sDhbWIyV59IBjyh0Y+X1pZx6e2Z6Dk4+uQmRY8LDHu/JHBpEjOg0mbM+uxMYDJUjv9a5RdHgIlk2Px8rzz8PK8xMxNi5y2HNlljRi/sSR/P0l8jE6D7+ev8ZwYaq3rLKB27qu31Po0PVyqlrQ3GnE8194dokvLSnluVHPzZnl1msohydq/XjDEVw/f5zbaxruK3V2ObnBDDahz9Hv9HCbbnzr5XQc+O0VOC8mwsEzn9OzYkZLp8Gu8NufWTn2rgrRcE5WNOP9jBL880gZmjoMmDg6Ev9z9fm4Zs4Y5Fa3IjWnBrtyqrE92/LO3/lJMVh5fiIuOz8RKZNHD1jNoaPbhJyqFvzgwmlafDlE5AJdh19bM/c9Jc86o96THth4xOPXCBTve2Epr24PbjAynOqWTix6fAfeW3OJx6/V/10WrfvHAeCWdfvOrZ3KNghyQmuXER9nlmNjRgkySxoRFhyEVXPG4JaLJ2Lx1HgEBVn+EJyaGI2rZ4+BUsoahKuRmlOD1/ecwStfFgwYFQaArPImmMyKKz0Q+SBdh19vv9z1nlDlKaerPB+wteedn9xgwfSvqfnDPs6RHPX41pOOlNTvQvYfWtvahZTHPscbd12MleefBwA4XGRpZXnDwXcX9GTREzvw0OoLce/yqXYd3/tnc7CoAReMPbdxwKfZlRgdFTZoSwZRD6UUDhc34v2MYvznWAXau004PykGD//XLHx9wXiMigqz+VgRwcykGMxMisHaFdPQ2mXE3rxapJ6uwa6cmrOjwhOiBeNyLKukzJsY55Wvi4jcR9/hl4M9PksvP7tKN/bb2iv5wS34+VUzsXDyKLsfc9y65Nv6PYVnw6/d3PC9drTDIPnBLXYv77R+T+GQ4Tc1p/rsuy62fm/2FdRji3UN68KnVjtWLAWE+rZu/ONwKd7PKEFudSuiwoJxw4JxuPniSZg/Ic6pvtzo8BB8bfYYfM06KpxnbY/45/4cHCluwPTzol1qDyIibeg7/PrIbHnyLe6emzLYxMxnPzuN9+51f7vCcP26w+k0mFDTOvAPgtf3nHH4XEfdNMnyzvUZwx6zRSebt5B+mM0KpypbsDe/Fnvz65CWWwODSeGiSSPx9I3zsHreWESFu+8lTkQwIynG8p+5GBcvuZSvUEQ+St/hl88sPkkp4LQDmxIE0jzp7dmVQ/bTDvcr7+ofhHetz+gzw90WV0O2N7lz4xnSL8vIawv25tchPb8O+wrq0GBdPnJKQhTuWJKMmy+eiJlJMcOcyT3cGayJyLt0/a+3f/gNCw74ZYl9Ru+ls4bzwaHBl5bzFk/9kTXYab/39qE+n7+wIxf3XzGjT9TMKKxHkI3haVdrtSf4asmZL+/+9ziJ1B8ppVBc3470/Drsza/DrlMdaNr+JQBg/MhIXHFhEpZOi8eSafF2LUtGRNRD3+HXjpfCGefF4ETFwCXISDtdDq6QcKzUvi2Oe3M2BD768Qm062Algx7r9xbi/itm9Lnt2y+nAwBevn2hXecI9PagjMKBWz6Tb6ps6jzbxpCeX4eyRsvGQokx4bhwdBC+vmQWlk5LwMTRkVxbl4icpu/wa8dr+vcum4oHNh71fDFkN28tUdfzwugIZ3pbh2PPV3uoqB4LJzu3SoEeoq0rG85wtzayx+HiBtz4t71QChg5IhRLpsbjvsumYsm0eExLjMauXbuwctEkrcskIj+g7/BrxzG23h4m7QyWdQwm9weg+rZut5zH1ZHT21/dP+wxN/4tfdBVCvoHw96f2/rVNpsValu7EB0RghFhnvkn3P/aRrN7f35KKbzyZQFu7Rdm+n8/7Ok9ZrT2D7tyagAAm3+0DHPGxZ1dg5eIyN30HX45YuSTtmUP3BrYEyOuelFhY9vf/r++yQ9usXkOe9/C/eBQ6dke6dnjYrHlx8vtK1Jn0nJr8dQnp3DSiy1LedUt2HCgBA+tvpBvmevQwaJ6XDgmFvO4aQQReZiuZ5A5M9j0zYvGu78Qckh1c5fHr+ELfa6u1viXz3OHvD+73BIcPfE3Yk/fcQ9XrjFY0OzZoKS103srNdz1RgZe230GJfWOt8uQZxlMZhwuasSiKdzAhIg8T9fh15nscM+lU9xfBznEG8FUKWBb1sARZl8z1HfKm6Oi/VU2d6Kozv4VO5zl7t+UXBtL7FW3dKLJuiwWB331J7u8GR0GEy7m7n1E5AX6bntw4qUxlMuhac5ba8Q6uqqELXrorhnqO6ZVVrvsmVSPndtWAHXqR9HrQVf9+ctBD1n0+A5nzkxeknHGsmLHxcn274pIROQsXSdFPYQSctxHh7Vdt1cvjE5M8iuqax9w2/AbX5Aj+LyiPwcK6zE5fgTOi+VWwUTkeboOv95aMosC28o/pXrkvI9tOTHsMUoBzZ0G3PH6AQBAcf3A8KsHrrSyDDVxdbhJrWxR8H9ms8LBwnq2PBCR1zgdfkVkoojsFJETIpItIg+4szCAI1rk2+zd5S7tdK2HK9Efm6G23z96e/7+5fOEb8uvaUVDuwGLGH6JyEtc6fk1Avi5UuqwiMQAOCQinymlhh/ushMHfskW/mr4h/4/R19YxYPc64B1h76LudIDEXmJ0yO/SqkKpdRh68ctAE4CcOs6Y3whJF9T0+LYMm9KKXx4qMSla3rjj0RHr9H7+MGWOuuZFHm6cvDVGRzhaGcEn1f05WBhAxKiw5EcP0LrUogoQLil51dEkgF8BcDwW105ID4q3J2nG9Rvr73Q49cg9zO5eccxd7n48c8dOr6t24Sd1p2t/EHPz6W502DX8eU2NggZytbjFX0+1+dvAtnrwJl6LJoyihuPEJHXuBx+RSQawEcAfqKUGrAwqYisFZGDInKwpsaxF/ngIEHWH67G5h8tc7VMm2Ijdb3aG9nQ7aZlzrRmz6ROX2r/efbTHMsHvWquaXXvpic/ePewW89H2ilv7EBZYwdSJrPlgYi8x6XwKyKhsATfd5VS/xjsGKXUOqVUilIqJTEx0eFrRIeH4PwxMa6UadMFY2IwaXSUR85NnlXW6B+7dLkj2Orpbfx9BXUA+tbUbTQPbAextc5vvy+lrWv4HeC4KozvyrD2+3JnNyLyJldWexAArwE4qZT6P/eV5D1KAUumxWtdBpFLOrpNHr/G4aIG+44rbhz09vq27j6fF9i5EsYZO3aZ02sLDA3vwJl6RIeH4MKxsVqXQkQBxJWR32UAvgPgchE5av3vWjfVRUR2+MfhUjy25aTHr3Pbq4618w81GFvd3Ik//mfwRWEcbfusa+1y205/5H0ZhfW4aPIoBAex35eIvMeV1R52K6VEKTVPKbXA+t9WdxZnL2dnCX93aTIA4LOfrsADV8xwY0VE7rM7z3av/I5T1V6sxH5DjcUuesL2VsP9Q/MRGyPJPe7fcMSBqkhPGtq6cbqqFYu4pTEReZmud3jzpMKnVuO2SyYBAGYkxeCnV83UuCKiwXUaOLJpS11r97DHHCnu27LBFmF9OGhtpeHObkTkbQEbfon8wZZjFcMfpIH+2xbb286w5bhjX489523r8nxPNDnuYGE9woKDMH/iSK1LIaIAE5Dh96PvL9G6BCK/tfTJgW0NP7RzebK0XPdv9dzWPfyKEf5ARFaJSI6I5InIg4Pc/zPrdvTHRGSHiEzWos4eBwrrMW9CHCJCg7Usg4gCkF+H37FxEdj64+UDbl/INSWJneLL1wAAFAlJREFUPKa8qXNAz29udatHrmXPxgi/+uhYn89b7Vg+zdeISDCAlwBcA2AWgFtFZFa/w44ASFFKzQPwIYCnvVvlOR3dJhwvbUIKWx6ISAM+G37Hj4wc9piQYMGscc4toXProolOPY6I9NVX29jed7e5617YrVElHrUIQJ5SqkAp1Q1gI4Abeh+glNqplGq3froPwAQv13jWkZIGGM0Ki6ZwshsReZ/Pht8FkzzbJzZhFPeZJ9I7LpB11ngAJb0+L7XeZss9AD7xaEVDyDjTABG+C0dE2vDrvX2nJ0YPuO26eWM1qISIPMHRdYEJEJHbAaQAuMzG/WsBrAWApKQkpKamur2GT490YEJ0EI7s32P3Y1pbWz1SizP0VAugr3r0VAugr3r0VAsQ2PX4dfh9/tavDLiN4ZfIfzD8nlUGoHev1gTrbX2IyJUAfgvgMqVUV//7AcuW9ADWAUBKSopauXKlWws1msz4wRef4lsLJ2Dlyjl2Py41NRXursVZeqoF0Fc9eqoF0Fc9eqoFCOx6fKLtQaxvbsZEOJbVYyJCLf8f7njG779UExGRjmUAmCEiU0QkDMAtADb3PkBEvgLgFQDXK6U02x0lu7wZ7d0mru9LRJrxifAbFhKEP1w/Gx9+f6lTj9/+0xVOBWAAmDOee84T6ZWw6xcAoJQyAvgRgO0ATgLYpJTKFpFHReR662HPAIgG8IF1O/rNNk7nURmF9QCARVMYfolIGz7T9tCzFfFw4qPCBtw2bmQklk6Px/bsqiEfGxYchG6TZTetnoHfy2YmIqus2aFaicg72PZwjnV7+a39bvt9r4+v9HpRg8gorMek0SOQFBuhdSlEFKB8YuTXXv/64TLER4cPep89XQz3Lp8y4DaOLBHpF/91+halFA4WNiAlmUucEZF2/Cr8xkWG2nHU8C+Xv/jazAGL9BORDnHo16fk17Shrq0bi9jvS0Qa8qvw29vaFVMdfkzP62jvUWK+thIRuUdPv+/F7PclIg35ZfgtfGo1fnPthQ4/rneLgyOLPfzvjXP7fM7d4Yi8g3+b+paMM/VIiA7D1IQorUshogDml+HXVb1zb+8X1/kT4gY9/uaLJ539ePmMBIyNG37rZSKiQHOgsB4pk0dD+JYaEWmI4beXoF7Px2qQrt9Lpsaf/fjYI1+zeR4uEUzkHcxQvqOiqQOlDR1seSAizfnMUmeu+smVM5Fb3Yol0+JtHrNmxVTUtHbh7kun4NW0AsuNNl5dYyNCcfChK9HSaexze0xEyKDBmYjcj9nXd2QUNgAAJ7sRkeYCZuR31rhY7PzFyiFXhIiJCMWT35yH6PAQLJxsWYqn5/8BYOLoEX2OT4gOx5R+vWtPfKNv/y8REVn6faPCgnHh2BitSyGiABcw4ddRy2ck4ujvr8JlMxPP3nbd3LEAgNghtlkeOSKMbQ9EXnK4uFHrEshOGYX1uGjyKIQE82WHiLTls89C31sxFTHhIVg6RBuDq0aO6LtbXE8HxHCTNZh9iYjOaWo3IKeqBRez5YGIdMBne37nTRiJ43+42ivXynv8GogIWjoN9j2AQ79ERGcdLKqHUmD4JSJd8Nnw602Ovk3H6EtEdM6BwnqEBgu+Mmmk1qUQEflu24MW7B3Q5cAvEdE5GWfqMXd8HCJCg7UuhYiI4dcTuNQZEZFFp8GE42VNXN+XiHSD4dcB9i6oz5FfIiKLI8WNMJgU1/clIt1g+HWA3W0Pni2DiMhnHCysB9B3zXQiIi0x/BIRkcccKKzH+UkxA5aOJCLSCsOvB6xZPhVXzUrC8hkJACxbHhMRBRqjyYzDRQ24eApHfYlIP5jKnGCr9/frC8YBAEZHheHvd6SgvduIw0WNeGLrSZyoaPZihURE2jtZ0YK2bhPX9yUiXWH4dZOTj65CWEjfgfQRYSG4dEaC3RPlBnPFBedhx6lqF6sjIvK+A9Z+30Vc6YGIdITh1wEhwZYUOz0xesB9kWFcv5KIqLeMM/WYMCoSY+MitS6FiOgshl8HxESE4q27F2HehDitSyEi0jWlFDIK63HZzEStSyEi6oPh10ErnHgid6XtgYjIFxXUtqGurZubWxCR7nC1By964IoZ+Oj7S9j/RkR+r2d934uTudIDEemLX4TfZ2+aj2XT4zFhlD77ylbPtawC8f/t3Xt4VPWdx/H3NwkhF3IhBAKEEELkUspFQuQiglC528rW1RaXFrTVri12l/VRV+s+PtR/quvaP/pUy9Ytbe2j1Npuq9vWotultf9IUIoICAKSWlgEaSzIigLy2z/mlzDJXMgkk5kzM5/X88yTM785c8739z3n/PKbc105cyTT6quoryrp9nf1wAyR5Dt+6sN0h5D1Wg6+S1VpIY1RrpEQEUmnrOj8Tquv4ombZ9IvP5jVufXK0ez6+mKGlBUBcP/yiTy2qpmvzGuMGPfaqbWpDk8k55w4fTbdIWS9ra1tNNcPxHTel4gEjM75TQEzo7T/hVQXF+azcEINCyfU8OjvDgDw/ZsuY9boQeSZMXJQCT995RCH3j2drpBFRHrs6MkPeKvtfVbNqk93KCIiEYK5qzQHzR83hKJ++RQW5LF2wVg+7fcADyrVI0FFks3pfKI+1XKw/XxfXd8gIsGjzm9A/dOCsWy/byFVA9T5FZHMsrW1jZLCfD4+vDzdoYiIRFDnN80eum4yV08eFlGel2dUliS34/voyqakTk9EJJqWg200jRxIQUCvwxCR3KZzftPs+uY6rm+uS8m8ZjdWp2Q+IpK7Tpw+y96j77F0YuSPehGRINDP8oBrrI5+m6A/3DW/Y1gXlYhIUGz707s4p/v7ikhwqfMbcNc3j4goKysqoC7sXsHLJkXuYZl9yaCIsgFF2tEvIn2rpbWNgjxj6kh1fkUkmNT5Dbjwe2TeuXhcaMBfqf7qfYvYsW5RxJXrW+9dwPdWX8aCj9V0Ks/PuzAtPWVORPrC1oNtTKytoLgwP92hiIhEpV2BGeBnX57FB2fPM2lEBQ9t2ttRXlHSL+r4g8v6A/Dtv5tK2/+d4fIH/idinGumDO+4HZFI7tG9zvqCc476QaVcMkRPdROR4FLnNwNMqw/tpT314TkABna596+L8Y+8qF8+wyuLeemeqzjvdw9/trmOypJ+rJwxkn/5xc6OcSePqGDHoRN9Eb6I5Agz4+HPTEl3GCIicanzm0EG9C/ggWsnMWfs4IS+N7SiqGP4wesmdww/urIJ5+C1wydYM7+RSeue7/hs/NAy9rz9HgATa8vZefhkL6MXERERST91fjPMiukjIwt7eAS3/UK5aPcZHlJe1NH5/eVX5zDq7l/1bCYigWQXH0VERLKSLniTqFyXq+ieWTM7TZGI9AWd8ysikqvU+c0C7f/GJ9VW8OKd8+OO21NT6ir7ZLoiIiIiqaTObxbI87dDqyotZOSgkouM3XOvrVvUZ9MWERERSQWd85sFZjRU8fdXjuaLsxt6NZ3yogJOfnAu5udlRdFvrSYiIiKSKbTnNwvk5Rn3LP0YQ8qLLj5yHM+tndsx3PXBGfFMGFbeq/mKiIiIpIo6v9KhtrKYG6bXJfSdW69s5KPzunhIMotWWRGR3KXTHqRHWh+4umN4eGUR9z2zK2KczXfMY/6//S6FUYl0j250JiKSu3q159fMlpjZXjPbb2Z3JysoSZ+lE0P3/L15Tuj84Yevj3xa06guF9WtmjUqYpwnb5lBQ3Vp0uIao8elioiISBL0eM+vmeUDjwALgUPAVjN71jm3O1nBSerNHTu4Y69u+N7ddpvvmEdVl8crA2z52lUYsOxbf+D4qTOMH3rhPODwp8W1u2H6SDa2vBUxnTXzG3lk8wF+sWY2k2srOHXmHCfeP8s3nnudfcdO9ahOty8cyzdfeKNH3xUREZHs0ps9v9OB/c65N51zZ4AfA8uTE5YEVUN1KRXFkXd9qCkvYkh5ERtvmclX5jUysCQ0zot3zufpW2fxifFDuPHyUXxt2XjWf66Jz1524dziT00ZDsAN0+u4c/F4Wh+4mkvrKsnLM8qL+lFXVcLyS2sB+I9VzQnFe8Ul1fzDVWM63ocPJ6quqjii7Ps3Xdbj6Un6nD77UbpDEBGRNOnNOb+1wJ/D3h8CZvQuHMl0Y2rKuGvJ+I737fcd3nBjZCfxsVXNzBlTTVG/fP55yThq4tytYvHHh/L6/UsoLszvtEf6zXdO8csdR8jPM26e08D58/DVjduYVl/FlBEVXH5JNdB5L/btC8cC8Ps33mH1hhbG1ZSxdsEYnt99lOWXDmfeuCEcP/Uhtzz+Mrv/9yRT6ir5zsomCvLzePA3e3hyy1uUFOazae1c6qpK2PX1xUxct6njDhm1lcX8++encf9/7aalta1TPZZNGkppYQFXjhvMbU/+EYD1n5vG0y//md/uOdbtPEfbc15bWczhv57u1verBxRy/NSZbs8v2wzt5Z1RREQkc1nXx9h2+4tm1wFLnHM3+/efB2Y4527rMt6XgC/5t+OAvT2YXTVwvEeBZh/lojPl4wLlorNk56PeOTc4idMLPDN7B/hTuuPwgrR+BykWCFY8QYoFghVPkGKB7I8nZpvdmz2/h4Hw+2KN8GWdOOe+C3y3F/PBzF52ziV2vDtLKRedKR8XKBedKR+9F6TOfpCWZ5BigWDFE6RYIFjxBCkWyO14enPO71ZgjJk1mFkhsAJ4NjlhiYiIiIgkX4/3/DrnzpnZbcAmIB/Y4JyLvNmriIiIiEhA9OohF865XwO/TlIs8fTqtIkso1x0pnxcoFx0pnxklyAtzyDFAsGKJ0ixQLDiCVIskMPx9PiCNxERERGRTNOrJ7yJiIiIiGSSQHd+c+XxyWbWamavmdl2M3vZl1WZ2Qtmts//HejLzcy+5XOyw8yawqaz2o+/z8xWp6s+iTKzDWZ2zMx2hpUlrf5mNs3nd7//rqW2homJkY91ZnbYryPbzWxZ2Gf3+LrtNbPFYeVRtx9/keoWX/6Uv2A1kMyszsw2m9luM9tlZv/oy3N2/cgWcZZt0tb1BOMJTDtsZuPC6r/dzE6a2dpU5iZGO5SW7S5GLA+Z2R4/v5+bWaUvH2Vmp8NytP5i84xVrwTjSUsbHSOWp8LiaDWz7SnMTWa02c65QL4IXUR3ABgNFAKvAhPSHVcf1bUVqO5S9q/A3X74buBBP7wMeA4wYCawxZdXAW/6vwP98MB0162b9Z8LNAE7+6L+QIsf1/x3l6a7zj3IxzrgjijjTvDbRn+gwW8z+fG2H+AnwAo/vB74crrrHCcXw4AmP1wGvOHrnLPrR7a84izbpK3rCcbTSgDbYV+/t4H6VOYmRjuUlu0uRiyLgAI//GBYLKPCx+synajzjFWvBONJ2rIhgTY6WixdPn8YuC+FucmINjvIe35z/fHJy4Ef+uEfAn8TVv64C3kJqDSzYcBi4AXnXJtz7l3gBWBJqoPuCefci0Bbl+Kk1N9/Vu6ce8mFtprHw6YVSDHyEcty4MfOuQ+dcweB/YS2najbj/+F/Angp/774bkNHOfcEefcNj/8HvA6oadL5uz6kS3iLNtYElrXkxRmENrhq4ADzrl4DxtJem6C1C5Hi8U597xz7px/+xKhZw3EdJF5xqpXt+OJo0/b6Hix+Gl9BtgYL8Ak5yYj2uwgd36jPT45XsOYyRzwvJm9YqEn4gHUOOeO+OG3gRo/HCsv2ZavZNW/1g93Lc9Et/nDQhvCDj0lmo9BwF/D/mlkTD7MbBQwFdiC1o+s0mXZQnLW9UQFtR1eQefOSzpy0y6o290XCO0BbNdgZn80s9+b2ZywGGPNM1a9EhW0NnoOcNQ5ty+sLGW5CXKbHeTOby65wjnXBCwF1pjZ3PAP/a+bnL0tR67X3/sO0AhcChwhdCgrZ5jZAOBnwFrn3Mnwz7R+ZLYoyzZd63rg2mF/ruc1wNO+KDDtQFC2OzO7FzgHPOGLjgAjnXNTgduBJ82svLvT60W9ArNswtxA5x9OKctN0NvsIHd+u/X45GzgnDvs/x4Dfk7ocMhRv3u//ZDEMT96rLxkW76SVf/DdD4clpF5cc4ddc595Jw7DzxGaB2BxPPxF0KHlQq6lAeWmfUj1Ig+4Zz7T1+s9SMLRFu2SVzXExLQdngpsM05d9THlpbchAnUdmdmNwKfBFb6DhX+9IK/+OFXCJ1XO/Yi84xVr24LWhvtv38t8FRYjCnJTSa02UHu/ObE45PNrNTMytqHCZ3Ev5NQXduvblwNPOOHnwVW+SskZwIn/KGETcAiMxvoD7cs8mWZKin195+dNLOZ/vynVWHTyhjtjYb3aULrCITyscLM+ptZAzCG0MUAUbcf/w9iM3Cd/354bgPHL7PvAa87574Z9pHWjwwXa9kma11PMJagtsOd9tylIzddBGa7M7MlwF3ANc6598PKB5tZvh8eTSgXb15knrHqlUg8QWujFwB7nHMdpwikIjcZ02a7XlyF2tcvQlcBvkHo18m96Y6nj+o4mtBVnq8Cu9rrSei8n98C+4D/Bqp8uQGP+Jy8BjSHTesLhE6m3w/clO66JZCDjYQOx5wldP7OF5NZf6CZUEN0APg2/uEuQX3FyMePfH13+MZiWNj49/q67SXsqtdY249f51p8np4G+qe7znFycQWhw2M7gO3+tSyX149secVZtklb1xOIJXDtMFBKaC9gRVhZynITox1Ky3YXI5b9hM4JbV931vtx/9Yvw+3ANuBTF5tnrHolGE9a2uhosfjyHwC3dhk3FbnJiDZbT3gTERERkZwR5NMeRERERESSSp1fEREREckZ6vyKiIiISM5Q51dEREREcoY6vyIiIiKSM9T5FREREZGcoc6viIiIiOQMdX5FREREJGf8PzzDsBetFVYZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "#20540\tloss=0.051598\tdelay=50: 20540it [15:24, 19.05it/s]"
     ]
    }
   ],
   "source": [
    "def trainer_thread_method():\n",
    "    \"\"\" train model on batches, emulate network latency \"\"\"\n",
    "    train_loader = torch.utils.data.DataLoader(\n",
    "        datasets.MNIST('../data', train=True, download=True,\n",
    "                       transform=transforms.Compose([\n",
    "                           transforms.ToTensor(),\n",
    "                           transforms.Normalize((0.1307,), (0.3081,)),\n",
    "                           transforms.Lambda(lambda x: x.view(-1))\n",
    "                       ])),\n",
    "        batch_size=batch_size, shuffle=True,\n",
    "    )\n",
    "\n",
    "    while True:\n",
    "        for xb, yb in train_loader:\n",
    "            xb, yb = xb.to(device), yb.to(device)\n",
    "\n",
    "            with lock_model:\n",
    "                model.train(True)\n",
    "                initial_step_index = len(train_history)\n",
    "                logits = model(xb)\n",
    "                loss = F.cross_entropy(logits, yb)\n",
    "                \n",
    "                opt.zero_grad()\n",
    "                loss.backward()\n",
    "                grads = [param.grad.clone() if param.grad is not None else None\n",
    "                         for param in non_expert_params]\n",
    "\n",
    "            emulate_latency()\n",
    "\n",
    "            with lock_model:\n",
    "                model.train(True)\n",
    "                opt.zero_grad()\n",
    "                for param, grad in zip(non_expert_params, grads):\n",
    "                    param.grad[...] = grad\n",
    "                opt.step()\n",
    "                train_history.append(dict(\n",
    "                    loss=loss.item(), \n",
    "                    delay_steps=len(train_history) - initial_step_index,\n",
    "                ))\n",
    "                progress.desc = f'#{len(train_history)}\\tloss={loss.item():4f}\\tdelay={train_history[-1][\"delay_steps\"]}'\n",
    "                progress.update(1)\n",
    "                \n",
    "                if len(train_history) % eval_interval == 0 or len(train_history) >= total_steps:\n",
    "                    need_to_eval.set()\n",
    "            if len(train_history) >= total_steps:\n",
    "                return\n",
    "                \n",
    "\n",
    "def evaluate():\n",
    "    test_loader = torch.utils.data.DataLoader(\n",
    "            datasets.MNIST('../data', train=False, transform=transforms.Compose([\n",
    "                transforms.ToTensor(),\n",
    "                transforms.Normalize((0.1307,), (0.3081,)),\n",
    "                transforms.Lambda(lambda x: x.view(-1))\n",
    "            ])),\n",
    "            batch_size=batch_size, num_workers=4, pin_memory=True,\n",
    "    )\n",
    "    with lock_model, torch.no_grad():\n",
    "        model.train(False)\n",
    "        loss_numerator = acc_numerator = denominator = 0.0\n",
    "        for xb, yb in test_loader:\n",
    "            xb, yb = xb.to(device), yb.to(device)\n",
    "            logits = model(xb)\n",
    "            loss_numerator += F.cross_entropy(logits, yb).item() * len(yb)\n",
    "            acc_numerator += (logits.argmax(-1).to(yb.dtype) == yb).to(torch.float32).sum()\n",
    "            denominator += len(yb)\n",
    "    return dict(loss=loss_numerator / denominator,\n",
    "                acc=acc_numerator / denominator,\n",
    "                num_updates=len(train_history))\n",
    "            \n",
    "\n",
    "# finally, run training\n",
    "trainers = [threading.Thread(target=trainer_thread_method) for i in range(num_trainers)]\n",
    "for trainer in trainers:\n",
    "    trainer.start()\n",
    "    \n",
    "while len(train_history) < total_steps:\n",
    "    need_to_eval.wait(), need_to_eval.clear()\n",
    "    val_metrics = evaluate()\n",
    "    val_history.append(val_metrics)\n",
    "        \n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=[12, 6])\n",
    "    plt.subplot(1, 2, 1); plt.title('train loss'); plt.ylim(0, 10)\n",
    "    plt.plot([info['loss'] for info in train_history])\n",
    "    \n",
    "    \n",
    "    plt.subplot(1, 2, 2); plt.title('val accuracy'); plt.grid()\n",
    "    plt.plot(*zip(*((info['num_updates'], info['acc']) for info in val_history)))\n",
    "    plt.show()\n",
    "\n",
    "for trainer in trainers:\n",
    "    trainer.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logs/delay1000ms_ffn1024_seed1337.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "#20543\tloss=0.018364\tdelay=72: 20543it [15:40, 19.05it/s]"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pickle\n",
    "!mkdir -p logs\n",
    "\n",
    "num_files = len(os.listdir('logs'))\n",
    "fname = f'logs/delay{delay_ms}ms_ffn{layer_dim}_seed{seed}.pkl'\n",
    "print(fname)\n",
    "with open(fname, 'wb') as f_out:\n",
    "    pickle.dump(dict(train_history=train_history, val_history=val_history), f_out)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py38",
   "language": "python",
   "name": "py38"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}