model_free_rl.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Author: Mikita Sazanovich\n",
    "\n",
    "%matplotlib inline\n",
    "from time import time\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "eps_greedy = 0.2\n",
    "gamma = 0.9\n",
    "field_size = 10\n",
    "cows = [(field_size - 1, field_size - 1)]\n",
    "pickup_reward = 100\n",
    "\n",
    "max_iterations = 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Action:\n",
    "    def __init__(self, to_state, reward):\n",
    "        self.to_state = to_state\n",
    "        self.reward = reward\n",
    "\n",
    "\n",
    "class State:\n",
    "    def __init__(self, x, y, cow_mask):\n",
    "        self.x = x\n",
    "        self.y = y\n",
    "        self.cow_mask = cow_mask\n",
    "        self.actions = []\n",
    "\n",
    "    def is_terminal(self):\n",
    "        return self.x == 0 and self.y == 0 and self.cow_mask != 0\n",
    "\n",
    "    def add_action(self, action):\n",
    "        self.actions.append(action)\n",
    "\n",
    "    def __eq__(self, other):\n",
    "        if isinstance(other, State):\n",
    "            return (self.x == other.x\n",
    "                    and self.y == other.y\n",
    "                    and self.cow_mask == other.cow_mask)\n",
    "        else:\n",
    "            return False\n",
    "\n",
    "    def __hash__(self):\n",
    "        return hash((self.x, self.y, self.cow_mask))\n",
    "\n",
    "    def __str__(self):\n",
    "        return str(self.x) + \" \" + str(self.y) + \" \" + str(self.cow_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_states(N, cows, pickup_reward):\n",
    "    def is_valid_cell(x, y):\n",
    "        return 0 <= x < N and 0 <= y < N\n",
    "\n",
    "    dirs = [(1, 0), (-1, 0), (0, 1), (0, -1)]\n",
    "    S = []\n",
    "    max_mask = 1 << len(cows)\n",
    "    for x in range(N):\n",
    "        for y in range(N):\n",
    "            for mask in range(max_mask):\n",
    "                S.append(State(x, y, mask))\n",
    "    for s in S:\n",
    "        if s.is_terminal():\n",
    "            s.add_action(Action(s, 0))\n",
    "            continue\n",
    "        x, y, mask = s.x, s.y, s.cow_mask\n",
    "        for drt in dirs:\n",
    "            xx = x + drt[0]\n",
    "            yy = y + drt[1]\n",
    "            if is_valid_cell(xx, yy):\n",
    "                new_mask = mask\n",
    "                if (xx, yy) in cows:\n",
    "                    cow_index = cows.index((xx, yy))\n",
    "                    new_mask |= 1 << cow_index\n",
    "                to_state = S[S.index(State(xx, yy, new_mask))]\n",
    "                if to_state.is_terminal():\n",
    "                    reward = pickup_reward * bin(new_mask).count('1')\n",
    "                else:\n",
    "                    reward = 0\n",
    "                s.add_action(Action(to_state, reward))\n",
    "    return S\n",
    "\n",
    "\n",
    "S = build_states(field_size, cows, pickup_reward)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "def measure_policy_performance(S, pi, max_iters=1000):\n",
    "    state = S[S.index(State(0, 0, 0))]\n",
    "    for _ in range(max_iters):\n",
    "        if pi[state] is None:\n",
    "            break\n",
    "        action = pi[state]\n",
    "        state = action.to_state\n",
    "        if state.is_terminal():\n",
    "            return action.reward\n",
    "    return 0.0\n",
    "\n",
    "\n",
    "def plot_method_results(S, method):\n",
    "    seed_n = 10\n",
    "    seeds = range(seed_n)\n",
    "    learning_started = time()\n",
    "    perf_mean = None\n",
    "    for seed in seeds:\n",
    "        np.random.seed(seed)\n",
    "        perf = method(S, max_iterations=max_iterations)\n",
    "        if perf_mean is None:\n",
    "            perf_mean = np.zeros(len(perf), dtype=float)\n",
    "        perf_mean += perf\n",
    "    perf_mean /= seed_n\n",
    "    print(f'measuring with {seed_n} seeds took', time() - learning_started)\n",
    "    line, = plt.plot(perf_mean, label=method, c='C8')\n",
    "    plt.legend(handles=[line])\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "measuring with 10 seeds took 0.40105700492858887\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl0VeW9//H3NxMooMggRaKG/rAqCQlDQBAHBkGsVhQEobQXay16LWqtrVinWnTVgf7Q0uu61onB8oMoON1er5ZJRb1IIgZEsMIFvMQBkRPRMJ2d5Pn9cQYDZDxDzjnJ57VWVs7ZZw/f7LPzyZPn7P1sc84hIiItV1qiCxARkfhS0IuItHAKehGRFk5BLyLSwinoRURaOAW9iEgLp6AXEWnhFPQiIi2cgl5EpIXLSHQBAF26dHE5OTmJLkNEJKW89957XznnujY0X1IEfU5ODiUlJYkuQ0QkpZjZJ42ZT103IiItnIJeRKSFU9CLiLRwCnoRkRZOQS8i0sI1GPRm9rSZfWlmG2tM62Rmy8xsS/D7CcHpZmZzzGyrmW0ws/7xLF5ERBrWmBb9PGDMEdNuA1Y4504DVgSfA1wEnBb8mgb8e2zKFBGRSDV4Hr1z7k0zyzli8lhgWPDxfOB1YEZw+gIXuD/hGjPraGbdnXOfx6pgkZDKyn2UlS2gqvpgoksRiVjXLiM57rj8uG4j0gumutUI7y+AbsHHPYCdNeYrC047KujNbBqBVj+nnHJKhGVIa+bzvcX/bPtT8JkltBaRSLVp0y1pgz7MOefMrMl3GHfOPQ48DlBYWKg7lEuT+f1fAXDO0DW0adPgVeAirVakZ93sMrPuAMHvXwanfwqcXGO+7OA0kZjzez4AMjM7JrgSkeQWadC/DEwNPp4KvFRj+r8Ez74ZDOxV/7zEi+ftISPjONLSMhNdikhSa7DrxswWEfjgtYuZlQG/Bx4AnjWznwOfABODs78C/BDYCuwHfhaHmkUA8PzlZGZ2SnQZIkmvMWfdTK7jpZG1zOuAX0ZblEhj+D0fWVkKepGG6MpYSVme51OLXqQRFPSSsvx+H1kKepEGKeglJTnn8LxyMtV1I9IgBb2kpKqqCpzz1KIXaQQFvaQkv38PAJmZJyS4EpHkp6CXlOSFLpZS141IgxT0kpL8XjmAum5EGkFBLynJ84eGP+ic4EpEkp+CXlJSaJybrCz10Ys0REEvKcnz7yEtrS3p6ccmuhSRpKegl5TkeeXqnxdpJAW9pCS/59MZNyKNpKCXlOT5fTqHXqSRFPSSkvyejyydcSPSKAp6SUka50ak8RT0knKqqg5RVbVPH8aKNJKCXlKO52mcG5GmUNBLyvnuYim16EUaQ0EvKcfzB8a50d2lRBpHQS8pJzxypYJepFEU9JJy1HUj0jQKekk5nn8PZulkZByX6FJEUoKCXlKO3ysnM/MEzHT4ijSGflMk5Xiehj8QaQoFvaQcv9+nD2JFmkBBLynH83xkZWmcG5HGUtBLyvG8crXoRZpAQS8pxbkqPO9rstRHL9JoCnpJKZ5XDjiNXCnSBAp6SSl+f/BiKXXdiDRaVEFvZjeb2YdmttHMFplZWzPraWbvmtlWMysys6xYFSsSaNFr+AORpog46M2sB3AjUOicywPSgUnAg8DDzrleQDnw81gUKgLfDX+grhuRxou26yYDOMbMMoBjgc+BEcCS4Ovzgcui3IZImKeuG5EmizjonXOfAn8C/pdAwO8F3gO+ds5VBmcrA3pEW6RISLhFr7NuRBotmq6bE4CxQE/gJKAdMKYJy08zsxIzK9m9e3ekZUgr43k+MjKOIy0tM9GliKSMaLpuLgC2O+d2O+c84HlgKNAx2JUDkA18WtvCzrnHnXOFzrnCrl27RlGGtCaeX+PciDRVNEH/v8BgMzvWzAwYCWwCVgFXBOeZCrwUXYki3/F7PvXPizRRNH307xL40HUd8EFwXY8DM4Bfm9lWoDPwVAzqFAGCI1dqnBuRJsloeJa6Oed+D/z+iMnbgEHRrFekLp6/nOM65Ce6DJGUoitjJWU45/BrLHqRJlPQS8qoqqrAOU8XS4k0kYJeUobfvwfQxVIiTaWgl5QRHudGLXqRJlHQS8oIXRWrFr1I0yjoJWWExrnRyJUiTaOgl5QRbtGr60akSRT0kjI8z0daWlvS049NdCkiKUVBLylD49yIREZBLynD7/nUbSMSAQW9pIxAi15BL9JUCnpJGX6vnKxMDWgm0lQKekkZgZEr1Ucv0lQKekkJVVWHqKrap4ulRCKgoJeU4HmBcW7URy/SdAp6SQnfjXOjrhuRplLQS0rw+0Pj3OjDWJGmUtBLSvA8jXMjEikFvaQEjXMjEjkFvaQEz+/DLJ2MjOMSXYpIylHQS0rwez4yMjpipkNWpKn0WyMpwdM4NyIRU9BLSvBrnBuRiCnoJSV4XrmuihWJkIJeUkJgnBsFvUgkFPSS9JyrwvO+VoteJEIKekl6geEPnFr0IhFS0EvS84fGudFtBEUioqCXpOeFx7lRi14kEgp6SXqh4Q8yszSgmUgkFPSS9NSiF4lOVEFvZh3NbImZfWRmm81siJl1MrNlZrYl+F0dqxKV70au7JjgSkRSU7Qt+j8DrzrnzgAKgM3AbcAK59xpwIrgc5GIBca56UBaWlaiSxFJSREHvZkdD5wHPAXgnPM7574GxgLzg7PNBy6Ltkhp3TwNfyASlWha9D2B3cBcM3vfzJ40s3ZAN+fc58F5vgC61bawmU0zsxIzK9m9e3cUZUhL5/d86p8XiUI0QZ8B9Af+3TnXD9jHEd00zjkHuNoWds497pwrdM4Vdu3aNYoypKXzvHJdLCUShWiCvgwoc869G3y+hEDw7zKz7gDB719GV6K0duq6EYlOxEHvnPsC2GlmpwcnjQQ2AS8DU4PTpgIvRVWhtGrOOXXdiEQpI8rlbwAWmlkWsA34GYE/Hs+a2c+BT4CJUW5DWrGqqgqc89R1IxKFqILeOVcKFNby0sho1isS4g9fLKXLMUQipStjJal9d7GUWvQikVLQS1ILjXOTpXFuRCKmoJekFhrnRi16kcgp6CWpfdd1oz56kUgp6CWp+T0faWltSE8/NtGliKSsaE+vTHme51FWVsbBgwcTXYrUwvPOp2PHwXz00UeJLkUkYdq2bUt2djaZmZkRLd/qg76srIwOHTqQk5ODmSW6HDnC/v07cM6jXbvTEl2KSEI459izZw9lZWX07NkzonW0+q6bgwcP0rlzZ4V8knKuErNW3x6RVszM6Ny5c1S9Dq0+6AGFfBJzrkpBL61etBmloJekFmjRpye6DJGUpqBPAatXryY3N5e+ffty4MCBmK33j3/842HPzz777Jisd968eXz22WdNXu6xxx5jwYIF4efOVeNcdYMt+nnz5jF9+vQmby8aO3bsIC8vL6brvOaaa9i0aRNw9HsTrSPfk5rbiofS0lJeeeWVOl+///776dWrF6effjqvvfZag+sbM2YMHTt25JJLLjls+r/927/Rq1cvzIyvvvoqPP2jjz5iyJAhtGnThj/96U+HLTNnzhzOPPNMpkyZwqxZs+jbty99+/YlLy+P9PR0fD5fE3/a+rf38MMPk5ubS15eHpMnT07MiR/OuYR/DRgwwCXKpk2bErbt+hw6dMhVVFQ455y79tpr3TPPPBPzbbRr1y7m63TOufPPP98VFxfX+lplZWWj11NVdcjt3bvBHTr0Vb3zzZ071/3yl79sUo3R2r59u8vNzY3b+iN5b+rbt/W9J/FQ33vy4Ycfuvz8fHfw4EG3bds29/3vf7/B42L58uXu5ZdfdhdffPFh09etW+e2b9/uTj31VLd79+7w9F27drm1a9e622+/3c2aNeuwZU4//XS3c+fOo7bx8ssvu+HDhzf2RzxMXdsrKytzOTk5bv/+/c455yZMmODmzp3rnHPO5/M1aRu1ZRVQ4hqRsWrR1/Dxx/fy3rofx/Tr44/vbVINmzdv5pZbbuH000/n448/5sknn+TZZ5/lrrvuYsqUKbz++uuHtWqmT5/OvHnzAMjJyeH3v/89/fv3p0+fPuFTEisqKvjZz35Gnz59yM/PZ+nSpdx2220cOHCAvn37MmXKFADat28PBP74//a3vyUvL48+ffpQVFQEwOuvv86wYcO44oorOOOMM5gyZQqBY+07S5YsoaSkhClTpoT/A8nJyWHGjBn079+f5557jieeeIKBAwdSUFDA+PHj2b9/PwD33HNPuDU0bNgwZsz4HcOH/5jc3LNYvXp1vftt586dDBs2jNNOO40//OEPANx999088sgj4XnuuOMO/vznP9e6/KRJk/jP//zP8POrrrqKJUuWsGPHDs4991z69+9P//79eeedd45a9sj/KC655BJef/11AP7xj38wZMgQ+vfvz4QJE6ioqKjzZxg2bBglJSW1vjd/+9vfGDRoEH379uXaa6+lqqoKCLxnt9xyCwUFBfz3f/83M2fOZODAgeTl5TFt2jScc7W+J6FtASxatIg+ffqQl5fHjBkzwvW0b9+eO+64g4KCAgYPHsyuXbuOqnnt2rUMGTKEfv36cfbZZ/PPf/4Tv9/P3XffTVFREX379g0fPyEvvfQSkyZNok2bNvTs2ZNevXqxdu1aiouLyc/P5+DBg+zbt4/c3Fw2btwIwMiRI+nQocNR2+/Xrx85OTlHTT/xxBMZOHDgUacjXnfddWzbto2LLrqIhx9++LDXFi1axOTJk8PPFyxYQH5+PgUFBfz0pz8FYPfu3YwfP56BAwcycOBA3n777Xq3B1BZWcmBAweorKxk//79nHTSSQDccMMNjBgxgoULF8a9la+gTwL79u1j7ty5nHPOOfziF7+gd+/ebNiwgX79+nHNNddw6aWXMmvWLBYuXNjgurp06cK6dev413/913Bo3nvvvRx//PF88MEHbNiwgREjRvDAAw9wzDHHUFpaetR6n3/+eUpLS1m/fj3Lly/nt7/9LZ9/Hrg75Pvvv88jjzzCpk2b2LZtW/hAD7niiisoLCxk4cKFlJaWcswxxwDQuXNn1q1bx6RJkxg3bhzFxcWsX7+eM888k6eeeqrWn6Wy0s+qVf+P2bMfCod3XdauXcvSpUvZsGEDzz33HCUlJVx99dXhrqDq6moWL17MT37yk1qXv/LKK3n22WcB8Pv9rFixgosvvpgTTzyRZcuWsW7dOoqKirjxxhvrraOmr776ivvuu4/ly5ezbt06CgsLmT17doPLHfnebN68maKiIt5++21KS0tJT08Pv2f79u3jrLPOYv369ZxzzjlMnz6d4uJiNm7cyIEDB/j73/9e53sC8NlnnzFjxgxWrlxJaWkpxcXFvPjii+F1Dx48mPXr13PeeefxxBNPHFXrGWecwerVq3n//feZOXMmt99+O1lZWcycOZMrr7yS0tJSrrzyysOW+fTTTzn55JPDz7Ozs/n0008ZOHAgl156KXfeeSe33norP/nJT2LePfbYY49x0kknsWrVKm6++ebw9P379/Pqq68yfvx4AD788EPuu+8+Vq5cyfr168MNhJtuuombb76Z4uJili5dyjXXXFPv9nr06MFvfvMbTjnlFLp3787xxx/P6NGjgcAf71mzZvHOO++Qm5vLDTfcwPr162P684bodIYafvCDuxKy3e7du5Ofn8+TTz7JGWecEdW6xo0bB8CAAQN4/vnnAVi+fDmLFy8Oz3PCCfUPJ/DWW28xefJk0tPT6datG+effz7FxcUcd9xxDBo0iOzsbAD69u3Ljh07OOeccxqsq+Yv+8aNG7nzzjv5+uuvqaio4MILL6x1mcsuuzj8s+zYsaPe9Y8aNYrOnQMDn40bN4633nqLX/3qV3Tu3Jn333+fXbt20a9fv/A8R7rooou46aabOHToEK+++irnnXcexxxzDHv37mX69OnhgP34448b/FlD1qxZw6ZNmxg6dCgQ+AMyZMiQRi8fsmLFCt577z0GDhwIwIEDBzjxxBMBSE9PD4cTwKpVq3jooYfYv38/Pp+P3NxcfvSjH9W57uLiYoYNG0bodp5TpkzhzTff5LLLLiMrKyv83+OAAQNYtmzZUcvv3buXqVOnsmXLFswMz/Oa/PPVdPfddzNw4EDatm3LnDlzolpXU/zHf/wHQ4cOpVOnwJhKK1euZMKECXTp0gUgPH358uWHfbbxzTffUFFREf5v+Ejl5eW89NJLbN++nY4dOzJhwgT+9re/hRscAwYMYMCAARw8eJC//vWvDBo0iPvvv59f//rXMf35FPRJYMmSJTz11FOMGzeOSZMmMXXqVE499dRa583IyKC6ujr8/Mh/+dq0aQMEAqCysjLmtYbW39RttGvXLvz4qquu4sUXX6SgoIB58+aFuzmOlJkVODwzMto0uJ0jTz8LPb/mmmuYN28eX3zxBVdffXWdy7dt25Zhw4bx2muvUVRUxKRJk4DAB2ndunVj/fr1VFdX07Zt26OWres9cc4xatQoFi1aVG/tDXHOMXXqVO6///5a605PTw9v9/rrr6ekpISTTz6Ze+65J6ougczMzPB+rOu9vuuuuxg+fDgvvPACO3bsYNiwYQ2ut0ePHuzcuTP8vKysjB49egCwZ88eKioq8DyPgwcPHnbcxNPixYsP67apS3V1NWvWrKn1OKjN8uXL6dmzZ/gP6bhx43jnnXfCQV9ZWckrr7zC008/zdatW5k5c2ad/3VGQ103SWD06NEUFRWxevVqjj/+eMaOHcsFF1xQayv21FNPZdOmTRw6dIivv/6aFStWNLj+UaNG8eijj4afl5eXA4Ff5NpaYOeeey5FRUVUVVWxe/du3nzzTQYNGtTon6dDhw58++23db7+7bff0r17dzzPq7c7ylUHgqUxp1cuW7YMn8/HgQMHePHFF8Ot6Msvv5xXX32V4uLiOv9zCLnyyiuZO3cuq1evZsyYMUCgxdq9e3fS0tJ45plnwn3jNeXk5FBaWkp1dTU7d+5k7dq1AAwePJi3336brVu3AoGukMb+R1DzvRk5ciRLlizhyy8Dt1/2+Xx88sknRy0TCvUuXbpQUVHBkiVLwq/V9Z4MGjSIN954g6+++oqqqioWLVrE+eef36gaIbB/QiEd+qyovu0BXHrppSxevJhDhw6xfft2tmzZEj6+rr32Wu69916mTJly2OcF8bR3717eeOMNxo4dG542YsQInnvuOfbs2QMQPhNn9OjR/OUvfwnPV1paWu+6TznlFNasWcP+/ftxzrFixQrOPPNMAGbPns0PfvADli5dyi233MLGjRuZMWNG+L+1WFLQJ5HOnTtz0003UVpayh//+MdwS62mk08+mYkTJ5KXl8fEiRPp169fg+u98847KS8vJy8vj4KCAlatWgXAtGnTyM/PD3/gF3L55ZeHP4QaMWIEDz30EN/73vca/XNcddVVXHfddXWeDnrvvfdy1llnMXTo0Hq7qkKnVjbmYpFBgwYxfvx48vPzGT9+PIWFgRufZWVlMXz4cCZOnFjr/qxp9OjRvPHGG1xwwQVkZWUBcP311zN//nwKCgr46KOPam1hDh06lJ49e9K7d29uvPFG+vfvD0DXrl2ZN28ekydPJj8/nyFDhjR6zJ6a703v3r257777GD16NPn5+YwaNSr8mUlNHTt25Be/+AV5eXlceOGF4a4eqPs96d69Ow888ADDhw+noKCAAQMGHBZ4Dbn11lv53e9+R79+/Q5r8Q8fPpxNmzbV+mFsbm4uEydOpHfv3owZM4ZHH32U9PR0FixYQGZmJj/+8Y+57bbbKC4uZuXKlUCg8TFhwgRWrFhBdnZ2+JTMOXPmkJ2dTVlZGfn5+eE+8y+++ILs7Gxmz57NfffdR3Z2Nt98802tP8MLL7zA6NGjD3tvc3NzueOOOzj//PMpKCgId6XMmTOHkpIS8vPz6d27N4899li92zvrrLO44oorwidIVFdXM23aNADy8/MpLS1l/vz5nHvuuY3e55GwI8+aSITCwkIXOgOguW3evDn8F1aSy/79n1BdfYj27X8Q8Tqqq6vDZ/ucdprGy5HUVVtWmdl7zrnabud6GLXoJWlFO87Npk2b6NWrFyNHjlTIS6umD2MlaTlXRVradx96vfbaa0f12/bs2ZMXXnih1uV79+7Ntm3bDpv2wQcfhM+JDmnTpg3vvvtujKpu2OWXX8727dsPm/bggw82+BmCSKQU9JK0nKvE0r7rV7/wwgujDsM+ffo0+AFavNX1h0kkXtR1A0dd3SmJF7h0WyNXikD0GdXqg75t27bs2bNHYZ9knAucxpimkSullXPBG4809tz92rT65lLo1Kzdu3cnuhSpobraw+//ksxMP+npem+kdQvdSjBSrT7oMzMzI749l8RPefm7rHv/Wvr1XUCnTgMSXY5ISmv1XTeSnDwvdPVupwRXIpL6FPSSlPxe4JLzrCwFvUi0FPSSlDx/YIyRzMz6R9oUkYYp6CUp+T0fGRkdSEvLSnQpIikv6qA3s3Qze9/M/h583tPM3jWzrWZWZGb6TZUm87xyteZFYiQWLfqbgM01nj8IPOyc6wWUAz+PwTaklfH8PjIza79JiIg0TVRBb2bZwMXAk8HnBowAQgNhzwcui2Yb0jr5PZ8+iBWJkWhb9I8AtwKh2+t0Br52zoUGpi4DekS5DWmFAi16Bb1ILEQc9GZ2CfClc+69CJefZmYlZlaiq1KlJuccfq+cLPXRi8RENC36ocClZrYDWEygy+bPQEf7biSqbODT2hZ2zj3unCt0zhWG7qcoAlBVVYFzfjLVdSMSExEHvXPud865bOdcDjAJWOmcmwKsAq4IzjYVeCnqKqVV8fuDF0up60YkJuJxHv0M4NdmtpVAn/1TcdiGtGBe8KpY9dGLxEZMBjVzzr0OvB58vA0YFIv1SusUHudGXTciMaErYyXpqOtGJLYU9JJ0PC80zo2CXiQWFPSSdPyej7S0NqSnH5voUkRaBAW9JB3PHxjnJnChtYhES0EvScfv+cjSODciMaOgl6TjeT6dcSMSQwp6STp+v09DFIvEkIJeko7nlevUSpEYUtBLUqmuPkRVVYW6bkRiSEEvSUUXS4nEnoJekorGuRGJPQW9JBW/xrkRiTkFvSQVT103IjGnoJek4g+Oc6P7xYrEjoJekkqgRZ9GRsbxiS5FpMVQ0EtS8bxyMjM7YqZDUyRW9NskScXv+cjK0jg3IrGkoJek4vl9OrVSJMYU9JJU/J7GuRGJNQW9JBXPK9cZNyIxpqCXpOFcVfDDWAW9SCwp6CVpeN7XgNPFUiIxpqCXpOEPj3OjPnqRWFLQS9Lw/IFxbnR6pUhsKeglaWjkSpH4UNBL0gh13eisG5HYUtBL0vD8gQHNMjM7JrgSkZZFQS9Jw++Vk57enrS0NokuRaRFUdBL0vA8n7ptROJAQS9JIzDOjc64EYk1Bb0kDb/nI0vn0IvEXMRBb2Ynm9kqM9tkZh+a2U3B6Z3MbJmZbQl+12+uNIrnletesSJxEE2LvhK4xTnXGxgM/NLMegO3ASucc6cBK4LPRerlnMPv92n4A5E4iDjonXOfO+fWBR9/C2wGegBjgfnB2eYDl0VbpLR8VVUVOOdXi14kDmLSR29mOUA/4F2gm3Pu8+BLXwDd6lhmmpmVmFnJ7t27Y1GGpDC/X+PciMRL1EFvZu2BpcCvnHPf1HzNOecAV9tyzrnHnXOFzrnCrl27RluGpDjPC45zo7NuRGIuqqA3s0wCIb/QOfd8cPIuM+sefL078GV0JUprEB7nRl03IjEXzVk3BjwFbHbOza7x0svA1ODjqcBLkZcnrUWo60YfxorEXkYUyw4Ffgp8YGalwWm3Aw8Az5rZz4FPgInRlSitgeeFxrlRH71IrEUc9M65twCr4+WRka5XWie/V05aWhbp6e0SXYpIi6MrYyUpBIY/6ESgR1BEYklBL0nB7/l0wxGROFHQS1LwPF0VKxIvCnpJCp5f49yIxIuCXpKCXy16kbhR0EvCVVcfoqqqQqdWisSJgl4SLjzOjbpuROJCQS8Jp3FuROJLQS8J59c4NyJxpaCXhPPC49yoj14kHhT0knD+8Dg3atGLxIOCXhIu0EefRmZmx0SXItIiKegl4QLj3HTETIejSDzoN0sSTuPciMSXgl4SzvP7yNIZNyJxo6CXhPN75WrRi8SRgl4SzvPUoheJJwW9JJRzVXheuca5EYkjBb0klOd9DTiNXCkSRwp6SajQODfqoxeJHwW9JFRo5MqsLA1oJhIvCnpJKC80oJn66EXiRkEvCaWRK0XiT0EvCaWRK0XiT0EvCeX3fKSntyctrU2iSxFpsRT0klCebgouEncKekkoz+9T/7xInCnoJaH8Xrla9CJxpqCXhPI8tehF4k1BLwnjnMPv9+kcepE4i0vQm9kYM/unmW01s9visQ1JfVVVFTjnV9eNSJzFPOjNLB14FLgI6A1MNrPesd6OpL7wODfquhGJq3i06AcBW51z25xzfmAxMDYO25EUFx7nJlPj3IjEU0Yc1tkD2FnjeRlwVhy2w2efPcf/7nwqHquWZlBVuQ/QODci8RaPoG8UM5sGTAM45ZRTIlpHZmZH2h3bK5ZlSTPrlHku7dufmegyRFq0eAT9p8DJNZ5nB6cdxjn3OPA4QGFhoYtkQ127jqJr11GRLCoi0mrEo4++GDjNzHqaWRYwCXg5DtsREZFGiHmL3jlXaWbTgdeAdOBp59yHsd6OiIg0Tlz66J1zrwCvxGPdIiLSNLoyVkSkhVPQi4i0cAp6EZEWTkEvItLCKehFRFo4cy6ia5ViW4TZbuCTCBfvAnwVw3JiTfVFR/VFL9lrVH2RO9U517WhmZIi6KNhZiXOucJE11EX1Rcd1Re9ZK9R9cWfum5ERFo4Bb2ISAvXEoL+8UQX0ADVFx3VF71kr1H1xVnK99GLiEj9WkKLXkRE6pEyQd/QDcfNrI2ZFQVff9fMcpqxtpPNbJWZbTKzD83splrmGWZme82sNPh1d3PVF9z+DjP7ILjtklpeNzObE9x/G8ysfzPWdnqN/VJqZt+Y2a+OmKfZ95+ZPW1mX5rZxhrTOpnZMjPbEvxe6+2xzGxqcJ4tZja1mWqbZWYfBd/smpSLAAAEHUlEQVS/F8ysYx3L1nssxLnGe8zs0xrv4w/rWLbe3/c41ldUo7YdZlZax7LNsg9jxjmX9F8Ehjv+H+D7QBawHuh9xDzXA48FH08Cipqxvu5A/+DjDsDHtdQ3DPh7AvfhDqBLPa//EPgvwIDBwLsJfK+/IHB+cEL3H3Ae0B/YWGPaQ8Btwce3AQ/WslwnYFvw+wnBxyc0Q22jgYzg4wdrq60xx0Kca7wH+E0jjoF6f9/jVd8Rr/9f4O5E7sNYfaVKi74xNxwfC8wPPl4CjDQza47inHOfO+fWBR9/C2wmcO/cVDIWWOAC1gAdzax7AuoYCfyPcy7SC+hixjn3JuA7YnLN42w+cFkti14ILHPO+Zxz5cAyYEy8a3PO/cM5Vxl8uobA3d0Spo791xiN+X2PWn31BbNjIrAo1ttNhFQJ+tpuOH5kkIbnCR7se4HOzVJdDcEuo37Au7W8PMTM1pvZf5lZbrMWBg74h5m9F7xf75Eas4+bwyTq/uVK5P4L6eac+zz4+AugWy3zJMO+vJrAf2i1aehYiLfpwe6lp+vo+kqG/XcusMs5t6WO1xO9D5skVYI+JZhZe2Ap8Cvn3DdHvLyOQHdEAfAX4MVmLu8c51x/4CLgl2Z2XjNvv0HBW09eCjxXy8uJ3n9HcYH/4ZPutDUzuwOoBBbWMUsij4V/B/4P0Bf4nED3SDKaTP2t+aT/faopVYK+MTccD89jZhnA8cCeZqkusM1MAiG/0Dn3/JGvO+e+cc5VBB+/AmSaWZfmqs8592nw+5fACwT+Pa6pUTd1j7OLgHXOuV1HvpDo/VfDrlCXVvD7l7XMk7B9aWZXAZcAU4J/iI7SiGMhbpxzu5xzVc65auCJOrad0GMxmB/jgKK65knkPoxEqgR9Y244/jIQOrvhCmBlXQd6rAX7854CNjvnZtcxz/dCnxmY2SAC+75Z/hCZWTsz6xB6TOBDu41HzPYy8C/Bs28GA3trdFE0lzpbUYncf0eoeZxNBV6qZZ7XgNFmdkKwa2J0cFpcmdkY4FbgUufc/jrmacyxEM8aa37uc3kd227M73s8XQB85Jwrq+3FRO/DiCT60+DGfhE4K+RjAp/G3xGcNpPAQQ3QlsC//FuBtcD3m7G2cwj8C78BKA1+/RC4DrguOM904EMCZxCsAc5uxvq+H9zu+mANof1Xsz4DHg3u3w+AwmZ+f9sRCO7ja0xL6P4j8Efnc8Aj0E/8cwKf+6wAtgDLgU7BeQuBJ2sse3XwWNwK/KyZattKoG87dAyGzkI7CXilvmOhGfffM8HjawOB8O5+ZI3B50f9vjdHfcHp80LHXY15E7IPY/WlK2NFRFq4VOm6ERGRCCnoRURaOAW9iEgLp6AXEWnhFPQiIi2cgl5EpIVT0IuItHAKehGRFu7/A0nWPV5ziErKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11202b630>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def train_by_value_iteration(S, max_iterations, delta_eps=1e-6):\n",
    "    V, pi = {}, {}\n",
    "    for state in S:\n",
    "        V[state] = 0.0\n",
    "        pi[state] = None\n",
    "\n",
    "    def find_best_action(state):\n",
    "        best_action, best_v = None, None\n",
    "        for action in state.actions:\n",
    "            to_state = action.to_state\n",
    "            cur_v = action.reward + gamma * V[to_state]\n",
    "            if best_v is None or cur_v > best_v:\n",
    "                best_action, best_v = action, cur_v\n",
    "        return best_action, best_v\n",
    "\n",
    "    perf = []\n",
    "    for _ in range(max_iterations):\n",
    "        delta = 0.0\n",
    "        for state in S:\n",
    "            if state.is_terminal():\n",
    "                continue\n",
    "            old_v = V[state]\n",
    "            _, best_v = find_best_action(state)\n",
    "            V[state] = best_v\n",
    "            delta = max(delta, abs(best_v - old_v))\n",
    "        for state in S:\n",
    "            best_action, _ = find_best_action(state)\n",
    "            pi[state] = best_action\n",
    "        perf.append(measure_policy_performance(S, pi))\n",
    "        if delta < delta_eps:\n",
    "            break\n",
    "    return perf\n",
    "\n",
    "plot_method_results(S, train_by_value_iteration)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Util methods for working with stochastic policies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def choose_action_from_stochastic_policy(pi, state):\n",
    "    p = []\n",
    "    for aindex in range(len(state.actions)):\n",
    "        p.append(pi[(state, aindex)])\n",
    "    return np.random.choice(len(state.actions), 1, p=p)[0]\n",
    "\n",
    "\n",
    "def find_best_action_quality(Q, state):\n",
    "    best_aindex, best_q = None, None\n",
    "    for aindex in range(len(state.actions)):\n",
    "        cur_q = Q[(state, aindex)]\n",
    "        if best_q is None or best_q < cur_q:\n",
    "            best_aindex, best_q = aindex, cur_q\n",
    "    return best_aindex, best_q\n",
    "\n",
    "\n",
    "def update_pi_eps_greedy(pi, Q, state):\n",
    "    best_aindex, best_q = find_best_action_quality(Q, state)\n",
    "    best_count = 0\n",
    "    for aindex in range(len(state.actions)):\n",
    "        if Q[(state, aindex)] == best_q:\n",
    "            best_count += 1\n",
    "    for aindex in range(len(state.actions)):\n",
    "        if Q[(state, aindex)] == best_q:\n",
    "            pi[(state, aindex)] = ((1 - eps_greedy) / best_count +\n",
    "                                   eps_greedy / len(state.actions))\n",
    "        else:\n",
    "            pi[(state, aindex)] = eps_greedy / len(state.actions)\n",
    "\n",
    "\n",
    "def determine_policy(S, Q):\n",
    "    pi = {}\n",
    "    for state in S:\n",
    "        best_aindex, _ = find_best_action_quality(Q, state)\n",
    "        if best_aindex is None:\n",
    "            pi[state] = None\n",
    "        else:\n",
    "            pi[state] = state.actions[best_aindex]\n",
    "    return pi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "measuring with 10 seeds took 26.173680067062378\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XmYU/X59/H3nczGsG9FZECWgbLNDOAAg7ggiNpi1Spq0Sq0UFp/Wmn7q4qPPtX++tRq7eVa259YFWytWne7KULBXQsoVQQVxFH2fZ81yf38kZNMtlmTTGaS+3VdXJNzck7yPWfCZ765z/ecI6qKMcaY9OVKdQOMMcYklwW9McakOQt6Y4xJcxb0xhiT5izojTEmzVnQG2NMmrOgN8aYNGdBb4wxac6C3hhj0lxWqhsA0KtXLx04cGCqm2GMMe3KmjVr9qpq78aWaxNBP3DgQFavXp3qZhhjTLsiIl80ZTkr3RhjTJqzoDfGmDRnQW+MMWnOgt4YY9KcBb0xxqS5RoNeRB4Wkd0isi5kXg8ReUVENjo/uzvzRUTuFZFNIvKBiIxLZuONMcY0rik9+sXA2RHzFgLLVXUosNyZBvgaMNT5Nx/4fWKaaYwxpqUaHUevqq+JyMCI2ecBU5zHS4CVwPXO/EfVf3/Cd0Skm4j0VdUdiWpwW3X48Afs2/cq/frNIienV5PW8Xqr2Lp1CR5vRXCeiJvj+87E56th587nUWLf6rFj/mCOVWwGwO3qQIcOBXTt6v8CdfToJ/TqdTo7d/2Vnj1Oo6pqG3v2LqNDXgEVleV06VxEr17T2LnzWcBFrecgIm5qavbFtxMa4ZJscvOOo7JyS9j8/A4DqKragU9r61kvi7y8flRUNmnIMHm5ffF4DoXt19Yg4qbvcRewZ+8r1NYebPJ6HfL6U1OzG0Xpe9wF7Nz5Al5fZRJbatqS3r2m0aVLcVLfo6UnTPUJCe+dQB/ncT8g9H/xVmdeVNCLyHz8vX4GDBjQwma0HZs++zUHDrxNVnY3+hdc3qR1Dh1aw6bPfu1MifNTQRWP5zBbti4OmR8qdvjn5PRG1Utt7X7KJi7lo49+RK9eZyDiYs+epcHlcnP64HLlsH7DdTFeJdb7JUJkm6WR+Y2t19z3aQ3KsaOfsnvPP5vx3uHtPXpkQzPXN+1dbm6fNhv0QaqqItLsO4yr6iJgEUBpaWm7v0O5z1cNgPpi90pjr1MDQGnps3TtUgLAipUj8Poq8XoryMn5Cqec/HbUeu/+ewZHj35Mx45DKRp9P++8eyYANTV7gst4vccAqKraRm5un7D1vb5KPJ4jUa87auRdHHfcuU1uf3PU1h7ktddPBKCg4HK+OuwWAD7+5Ga2bfsTAKee8h7Z2V3D2+qtYOWrRQAc3/diRoz4VYPv88WXD7Jp020ATBj/Ip07j0rkZjRoxcrR1NTuB6C4eBG9e01rdJ2Nm27jyy8fDE4H1i+b+DIdOxYmp6Em47R01M0uEekL4Pzc7czfBvQPWa7AmZcB/L0vxdvkNVT9y4q4g/Ncrlx8vmp8vhpcrtyY6wXmu1y59S4T+CMCoD5PxHPVsV/XHfu1EiG0naGP3fXMDxDJqXu+Ce2r731ag8uVG/wD2tT3jlyuuesb0xQtDfoXgdnO49nACyHzr3BG35QBhzKhPg8gzq5UX0uCvu6LlT/oa/BpvEEfCHOJCvZ6gz6J4eJyhQR2WBjnxHxcNy8ruH+a0r5UBr27BUHvtqA3raDR0o2IPI7/wGsvEdkK3AzcBvxFROYCXwAXO4v/A/g6sAmoAL6ThDa3TeL06NXTyIJ1AsvG7tFXR4VA6DKBn/UFQmhpJlawx2pnMsMlchujH7sQid3vcLly8Ho9bT7oXa5caj2HgOgAb2idUB7P4ZjzjYlHU0bdzKrnqagCpDPa5qp4G9U+BYK++T16V1SPvhqftzpmDzewTOBn/UF/OPjYp3VlHJerAz5fJV5fVdQ6TQ2neMUKY5H6DzwG/ki0+aB35+Kpal5Qhy4nkmNBb5LCzoxNFPX5fySkR1+Dz1fdQOkmJ7hsfX8MakODPqRHn53VBQCv52iM101d0Dd3vfo0VvNPJv/vQkMeN2WdyDJW89Y3piks6BMkcBC2JT366Bq9v3RT38HHQDi4Xbn19oTDevTeuqDPyvYHfegfgsjXTbaWhnHze/StG5Yt+QMW+9tNTr1lLGNawj5NCaIJ7dFXN9Kjzw37GUtd0CvekB59VlbniOejXzfZWtqjb0ppKbwU0rof73iD3h38vVpv3iSWBX2iBIM+3uGVOfh81Xh91bikvhq9f740EAie2tgHYxsO+tYJGGlkpE0k/6GfpvboUxeSLfk2EbqchJTkjEkkC/oEaVnpJtCjr6d000iPvqEerscbGvR1B2OznBp9rBOm0q1009oS1aNvrYPiJnNY0CdIoNfZvNJNfSdM1fhPmGqkRt9gjz4Y5Ipq2wr6ZB6MbRs9ekEku2nruKP3RTJPXDOZyYI+UTQxPXp3E3r0geUbqkEHSjOhvXkIDfrUlW5iB33j13VpSvvaQo/e1cBB8vrWiVzfmESyoE8QpSUHY1t2CYSmRIin1h/kXm/4VRCzAzX62uigD21HMoWdDduM3mt7Kd209FuKBb1JFgv6BNEWHIz11TO80uutQhu4BEJTBIZPRpZoAj36WMMrW0tySzepC0l3C4I67HiF24LeJIcFfYIEAt7X7OGVElaC8Qf90eDjlgq8RuBnQCDoI+e3pvouatac9eJZJllcLRg1Yz160xos6BOlhcMrQ3vzEFHWSELNPDC8MpVi9+gbv1J1U/ZH5P5sTRIM6qa3Idbv28bRm0SzoE+QltXoPVF18aaUNYKRqM2/jH/bCProseNNW6/xnm5TD4Img7sFAR32+xYbR2+Sw4I+QbRFo25i9eiTewq/252f8NdsrmSWblKpJe2Ldb1969GbREvd99w04PPVsH79tc49Yv29a0/tQdZvWMiwoTeya9ffOHz4AyqrtgKKy5XLV4fdQocO/nuztLRHH0+ftS2ESFiPPhh0jW9VKssyTRF6yeWmrxN+ID78dYxJjLb9P6eNq6rawa7dfwP892sFOHT4fQ4dfp8OeQVs/vwuwB9mnToN48iRtzl06L2QoPdGBX237hPo3n0SgpuuXcbGfN/+/edQUVHOgAHzABgzZgk7dzxHdc1uBMHtzqem9gAVFZupdW5NN2L4rzh85CM6dDiB44//FseObaJDXj/yOhSQ32EQvhiXLU600tJn2b3r72EnE7lcOQzoP5ev9JlR73onjvsz23c80+Sy05Ah19Gp47C429tc3bpNoHv3k+jzlfq3JZaBJ1xJj56ngXo5fPhDeveanqQWmkwl2oI6b6KVlpbq6tWrU92MZquo+Jy33zkDgOzsHsFQBRgy+Kd8tvk3AHTIG8DYsUt46+3TGTniDvr2vQCADR/fyN69/4p5X9hE+HLLI2zc+P8AmDD+r3TuPDIp72OMSQ0RWaOqpY0tZzX6OATGzvsfh98UPOwkKHdusOwQWsOP1aNPpNCLorWFko0xJjUs6OMQGtqRlxoIPwkqJxjooaNy/DX65FXPUnm3JWNM22FBH4fAkEqIFfThlzWoC/pW7NHXc0NuY0xmsaCPR0jpJvKEn8jLGtSVblqxRx/jyojGmMxjQR+HhsbMR17WoL4evSupPXoLemOMBX1cQks3kZrWo48+YSqRUnn/VGNM22FBHw+tP+hDhR+MDe3RR58wlUiBs07tZtPGZDb73x+Hhko3oc+F9uh9rXow1k6pN8ZY0MdFG7jiYmjQi7idHrWkZHil1eeNyWwW9PFosEcfehVL/3VcRLJaeXil3WzaGGNBH5eGSje+iDNlwd+zT0mP3m42bUxGs6CPQ4Olm4gTqKC+Hn3yfgVWujHGgAV9fBro0Xt91VHzWr9H79zIQuxgrDGZzII+DhoxvDI0tH31Bn3rXwLBevTGZLa4gl5EfiwiH4nIOhF5XETyRGSQiLwrIptE5EmR9O1ORp4wFXqd9fCgV+f5rFY9Yco/2ifbgt6YDNfioBeRfsA1QKmqjgbcwLeA24G7VLUQOADMTURD26SI0o3LVV/Q+0X36JN7wpS/Tbl2MNaYDBdv6SYL6CD+bmk+sAOYCjztPL8EOD/O92izIm/aEnpiUuygj+jR+5Lbow+0yXr0xmS2FqeMqm4Tkd8AXwKVwFJgDXBQ69JsK9Av7lam2L9XnU/nziMZMfzWsPlKZI8+L/h4587ng49zsnsA/h79wQP/Zvm/hgSf69ZtfDKaHOR25+MOaZcxJvPEU7rpDpwHDAKOBzoCZzdj/fkislpEVu/Zs6elzWgVR458yPbtT0bNjxxH3/e4bzJw4FVh87p2HcegQT8E/D36qurtYc8n+/IEw4f/kgEnzE/qexhj2rZ4SjdnAJ+r6h7130fvWWAy0E3q6hEFwLZYK6vqIlUtVdXS3r17x9GMFIoo3eTl9WPI4J+EzSvo9+268ewx6vHJLqv07HEynToOTep7GGPatniC/kugTETyRUSAacB6YAUw01lmNvBCfE1su6LOjBWJWib0YGuserzVz40xydbioFfVd/EfdH0P+NB5rUXA9cBPRGQT0BN4KAHtbJMia/RCdI89NNxjjbCxoDfGJFtcQz5U9Wbg5ojZm4EJ8bxuu6GRtw+MFfTuBp+3oDfGJJudGRuHppVusmI+DrBrxRtjks2CPg7RZ8Zaj94Y0/ZY0Mcj8lo3MXanHYw1xqSaBX0cIks3sS45HFa6ccXq0VvpxhiTXBb0cYgs3dBo6SZGj96uQ2OMSTIL+ng0qXRjwyuNMallQR+HppVu6ubFGmdvQW+MSTYL+jhE3UowZumm4R693bjbGJNsFvTxiOzRY5dAMMa0PRb0cYgu3TS/Ry826sYYk2QW9HFoWumm4ROmrHRjjEk2C/o4RPXoY5ZuQsfRW+nGGNP6LOjjob5Ge+yhbHilMSYVLOhjqKnZR23tIbze8Pu+1tYeoqZmHwDV1bupqCwnbBfGGF4Z2uu3g7HGmFRI7p2p26Gqqh28+dbJAHTpXMz48c8Fn3vt9XEATJ78Jm++eTIEa/QuwEeWuyMA2dndqa09AIA7q2Nw/Sx3p6j3s0sgGGOSzYI+Qk1N3f1rDx/5AI245jxAddUOCDkQO6lsKTU1+8jPH+JML8fjOYyqh9ycXsHlBgz4Hl27jiU/fxAgqHobLfcYY0y8LOgjiGRHzIkOeq+vKmw6P3+QE95+2dldyc7uGrVeVlZHevY8LSHtNMaYprIafYTIkTEacT0bAK+3orWaY4wxcbOgjxB9YbIYQe851jqNMcaYBLCgjxB56eFYNXrr0Rtj2hML+khRpRor3Rhj2jcL+giRNflYNXqP10o3xpj2w4I+guKNmBOrRn+kdRpjjDEJYEEfKapHH12jr/Ucbq3WGGNM3CzoI0SXamKUbizojTHtiAV9hMjSTcwavQW9MaYdsaCPFFWqsaA3xrRvFvQRIq8xH6tG76m1g7HGmPbDgj5CZNDH6tHbwVhjTHtiQR8ltAfvshq9Mabds6CPEH2jkOjSTaxevjHGtFVxBb2IdBORp0XkYxHZICKTRKSHiLwiIhudn90T1djWEHqtGxF3zB69Mca0J/H26O8BXlLV4UAJsAFYCCxX1aHAcme6/dDwoK+v9243DDHGtBctDnoR6QqcCjwEoKo1qnoQOA9Y4iy2BDg/3ka2ln3732D7jqeD0yJZ9fbo7V6vxpj2Ip47TA0C9gCPiEgJsAZYAPRR1R3OMjuBPrFWFpH5wHyAAQMGxNGMxFm7dnbYtIgQu0bvvxNVp07D8fk8rdAyY4xpuXhKN1nAOOD3qjoWOEZEmUb9g9BjJqWqLlLVUlUt7d27dxzNSI7u3cpQ9QZ79Mf1OS/seZEsJk74O5PKXk5F84wxpsniCfqtwFZVfdeZfhp/8O8Skb4Azs/d8TWxdUSOn3e5cvxB79ToxRV+L1mX2O12jTHtQ4uDXlV3AltE5KvOrGnAeuBFIFADmQ28EFcLW4nPVxM2La4cVD3Bg7OuiJuG28FYY0x7EW+39IfAYyKSA2wGvoP/j8dfRGQu8AVwcZzv0Sp8vuqw6boevb/yFNmjF+vRG2PaibjSSlXXAqUxnpoWz+umQlTQS6BH73WmI4LeZT16Y0z7YGfGOiKDPtCDD9TuxZUT/rz16I0x7YQFvcMbo3QDdbV7q9EbY9orC3qHRhyMDQS7T2v901E1egt6Y0z7YEHvqLd046sNmw4+b6UbY0w7YUHvqLd0E+jRW+nGGNNOSaw7KLWmNWvWfGXdunW7JkyYkNJ2eL1V1NbuC05nZXXB4zlMdnYPamv3k5XVDY/nYPB5lyuHnJy2d0avMSb95OXlUVBQQHZ2ZIdT1qhqrJGPYVJef8jKyvpDcXExw4cPd64tkxq1tYeorKzbHbm5x1FdvZMOHfpTWZlNXl4/qqq2BZ93uzvSsePgVDTVGJNBVJV9+/axdetWBg0a1KLXaAulm9E5OTkpDXm/iG82TnvqvvFIxNOpbq8xJhOICD179qSqqqrFr9EWgt7VFkIz8nLEEgx258zYqDamvs3GmMwQb0a2haBvIyKPVUjEfAt2Y0z7ZEHviDwoLeLfNYry1ltrKCkp4+STL6KyMvD1Kf7gv/XWW8OmTzrppLhfE2Dx4sVs37692ev97//+L48++miL3u/qq69u9nqtoby8nD//+c8pee9U7Ze7776bioqK4HSnTp3ier3mrh/5uQ61Zs0aioqKKCws5Jprron6fxfpt7/9LYWFhYgIe/fuDc5XVa655hoKCwspLi7mvffea1YbG3PHHXcwZswYxowZw+jRo3G73ezfvx+Au+66i1GjRjF69GhmzZoVVVK55pprwvZZdXU1l1xyCYWFhUycOJHy8vKEtrUpLOiDwks3NTW1HDtWAar85S//4Prrf8IbbzxFhw55/gUSUG6K/A/x1ltvxf2a0HDQe73emPMBfvCDH3DFFVckpA1tRaqC3uNJ3Q1pIoO+tTUU9FdeeSUPPvggGzduZOPGjbz00ksNvtbkyZNZtmwZJ5xwQtj8f/7zn8HXWLRoEVdeeWVC2h5w7bXXsnbtWtauXcuvfvUrTjvtNHr06MG2bdu49957Wb16NevWrcPr9fLEE08E11u9ejUHDhwIe62HHnqI7t27s2nTJn784x9z/fXXA3Ds2DFqa2sT2u76tKmg//TTX7DmvUsT+u/TT3/RxHf39yw++WQzN974G0aPLmXTpi94+OE/8txzL3PLLb9k3ryFvP76Ki6++Opgf/7qq69m8eLFAAwcOJCbb76ZcePGUVRUxMcffwzA0aNH+c53vkNRURHFxcU888wzLFy4kMrKSsaMGcNll10G1PWcVJVrr72W0aNHU1RUxJNPPgnAypUrmTJlCjNnzmT48OFcdtllUT2ip59+mtWrV3PZZZcxZswYKisrGThwINdffz3jxo3jqaee4sEHH2T8+PGUlJRw4YUXBkPhlltu4Te/+Q0AU6ZM4frrr2fChAkMGzaM119/vcG9t2XLFqZMmcLQoUP5+c9/DsDPfvYz7r777uAyN954I/fcc0/M9VeuXMlpp53Geeedx+DBg1m4cCGPPfYYEyZMoKioiM8++wzwB/fUqVMpLi5m2rRpfPnllwDMmTOHa665hpNOOonBgwfz9NP+W0IuXLiQ119/nTFjxnDXXXfh9Xq59tprGT9+PMXFxTzwwAMNbtftt99OUVERJSUlLFzov69Offtvzpw5/OAHP2DixIlcd911Ya9TX7tjmTNnDldeeSVlZWUMHjyYlStX8t3vfpcRI0YwZ86c4HJLly5l0qRJjBs3josuuoijR49y7733sn37dk4//XROP/30sH1fUlJCWVkZu3btarBNn3/+OZMmTaKoqIibbrqp3naef/75nHjiiYwaNYpFixYF93fk5zpgx44dHD58mLKyMkSEK664gueffx6Px8P48eNZuXIlADfccAM33ngjAGPHjmXgwIFR7/3CCy9wxRVXICKUlZVx8OBBduzw39jujjvuCP5+b7755gbbC/DSSy8xbtw4SkpKmDYt+nqMjz/+OLNmzQpOezweKisr8Xg8VFRUcPzxxwMEP1u//vWvo9o6e7b/yu0zZ85k+fLlqCqffvopw4YN46c//SkbNmyodz8nhKqm9N/atWvLP/roI1VV/eST/9HVa2Yl9N8nn/yPNubo0aP6wAN3aVnZWC0rG6v33XeL7tu3VQ8d+kAPH9mgl156rj7xxB/10KEP9G9/e0jPOutUraj4UlVVr7rqKn3kkUdUVfWEE07Qe++9V1VV77//fp07d66qql533XW6YMGC4Pvt379fVVU7duwY1o7A9NNPP61nnHGGejwe3blzp/bv31+3b9+uK1as0C5duuiWLVvU6/VqWVmZvv7661Hbc9ppp+mqVauC0yeccILefvvtwem9e/cGH994443BNt988816xx13BF/jJz/5iaqq/v3vf9dp06bVu/8eeeQRPe6443Tv3r1aUVGho0aN0lWrVunnn3+uY8eOVVVVr9ergwcPDnvvUCtWrNCuXbvq9u3btaqqSo8//nj92c9+pqqqd999d3D/nXPOObp48WJVVX3ooYf0vPPOU1XV2bNn68yZM9Xr9epHH32kQ4YMCb7ujBkzgu/zwAMP6C9+8QtVVa2qqtITTzxRN2/eHLNN//jHP3TSpEl67NgxVVXdt29fg/tv9uzZOmPGDPV4PMH9ctVVVzXY7lhmz56tl1xyifp8Pn3++ee1c+fO+sEHH6jX69Vx48bp+++/r3v27NFTTjlFjx49qqqqt912m/785z9XVf/ve8+ePcHXA/TFF19UVdVrr702uP31tekb3/iGLlmyRFVVf/vb30Z9TgMC+yPwOw/sl/qWX7VqVdjn6LXXXgv+btatW6fDhw/XV155RceMGaPV1dVh60Zu04wZM8I++1OnTtVVq1bpyy+/rN/73vfU5/Op1+vVGTNm6Kuvvlpve3fv3q0FBQXBz0BgmYBjx45p9+7dw+bffffd2rFjR+3Vq5deeumlYfPvvPPOqH0watQo3bJlS3B68ODBwW05fPiwPvjgg3rSSSfp5MmT9eGHHw7+TiOtX78+ah6wWpuQsykfRx9q2LD/m5L37du3L6NHD+e+3/6cYUMHAtCpU2+OHt1fdwkEEbKyuuBy5TX4WhdccAEAJ554Is8++ywAy5YtC/t617179wZf44033mDWrFm43W769OnDaaedxqpVq+jSpQsTJkygoKAAgDFjxlBeXs7JJ5/c6DZecsklwcfr1q3jpptu4uDBgxw9epSzzjqr0W1prK44ffp0evbsGVzvjTfe4Ec/+hE9e/bk/fffZ9euXYwdOza4TCzjx4+nb9++AAwZMoQzzzwTgKKiIlasWAHA22+/Hdyvl19+eVjP+fzzz8flcjFy5MhgrzXS0qVL+eCDD4I9/kOHDrFx48aY45OXLVvGd77zHfLz8wHo0aMH0PD+u+iii3C7o8+abqjdsXzjG99ARCgqKqJPnz4UFRUBMGrUKMrLy9m6dSvr169n8uTJANTU1DBp0qSYr5WTk8M555wD+H+Xr7zySoNtevPNN3nmmWeC8wOlhkj33nsvzz33HOD/Rrdx48YGf78NGTVqFJdffjnnnHMOb7/9Njk5OY2vFMPSpUtZunQpY8eOBfzfpjdu3Mipp54as7179uzh1FNPDf7+A7/jgL/+9a9Mnjw5OP/AgQO88MILfP7553Tr1o2LLrqIP/3pT0ydOpWnnnoq+K2kqTp37sy8efOYN28eGzZsYO7cuSxYsIDDhw+3aPvr06aCPlWefvppFi36Ld++7EdceOHZzJp1LqNHF5Gd3Y3a2sDZsEJ+/gnk5n6Az1dXz488EJObmwuA2+1OSp028PrNfY+OHTsGH8+ZM4fnn3+ekpISFi9eXO+HsznbEjn8KzA9b948Fi9ezM6dO/nud7/b4GuEbpvL5QpOu1yuJm1n6Ppaz0E+VeW+++6r949bUzS0/0L3czxCtz1yv3g8HtxuN9OnT+fxxx9v9LWys7ODv4+mfmYaG863cuVKli1bxttvv01+fj5TpkxpdJx3v3792Lp1a3B669at9OvXLzj94Ycf0q1bN3bvbvzuo/369WPLli1Rr6Wq3HDDDXz/+9+Pu70ATzzxRFjZZtmyZQwaNIjAfa4vuOAC3nrrrWANvrCwEICKigoKCwvZtGlTsK0FBQV4PB4OHToU9gexvLycJUuW8Pjjj1NSUsItt9zSaLuaq03V6FPlzDPP5I9/vJ+XX/4TXbp04tJLF3DGGWfwxRfbopYdMKA/n3yymerqag4ePMjy5csbff3p06dz//33B6cDB2uys7NjHow55ZRTePLJJ/F6vezZs4fXXnuN5lwionPnzhw5cqTe548cOULfvn2pra3lsccea/LrNuSVV15h//79VFZW8vzzzwd7mt/85jd56aWXWLVqVVzhGnDSSScFvx099thjnHLKKQ0uH7kvzjrrLH7/+98H9/unn37KsWPHYq47ffp0HnnkkWANPjDqoiX7r7ntbkxZWRlvvvkmmzZtAvwH9j799FOg8d9/Y22aPHly2PxYDh06RPfu3cnPz+fjjz/mnXfeCT5X3+e6b9++dOnShXfeeQdV5dFHH+W8884D4Nlnn2X//v289tpr/PCHP+TgwYNR64c699xzefTRR1FV3nnnHbp27Urfvn0566yzePjhhzl69CgA27ZtY/fu3fW2t6ysjNdee43PP/8cqPsdB7bx1VdfDbYRYMCAAbzzzjtUVFSgqixfvpwRI0YwY8YMdu7cSXl5OeXl5eTn5wd/N+eeey5LliwB/J3KqVOnIiKUl5dzxhlncP7559OtWzfefPNNnnzyyeA32USyoHcoSs+ePbnyym/zxhtPceutt8b8Ct5/QAHnn38mpaXTufjii4NfERty0003ceDAAUaPHk1JSUmwDDF//nyKi4ujDlp985vfpLi4mJKSEqZOncqvf/1rjjvuuCZvS+CgYOBgbKRf/OIXTJw4kcmTJzN8+PAmv25DJkyYwIUXXkjB8WNuAAAPi0lEQVRxcTEXXnghpaX+y2/k5ORw+umnc/HFF8fcn81133338cgjj1BcXMwf//jHeg/uBhQXF+N2uykpKeGuu+5i3rx5jBw5knHjxjF69Gi+//3v19vDPfvsszn33HMpLS1lzJgxwQPVLdl/zW13Y3r37s3ixYuZNWsWxcXFTJo0KXjwf/78+Zx99tlhB2Ob06Z77rmH+++/n6KiIrZti+7sgH/feDweRowYwcKFCykrKws+V9/nGuB3v/sd8+bNo7CwkCFDhvC1r32NvXv3snDhQv7whz8wbNgwrr76ahYsWAD4y0MFBQVs3bqV4uJi5s2bB8DXv/51Bg8eTGFhId/73vf43e9+B/g7bZdeemnwYPLMmTM5cuRIve3t3bs3ixYt4oILLqCkpCSsxPncc89x5plnhn1LmzhxIjNnzgwOuPD5fMyfP7/B/Tx37lz27dtHYWEhd955J7fddhvg/3Z16623snbtWhYsWNDisldTpPyiZv/5z3/Ks7OzTxg5cmRK21FR8QU+rcHn9X+d69KliKqqHdTU+Mfuduw4FLc7j5qafVRVbSc7uxsdOvRPZZPbBZ/PFxztM3To0FQ3x5h2a8OGDYwYMSJsXlMvamY9+iBfyGUP/AInTZmWWb9+PYWFhUybNs1C3pgUsoOxDv83m8hglxiPM/dSCC+//HLUCIxBgwYFRzJEGjlyJJs3bw6b9+GHH3L55ZeHzcvNzeXdd99NbGObIVVt+uUvf8lTTz0VNu+iiy4KjiE3JlGsdOM4dmwTIm48Hv9BnC5diqiu3kt1tf8kjI4dh+F251JTc4Cqqq1WujHGtKr2XrrxpfqPDfgPxkbujrZwVU1jjIk3I9tC0K+rqamJe0Piphrj+jV10xb6xphUUOfGI3l5DZ+s2ZCU1+g9Hs+8devW7YpnIxKhunonLlcuXq9/zHReXhZebwW1tf4x77m5/huCB+a53UfIzj6ayiYbYzJE4FaCLZXyGj1AaWmprl69OqVteP2NMnr1msb27f4TRaZN/Yzdu1/mw3X/BcDkk14nL+94dux4lvUbruW4PucxatSdqWyyMSbDtacafZvg81XjcuWGzXO5Qq63YUMtjTHtlKWXw+erxh0V9HXTYrvKGNNOWXrhP9gRs0fvDpkO9uhtPL0xpn2xoAdUawBilG5Ce/QW7MaY9inuoBcRt4i8LyJ/c6YHici7IrJJRJ4UkZZdWLoV+XyBoA9valjQW43eGNNOJSK9FgCh98G6HbhLVQuBA8DcBLxHUnl91UB0jz68Zm9Bb4xpn+JKLxEpAGYAf3CmBZgKPO0ssgQ4P573aA0+b+ygtx69MSYdxHvC1N3AdUBnZ7oncFBVAxf43gr0i7Via9u//02+3PIIJcWLEHHx8Sc/Y9++V6mtPUD//nOAhoM+cPDV5faf2OXO6tQazTbGmLi1OOhF5Bxgt6quEZEpLVh/PjAf/HdtSbZDh95j374V+HxVuN35bNtWd+ec8nL/3Z9crlzGjf1zsGafldWVIYP/G3CR5QT7V3qfyZAh11HQ79tJb7MxxiRCPD36ycC5IvJ1IA/oAtwDdBORLKdXXwDEvEWNqi4CFoH/zNg42tEkql7AGS/vzo+5jMuVQ/fuE4PTIsLAgf8VtoyIm4EnfD9yVWOMabNaXHhW1RtUtUBVBwLfAv6lqpcBK4CZzmKzgRfibmUCBKpJgQOvsUSWbowxJh0k4wjj9cBPRGQT/pr9Q0l4j2YL9ui9DQS924LeGJN+EnL1SlVdCax0Hm8GJiTidRMpGPTOyVGxWI/eGJOOMmbMoM8p3fh81fVe+96C3hiTjjIm6EMPxqrWxlwm8qJmxhiTDjIo6J0evbcaXz0HZMXV5q/WYIwxzZZBQV/Xo68v6K1Hb4xJRxkU9IEafU3whKhIVqM3xqSjDAr6xnv07eBCm8YY02wZFPR1o25inTQlkoXLlfJ7pRtjTMJlUNA33KO3so0xJl1lYNDXxDw71oLeGJOuMjDo6+vRW33eGJOeMijo62r0FvTGmEySQUHfcI9exN3aTTLGmFaRcUHv9VXXO47eGGPSUVoHfU3NfjyeI0DoJRAq8XqPRS1b34XOjDGmvUvbgeM1Nft4/Y0JuFx5nD7lo2CPfvuOv8COwFIuwAdATk6PlLTTGGOSLa2DHsDnq0JVqbtfeZ2TJq2g1nOQyoov6NKlqLWbaIwxrSJtgz70gKvPVxPs0YfKzT2ODh0K6NJ5dGs2zRhjWlXa1ujDg746qkdvlzwwxmSKNA76upE1Po3u0duZsMaYTJHGQR/So/dG9+jtBCljTKZI46AP6dH7qmPU6KV1G2SMMSmSxkEfWaOPPhhrjDGZIIOCPnJ4pZ0gZYzJDBkU9NajN8ZkpgwK+ugTpowxJhNkUNBbj94Yk5nSNui9FvTGGAOkcdA3fgkEG15pjMkMGRH0Xl+V1eiNMRkrjYO+BhH/2a8+b1WKW2OMMamTxkFfTVZWZ4CQG42k7eYaY0y9Wpx8ItJfRFaIyHoR+UhEFjjze4jIKyKy0fnZPXHNbbrwoK9w2mxXqzTGZJ54urge4L9VdSRQBlwlIiOBhcByVR0KLHemW53PV4PbnY9IFp5g0Idurp0Za4zJDC0OelXdoarvOY+PABuAfsB5wBJnsSXA+fE2siV8vmpcrlxcrtxg6cZ69MaYTJSQorWIDATGAu8CfVQ1cFfWnUCfetaZLyKrRWT1nj17EtGMMP6gzwkL+p49Tw0+f/zx30r4expjTFsUdxdXRDoBzwA/UtXDInXj01VVRSRmjURVFwGLAEpLSxNeR/HX6LvgcuXgqT0MQM+epzFq5G+cdtv16I0xmSGuHr2IZOMP+cdU9Vln9i4R6es83xfYHV8TWya0dOPxHgEITrtcuYT+QTLGmHQWz6gbAR4CNqjqnSFPvQjMdh7PBl5oefNaLizoPf6gd9vtA40xGSie0s1k4HLgQxFZ68z7P8BtwF9EZC7wBXBxfE1sGZ+3GrcT9DU1/mMAdp9YY0wmanHQq+ob1H/BmGktfd1E8Yb16P01egt6Y0wmSttTRVVrcLlycbtygxc0s6A3xmSitA36QI1eXHWja1wuG2ljjMk8aRn0qorPVxMs3QRYj94Yk4nSMuh9vhqAYOkmwILeGJOJ0jTo/deid7mtR2+MMekd9Fa6McaYNA96yQk7AGtBb4zJROkd9M5FzQJs1I0xJhOld9CH1ehdEdejN8aYzJCWyRdWo3dbucYYk9nSP+iDlyO2O0oZYzJTmga9fxy9O2LUjTHGZKI0DfrYwyuNMSYTpdVNVFW9fPHFInJyegJWozfGGEizoN+1+x98tvk3ZGV1BvxB36njV8nL60+vXqenuHXGGJMaaRX0Pq+/ZBO4o5TLlUuHDv2ZfNLKFLbKGGNSKy1r9AFWnzfGmLQPejsT1hhjLOiNMSbNpVnQ150UJZKNiDuFbTHGmLYhrYJe8QUfW33eGGP80iroA2fEgpVtjDEmIM2Cvjr42Hr0xhjjl1ZBr2E9egt6Y4yBNAt6b0iP3m1Bb4wxQJoFvZVujDEmmgW9McakOQt6Y4xJc2kc9Da80hhjIO2C3kbdGGNMpKQEvYicLSKfiMgmEVmYjPeIxUo3xhgTLeFBL/4LzNwPfA0YCcwSkZGJfp9Y7MxYY4yJlowe/QRgk6puVtUa4AngvCS8T5SwHr3dQtAYY4Dk3GGqH7AlZHorMDEJ78P27U/x5ZaHgtOVlV8GH7vEevTGGAMpvJWgiMwH5gMMGDCgRa+Rnd2NjvmFwemO+YXkdSigumoHffqck5B2GmNMe5eMoN8G9A+ZLnDmhVHVRcAigNLSUo18vil6955O797TW7KqMcZkjGTU6FcBQ0VkkIjkAN8CXkzC+xhjjGmChPfoVdUjIlcDLwNu4GFV/SjR72OMMaZpklKjV9V/AP9IxmsbY4xpnrQ6M9YYY0w0C3pjjElzFvTGGJPmLOiNMSbNWdAbY0yaE9UWnauU2EaI7AG+aOHqvYC9CWxOe2DbnBlsmzNDPNt8gqr2bmyhNhH08RCR1apamup2tCbb5sxg25wZWmObrXRjjDFpzoLeGGPSXDoE/aJUNyAFbJszg21zZkj6Nrf7Gr0xxpiGpUOP3hhjTAPaddCn6ibkySYiD4vIbhFZFzKvh4i8IiIbnZ/dnfkiIvc6++ADERmXupa3nIj0F5EVIrJeRD4SkQXO/LTdbhHJE5F/i8h/nG3+uTN/kIi862zbk87lvhGRXGd6k/P8wFS2v6VExC0i74vI35zptN5eABEpF5EPRWStiKx25rXaZ7vdBn0qb0LeChYDZ0fMWwgsV9WhwHJnGvzbP9T5Nx/4fSu1MdE8wH+r6kigDLjK+X2m83ZXA1NVtQQYA5wtImXA7cBdqloIHADmOsvPBQ448+9ylmuPFgAbQqbTfXsDTlfVMSFDKVvvs62q7fIfMAl4OWT6BuCGVLcrgds3EFgXMv0J0Nd53Bf4xHn8ADAr1nLt+R/wAjA9U7YbyAfew39/5b1AljM/+DnHf4+HSc7jLGc5SXXbm7mdBU6oTQX+Bkg6b2/IdpcDvSLmtdpnu9326Il9E/J+KWpLa+ijqjucxzuBPs7jtNsPzlf0scC7pPl2O2WMtcBu4BXgM+CgqnqcRUK3K7jNzvOHgJ6t2+K43Q1cB/ic6Z6k9/YGKLBURNY498uGVvxsp+zm4KblVFVFJC2HS4lIJ+AZ4EeqelhEgs+l43arqhcYIyLdgOeA4SluUtKIyDnAblVdIyJTUt2eVnayqm4Tka8Ar4jIx6FPJvuz3Z579E26CXka2SUifQGcn7ud+WmzH0QkG3/IP6aqzzqz0367AVT1ILACf+mim4gEOmGh2xXcZuf5rsC+Vm5qPCYD54pIOfAE/vLNPaTv9gap6jbn5278f9An0Iqf7fYc9Jl2E/IXgdnO49n4a9iB+Vc4R+rLgEMhXwfbDfF33R8CNqjqnSFPpe12i0hvpyePiHTAf0xiA/7An+ksFrnNgX0xE/iXOkXc9kBVb1DVAlUdiP//679U9TLSdHsDRKSjiHQOPAbOBNbRmp/tVB+kiPMAx9eBT/HXNW9MdXsSuF2PAzuAWvz1ubn4a5PLgY3AMqCHs6zgH330GfAhUJrq9rdwm0/GX8f8AFjr/Pt6Om83UAy872zzOuBnzvzBwL+BTcBTQK4zP8+Z3uQ8PzjV2xDHtk8B/pYJ2+ts33+cfx8Fsqo1P9t2ZqwxxqS59ly6McYY0wQW9MYYk+Ys6I0xJs1Z0BtjTJqzoDfGmDRnQW+MMWnOgt4YY9KcBb0xxqS5/w+LvmUprQMyywAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1122ad7b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def train_by_monte_carlo_method(S, max_iterations):\n",
    "    Q, cumulative, total, pi = {}, {}, {}, {}\n",
    "    for state in S:\n",
    "        for aindex in range(len(state.actions)):\n",
    "            Q[(state, aindex)] = 0.0\n",
    "            cumulative[(state, aindex)] = 0\n",
    "            total[(state, aindex)] = 0\n",
    "            pi[(state, aindex)] = 1.0 / len(state.actions)\n",
    "\n",
    "    def run_single_episode():\n",
    "        episode_log = []\n",
    "        state = S[S.index(State(0, 0, 0))]\n",
    "        while True:\n",
    "            choice_index = choose_action_from_stochastic_policy(pi, state)\n",
    "            episode_log.append((state, choice_index))\n",
    "            action = state.actions[choice_index]\n",
    "            state = action.to_state\n",
    "            if state.is_terminal():\n",
    "                return episode_log, action.reward\n",
    "\n",
    "    perf = []\n",
    "    for _ in range(max_iterations):\n",
    "        episode_log, reward = run_single_episode()\n",
    "        seen_pairs = set()\n",
    "        seen_states = set()\n",
    "        for ind, pair in enumerate(episode_log):\n",
    "            if pair in seen_pairs:\n",
    "                continue\n",
    "            seen_pairs.add(pair)\n",
    "            cumulative[pair] += reward * np.power(gamma,\n",
    "                                                 len(episode_log) - ind - 1)\n",
    "            total[pair] += 1\n",
    "            Q[pair] = cumulative[pair] / total[pair]\n",
    "            seen_states.add(pair[0])\n",
    "        for state in seen_states:\n",
    "            update_pi_eps_greedy(pi, Q, state)\n",
    "        perf.append(measure_policy_performance(S, determine_policy(S, Q)))\n",
    "    return perf\n",
    "\n",
    "\n",
    "plot_method_results(S, train_by_monte_carlo_method)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "measuring with 10 seeds took 31.567353010177612\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xt0VOW9xvHvLwkBUa4REBIwWJBgyA0DRRFEEeEURBBKRVRAEe1CxZ4uRURFq7Vo8Vi7qK0IFOlCuYlNW0VRhAK1BQGDIBRBwQIiBAooyC2Z9/wxkzFAuGVm9kwmz2etrMzesy/vOwxPfvPuvWebcw4REYlfCdFugIiIRJaCXkQkzinoRUTinIJeRCTOKehFROKcgl5EJM4p6EVE4pyCXkQkzinoRUTiXFK0GwBw4YUXuvT09Gg3Q0SkUlm5cuVu51yDMy0XE0Gfnp7OihUrot0MEZFKxcy+PJvlNHQjIhLnFPQiInFOQS8iEucU9CIicU5BLyIS584Y9GY2xcx2mdnaMvPqm9l7ZrYx8LteYL6Z2W/NbJOZfWJmbSPZeBERObOzqeinAj1OmPcwsMA51xJYEJgG+B+gZeBnOPD78DRTREQq6ozn0TvnFptZ+gmzbwS6BB6/CiwCRgXmT3P++xP+y8zqmllj59yOcDW4Kti7dxn/3fthtJshIh5ocGFXatfOjug+KnrBVKMy4f010CjwOBXYWma5bYF5JwW9mQ3HX/XTrFmzCjYjPm36/Fm++WY1YNFuiohEWPXqjWI26IOcc87MzvkO4865icBEgPz8fN2hvIySkkM0aHA92Vka+RKR0FX0rJudZtYYIPB7V2D+dqBpmeXSAvPkHPh8R0hIqB7tZohInKho0P8FGBx4PBgoKDP/9sDZNx2A/RqfP3cKehEJpzMO3ZjZ6/gPvF5oZtuAscA4YJaZ3Ql8CQwILP428CNgE/AdMDQCbY57Pt9RBb2IhM3ZnHUz8BRPdS1nWQeMCLVRVZ2CXkTCSVfGxiAN3YhIOCnoY4xzJTh3TEEvImGjoI8xPt9RABITkqPcEhGJFwr6GOPzHQFQRS8iYaOgjzGlQW+q6EUkTBT0MaY06BNV0YtImCjoY0yJhm5EJMwU9DFGY/QiEm4K+hijoBeRcFPQx5jS0ysV9CISLgr6GKOKXkTCTUEfYxT0IhJuCvoY4yspDXqdRy8i4aGgjzEaoxeRcFPQxxgN3YhIuCnoY4yCXkTCTUEfYxT0IhJuCvoY833Q62CsiISHgj7G+O8ulYyZRbspIhInFPQxpkS3ERSRMFPQxxinG4OLSJgp6GOMz3eEBNP4vIiEj4I+xpT4jpCQqIpeRMJHQR9jfBqjF5EwU9DHGAW9iISbgj7G+HQwVkTCTEEfY1TRi0i4KehjjIJeRMJNQR9jSq+MFREJFwV9jFFFLyLhFlLQm9nPzOxTM1trZq+bWQ0za25my8xsk5nNNNPVP+dCFb2IhFuFg97MUoH7gXznXBsgEbgZeBZ4wTnXAtgL3BmOhlYVOutGRMIt1KGbJOA8M0sCagI7gGuBOYHnXwX6hLiPKkVDNyISbkkVXdE5t93MxgP/AQ4B84GVwD7nXHFgsW1AasitjGObN09g85bfBaedO0piQo0otkhE4k2Fg97M6gE3As2BfcBsoMc5rD8cGA7QrFmzijaj0vv227UkJdWiSZMfA2Ak0Lhx/yi3SkTiSYWDHrgO2OycKwIws7lAR6CumSUFqvo0YHt5KzvnJgITAfLz810I7ajUfL4jnHdeGi1+8GC0myIicSqUMfr/AB3MrKb5b4fUFVgHLARKS9LBQEFoTYxvutGIiERahYPeObcM/0HXVcCawLYmAqOA/zWzTUAKMDkM7YxbOstGRCItlKEbnHNjgbEnzP4CaB/KdqsS/1k2KdFuhojEMV0ZG2W6QEpEIk1BH2UKehGJNAV9lOkCKRGJNAV9lCnoRSTSFPRRprNuRCTSFPRR5JxTRS8iEaegjyLnjgGORAW9iESQgj6KfL4jAKroRSSiFPRRpKAXES8o6KPI5zsKoPPoRSSiFPRRpIpeRLygoI+iEgW9iHhAQR9FquhFxAsK+ig6cvhrQGP0IhJZCvooWv/vRwBIqlYnyi0RkXimoI8iM+OCCzKodUFmtJsiInFMQR9FPt8x6tW7Ev+dGEVEIkNBH0X6QjMR8YKCPkqc8+HcUR2IFZGIU9BHyfdXxaqiF5HIUtBHyffn0KuiF5HIUtBHiS6WEhGvKOijpDTo9V30IhJpCvooUUUvIl5R0EeJgl5EvKKgjxIFvYh4RUEfJfqKYhHxioI+SpzuLiUiHlHQR4mGbkTEKwr6KNHQjYh4RUEfJaroRcQrIQW9mdU1szlm9m8zW29mV5hZfTN7z8w2Bn7XC1dj44m+60ZEvBJqRf8i8I5zLgPIAdYDDwMLnHMtgQWBaTmBKnoR8UqFg97M6gCdgckAzrmjzrl9wI3Aq4HFXgX6hNrIeHPgwAb2/vdDQEEvIpEXSkXfHCgC/mhmH5vZJDM7H2jknNsRWOZroFF5K5vZcDNbYWYrioqKQmhG5bPp81+ze88HVK9+EQkJ1aLdHBGJc6EEfRLQFvi9cy4POMgJwzTOOQe48lZ2zk10zuU75/IbNGgQQjMqn5KS76hT53KuvGIhZjoeLiKRFUrKbAO2OeeWBabn4A/+nWbWGCDwe1doTYw/Pt8REhNr6mIpEfFEhYPeOfc1sNXMWgVmdQXWAX8BBgfmDQYKQmphHPL5jmhsXkQ8kxTi+vcB080sGfgCGIr/j8csM7sT+BIYEOI+4o6CXkS8FFLQO+cKgfxynuoaynbjnT/oNWwjIt7QkcAoUEUvIl5S0EeBz3dUQS8inlHQR4HPd0T3ihURzyjoPeac09CNiHhKQe8x5/RlZiLiLQW9x/StlSLiNQW9x3TDERHxmoLeY76S0qDXefQi4g0Fvcf0PfQi4jUFvccU9CLiNQW9x3zBs240dCMi3lDQe+z7MXpV9CLiDQW9x4JDN4kKehHxhoLeYxqjFxGvKeg95nPHAEgw3StWRLyhoPeYc8UAmIV6zxcRkbOjoPeYcyUAmCVGuSUiUlUo6D2mil5EvKag95gqehHxmoLeY98HvSp6EfGGgt5j3w/dqKIXEW8o6D2mil5EvKag95gqehHxmoLeY86nil5EvKWg95gqehHxmoLeYzq9UkS8pqD3mL+iT8DMot0UEakiFPQec65E4/Mi4ikFvcccJRq2ERFPKeg95q/oFfQi4p2Qg97MEs3sYzP7W2C6uZktM7NNZjbTzHRz1DKcK9bQjYh4KhwV/UhgfZnpZ4EXnHMtgL3AnWHYR9xQRS8iXgsp6M0sDegJTApMG3AtMCewyKtAn1D2EW+cr5gEVfQi4qFQK/rfAA8BvsB0CrDPlV4VBNuA1BD3EVdU0YuI1yoc9GbWC9jlnFtZwfWHm9kKM1tRVFRU0WZUOjq9UkS8FkpF3xHobWZbgBn4h2xeBOra90mWBmwvb2Xn3ETnXL5zLr9BgwYhNKNyca4YS1BFLyLeqXDQO+dGO+fSnHPpwM3AB865QcBCoH9gscFAQcitjCOq6EXEa5E4j34U8L9mtgn/mP3kCOyj0vKfXqmKXkS8E5bS0jm3CFgUePwF0D4c241HOhgrIl7TlbEe0wVTIuI1Bb3HVNGLiNcU9B7zqaIXEY8p6D2mil5EvKag95hOrxQRrynoPeY/GKuXXUS8o8TxmvOpohcRTynoPaYLpkTEawp6j/k0Ri8iHlPQe0wVvYh4TUHvMZ1eKSJeU9B7TF+BICJeU9B7TBW9iHhNQe+ho0f34PMdVUUvIp5S4njkyNHd/OMfV+JcCYmJ50W7OSJShaii98jRo7txroSmTYdycbPh0W6OiFQhCnqP+HxHAKhfryPJySlRbo2IVCUKeo/4SvxBn5CQHOWWiEhVo6D3SGlFn5BQPcotEZGqRkHvEZ87CijoRcR7CnqPqKIXkWhR0Hvk+zF6Bb2IeEtB75FgRZ+ooBcRbynoPVIa9Imq6EXEYwp6j2iMXkSiRUHvkdKgN9N59CLiLQW9R3y+I5glkZCgrxcSEW8p6D3i8x3VsI2IRIWC3iMlviMKehGJCgW9R3y+I/qeGxGJCgW9RxT0IhItFQ56M2tqZgvNbJ2ZfWpmIwPz65vZe2a2MfC7XviaW3n5NHQjIlESSkVfDPzcOXcZ0AEYYWaXAQ8DC5xzLYEFgekqT0EvItFS4XP9nHM7gB2Bx9+a2XogFbgR6BJY7FVgETAqpFZWQnv3LuPw4e3B6cOHt5OUVDuKLRKRqiosJ3WbWTqQBywDGgX+CAB8DTQ6xTrDgeEAzZo1C0czYobPd4SPC2/DuZLj5jds+KMotUhEqrKQg97MLgDeAB5wzn1jZsHnnHPOzFx56znnJgITAfLz88tdprLy+Y7hXAkXX/xTUpsMCM6vXv2iKLZKRKqqkILezKrhD/npzrm5gdk7zayxc26HmTUGdoXayMqmtJJPTk7hvPPi69OKiFQ+oZx1Y8BkYL1z7v/KPPUXYHDg8WCgoOLNq5ycKwbATF93ICLRF0oSdQRuA9aYWWFg3iPAOGCWmd0JfAkMOMX6cau0ojdLjHJLRERCO+tmKWCneLprRbcbD0or+gRV9CISA3RlbASooheRWKKgjwCN0YtILFHQR4AqehGJJQr6CAheKKWgF5EYoKCPgNKgT1DQi0gMUNBHgMboRSSWKOgjQGP0IhJLFPQR8H3Qq6IXkehT0EfA90M3quhFJPoU9BGgil5EYomCPgJU0YtILFHQR4AOxopILFHQR4BOrxSRWKKgjwCfKnoRiSEK+ghQRS8isURBHwEaoxeRWKKgjwBV9CISSxT0EaCKXkRiiYI+AnTBlIjEEgV9BOiCKRGJJSo5IyDSFf2xY8fYtm0bhw8fjsj2RSS21KhRg7S0NKpVq1ah9RX0EfB9RR+ZD0zbtm2jVq1apKenY2YR2YeIxAbnHHv27GHbtm00b968QtvQ0E0EOOcDIlfRHz58mJSUFIW8SBVgZqSkpIT0CV5BHwFejNEr5EWqjlD/vyvoI0CnV4pILFHQR4C/ok+I2Bh9ZbBkyRIyMzPJzc3l0KFDYdvuM888c9z0lVdeGZbtTp06la+++uqc1/vDH/7AtGnTKrS/e++995zXi2X79u3jpZdeOuXz77zzDq1ataJFixaMGzfujNsbM2YMTZs25YILLjhu/uLFi2nbti1JSUnMmTPnuOd69OhB3bp16dWr1zm1/fnnn8fM2L17d7nPp6enB59LT08nKyuL3Nxc8vPzg8usXr2aK664gqysLG644Qa++eYbwH/yxODBg8nKyqJ169b86le/Oqe2hUPVTaIIcq6kSp5Df/ToUQ4ePAjA9OnTGT16NIWFhZx33nlh28eJQf/hhx+GZbunC/qSkpJTrnfPPfdw++23h6UNXjpdnyrqdEFfUlLCiBEjmDdvHuvWreP1119n3bp1p93eDTfcwPLly0+a36xZM6ZOncott9xy0nMPPvggf/rTn86p3Vu3bmX+/Pk0a9bsrNdZuHAhhYWFrFixIjhv2LBhjBs3jjVr1tC3b19+/etfAzB79myOHDnCmjVrWLlyJS+//DJbtmwBYO/evefU1opS0EeAc8WeDdt89tlTrFx1S1h/PvvsqXNqw/r16/n5z39Oq1at+Oyzz5g0aRKzZs3iscceY9CgQSxatOi4Cuvee+9l6tSpgL86Gjt2LG3btiUrK4t///vfABw4cIChQ4eSlZVFdnY2b7zxBg8//DCHDh0iNzeXQYMGAQSrPeccDz74IG3atCErK4uZM2cCsGjRIrp06UL//v3JyMhg0KBBOOeOa/+cOXNYsWIFgwYNCn4CSU9PZ9SoUbRt25bZs2fzyiuv0K5dO3JycujXrx/fffcdAE888QTjx48HoEuXLowaNYr27dtz6aWXsmTJktO+blu3bqVLly60bNmSJ598EoDHH3+c3/zmN8FlxowZw4svvlju+jt27KBz587k5ubSpk2b4P5++tOfkp+fT2ZmJmPHjg0uf2Kffvvb33LZZZeRnZ3NzTffDMDy5cu54ooryMvL48orr2TDhg0n7ffAgQN07do1+G9WUFAAwMMPP8znn39Obm4uDz744HHrLF++nBYtWnDJJZeQnJzMzTffTEFBAfv376dVq1bB/QwcOJBXXnkFgA4dOtC4ceOT9p+enk52djYJCSfHV9euXalVq9ZJ81euXMnVV1/N5ZdfTvfu3dmxY0fwuZ/97Gc899xzx42D79mzh+uvv57MzEyGDRt20numPJ999hmdO3cGoFu3brzxxhuAf3z94MGDFBcXc+jQIZKTk6lduzYA+fn5DBo0iA8++OCs9lFRVa/s9IC/oo/v8fmDBw8ya9YsJk+eDMDQoUN54oknqFWrFnl5eSxdupRevXrRv39/Fi1adNptXXjhhaxatYqXXnqJ8ePHM2nSJJ566inq1KnDmjVrAH/l069fPyZMmEBhYeFJ25g7dy6FhYWsXr2a3bt3065du+B/uo8//phPP/2UJk2a0LFjR/7xj39w1VVXBdft378/EyZMYPz48cd9FE9JSWHVqlWA/z/+XXfdBcCjjz7K5MmTue+++05qR3FxMcuXL+ftt9/mySef5P333z9lv5cvX87atWupWbMm7dq1o2fPntxxxx3cdNNNPPDAA/h8PmbMmFFuVQvw2muv0b17d8aMGUNJSUnwj88vf/lL6tevT0lJCV27duWTTz4hOzv7pD41adKEzZs3U716dfbt2wdARkYGS5YsISkpiffff59HHnkkGFilatSowZtvvknt2rXZvXs3HTp0oHfv3owbN461a9eW+++zfft2mjZtGpxOS0tj2bJl1KlThwkTJjBkyBBGjhzJ3r17g69zuBw7doz77ruPgoICGjRowMyZMxkzZgxTpkyhoKCA1NRUcnJyjlvnySef5KqrruLxxx/nrbfeCr7PwR/c119/PWbG3XffzfDhwwHIzMykoKCAPn36MHv2bLZu3Qr4318FBQU0btyY7777jhdeeIH69esD/j8O8+bNY8KECYwYMYLbbruNIUOG0KRJk7C+Bgr6CPBX9N68tJde+pgn+zlR48aNyc7OZtKkSWRkZIS0rZtuugmAyy+/nLlz5wLw/vvvM2PGjOAy9erVO+02li5dysCBA0lMTKRRo0ZcffXVfPTRR9SuXZv27duTlpYGQG5uLlu2bDku6E/lJz/5SfDx2rVrefTRR9m3bx8HDhyge/fuZ+xL6cfzU+nWrRspKSnB9ZYuXcoDDzxASkoKH3/8MTt37iQvLy+4zInatWvHHXfcwbFjx+jTpw+5ubkAzJo1i4kTJ1JcXMyOHTtYt25dMOjL9ik7O5tBgwbRp08f+vTpA8D+/fsZPHgwGzduxMw4duzYSft1zvHII4+wePFiEhIS2L59Ozt37jxtX8/0OsyePZsRI0awevXqCm/nVDZs2MDatWvp1q0b4B9GKg3dZ555hvnz55+0zuLFi4PvxZ49ex73/lu6dCmpqans2rWLbt26kZGRQefOnZkyZQr3338/Tz31FL179yY5ORnw/0FPTEzkq6++Yu/evXTq1InrrruOSy65hMTERHr16kWvXr0oKipi9OjRNGvWjA8//JD27duH7TWIyNCNmfUwsw1mtsnMHo7EPmJZVajo58yZQ2pqKjfddBO/+MUv+PLLL0+5bFJSEj6fLzh94vnA1atXByAxMZHi4uKwt7V0++e6j/PPPz/4eMiQIUyYMIE1a9YwduzYU57TfC59OfGUudLpYcOGMXXqVP74xz9yxx13nHL9zp07s3jxYlJTUxkyZAjTpk1j8+bNjB8/ngULFvDJJ5/Qs2fP49patk9vvfUWI0aMYNWqVbRr147i4mIee+wxrrnmGtauXctf//rXcvs5ffp0ioqKWLlyJYWFhTRq1OiM53inpqYGK1zwX/SXmpoKgM/nY/369dSsWTMiY9bOOTIzMyksLKSwsJA1a9Ywf/58Pv/8czZv3kxOTg7p6els27aNtm3b8vXXX5+xLwANGzakb9++wU9cGRkZzJ8/n5UrVzJw4EB+8IMfAP5PXj169KBatWo0bNiQjh07Hje2v3//fl5++WV69+7Nxo0bmTJlSvAPc7iEPejNn3C/A/4HuAwYaGaXhXs/scz5ikmI84Ox119/PTNnzmTJkiXUqVOHG2+8keuuu67cKvbiiy9m3bp1HDlyhH379rFgwYIzbr9bt2787ne/C06XBkC1atXKrTI7derEzJkzKSkpoaioiMWLF59TRVSrVi2+/fbbUz7/7bff0rhxY44dO8b06dPPerun89577/Hf//6XQ4cO8ec//5mOHTsC0LdvX9555x0++uijU35yAPjyyy9p1KgRd911F8OGDWPVqlV88803nH/++dSpU4edO3cyb968ctf1+Xxs3bqVa665hmeffZb9+/dz4MAB9u/fHwyy0uMoJ9q/fz8NGzakWrVqLFy4MPhH/nSvYbt27di4cSObN2/m6NGjzJgxg969ewPwwgsv0Lp1a1577TWGDh1a7r9vKFq1akVRURH//Oc/Af9QzqeffkpWVha7du1iy5YtbNmyhbS0NFatWsVFF11E586dee211wCYN29e8P138ODBYB8PHjzI/PnzadOmDQC7du0C/K/t008/zT333AP4Dx5/8MEHwXX+9a9/BT8F33rrrbRt25bNmzczbdo0/v73v3P77bdTo0aNsL4Gkajo2wObnHNfOOeOAjOAGyOwn5hVFSr6UikpKYwcOZLCwkKeeeYZEhNP7nfTpk0ZMGAAbdq0YcCAAeTl5Z1xu48++ih79+6lTZs25OTksHDhQgCGDx8eHHIoq2/fvmRnZ5OTk8O1117Lc889x0UXXXTW/RgyZAj33HPPKU8Hfeqpp/jhD39Ix44dQx6qKtW+fXv69etHdnY2/fr1Cx4fSE5O5pprrmHAgAHlvp6lFi1aRE5ODnl5ecycOZORI0cGpzMyMrjllluCfzxOVFJSwq233kpWVhZ5eXncf//91K1bl4ceeojRo0eTl5d3yk8kgwYNYsWKFWRlZTFt2rTg65GSkkLHjh1p06bNSQdjk5KSmDBhAt27d6d169YMGDCAzMxMNmzYwKRJk3j++efp1KkTnTt35umnnwbgoYceIi0tje+++460tDSeeOIJAD766CPS0tKYPXs2d999N5mZmcH9dOrUiR//+McsWLCAtLQ03n33XZKTk5kzZw6jRo0iJyeH3NzcM56tNXbsWBYvXkxmZiZz584NnpGzc+dOrrrqKnJycmjfvj09e/akR48eALz++utceumlZGRk0KRJE4YOHQrAiBEjOHDgAJmZmbRr146hQ4cGK/YBAwawYcMGxo0bR8uWLU/bplBYuI/0mll/oIdzblhg+jbgh865U540nJ+f78p+lDlbX301m/9snXzmBT12+PAOqlWrR8crF0Vk++vXr6d169YR2bZEn8/nC54ZE8n//FK5lPf/3sxWOufyT7FKUNTGF8xsODAcOKfzV8uqVq0u59dsEc5mhcX5NVtQr16HaDdDKqF169bRq1cv+vbtq5CXsIlE0G8HmpaZTgvMO45zbiIwEfwVfUV21KBBNxo06FaRVUU88e677zJq1Kjj5jVv3pw333yz3OUvu+wyvvjii+PmrVmzhttuu+24edWrV2fZsmXhbazErUgE/UdASzNrjj/gbwZOvoRNpAro3r37aQ+ono2srKxyz00XOVthD3rnXLGZ3Qu8CyQCU5xzn4Z7P1Wdc07fYClSRYR6LDUiY/TOubeBtyOxbfFfmbhnzx59J71IFVB645FQTrmM75O941RaWhrbtm2jqKgo2k0REQ+U3kqwohT0lVC1atUqfEsxEal69O2VIiJxTkEvIhLnFPQiInEu7F+BUKFGmBUBp/76w9O7ECj//l/xS32uGtTnqiGUPl/snGtwpoViIuhDYWYrzua7HuKJ+lw1qM9Vgxd91tCNiEicU9CLiMS5eAj6idFuQBSoz1WD+lw1RLzPlX6MXkRETi8eKnoRETmNSh308XoTcjObYma7zGxtmXn1zew9M9sY+F0vMN/M7LeB1+ATM2sbvZZXnJk1NbOFZrbOzD41s5GB+XHbbzOrYWbLzWx1oM9PBuY3N7Nlgb7NNLPkwPzqgelNgefTo9n+ijKzRDP72Mz+FpiO6/4CmNkWM1tjZoVmtiIwz7P3dqUN+ji/CflUoMcJ8x4GFjjnWgILAtPg73/LwM9w4PcetTHcioGfO+cuAzoAIwL/nvHc7yPAtc65HCAX6GFmHYBngReccy2AvcCdgeXvBPYG5r8QWK4yGgmsLzMd7/0tdY1zLrfMqZTevbedc5XyB7gCeLfM9GhgdLTbFcb+pQNry0xvABoHHjcGNgQevwwMLG+5yvwDFADdqkq/gZrAKuCH+C+eSQrMD77P8d/j4YrA46TAchbttp9jP9MCoXYt8DfA4rm/Zfq9BbjwhHmevbcrbUUPpAJby0xvC8yLV42cczsCj78GGgUex93rEPiIngcsI877HRjGKAR2Ae8BnwP7nHPFgUXK9ivY58Dz+4EUb1scst8ADwG+wHQK8d3fUg6Yb2YrA/fLBg/f2/qa4krIOefMLC5PlzKzC4A3gAecc9+UvbFKPPbbOVcC5JpZXeBNICPKTYoYM+sF7HLOrTSzLtFuj8eucs5tN7OGwHtm9u+yT0b6vV2ZK/qzugl5HNlpZo0BAr93BebHzetgZtXwh/x059zcwOy47zeAc24fsBD/0EVdMystwsr2K9jnwPN1gD0eNzUUHYHeZrYFmIF/+OZF4re/Qc657YHfu/D/QW+Ph+/tyhz0wZuQB47S3wz8JcptiqS/AIMDjwfjH8MunX974Eh9B2B/mY+DlYb5S/fJwHrn3P+VeSpu+21mDQKVPGZ2Hv5jEuvxB37/wGIn9rn0tegPfOACg7iVgXNutHMuzTmXjv//6wfOuUHEaX9Lmdn5Zlar9DFwPbAWL9/b0T5IEeIBjh8Bn+Ef1xwT7faEsV+vAzuAY/jH5+7EPza5ANgIvA/UDyxr+M8++hzAd/J+AAAAhklEQVRYA+RHu/0V7PNV+McxPwEKAz8/iud+A9nAx4E+rwUeD8y/BFgObAJmA9UD82sEpjcFnr8k2n0Ioe9dgL9Vhf4G+rc68PNpaVZ5+d7WlbEiInGuMg/diIjIWVDQi4jEOQW9iEicU9CLiMQ5Bb2ISJxT0IuIxDkFvYhInFPQi4jEuf8HgT2G+nZhcfgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1127cc2b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def train_by_sarsa(S, max_iterations, alpha=0.1):\n",
    "    Q, pi = {}, {}\n",
    "    for state in S:\n",
    "        for aindex in range(len(state.actions)):\n",
    "            Q[(state, aindex)] = 0.0\n",
    "            pi[(state, aindex)] = 1.0 / len(state.actions)\n",
    "    perf = []\n",
    "    for _ in range(max_iterations):\n",
    "        state = S[S.index(State(0, 0, 0))]\n",
    "        aindex = choose_action_from_stochastic_policy(pi, state)\n",
    "        while not state.is_terminal():\n",
    "            action = state.actions[aindex]\n",
    "            to_state = action.to_state\n",
    "            to_aindex = choose_action_from_stochastic_policy(pi, to_state)\n",
    "            Q[(state, aindex)] += alpha * (action.reward +\n",
    "                                           gamma * Q[(to_state, to_aindex)] -\n",
    "                                           Q[(state, aindex)])\n",
    "            update_pi_eps_greedy(pi, Q, state)\n",
    "            state, aindex = to_state, to_aindex\n",
    "        perf.append(measure_policy_performance(S, determine_policy(S, Q)))\n",
    "    return perf\n",
    "\n",
    "\n",
    "plot_method_results(S, train_by_sarsa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "measuring with 10 seeds took 32.66626286506653\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XlwVfX9//Hnm7AWFWQpBYIGCrKHxSQVobK5ULUiqyC24Ia2Upc6CiKKFceNfl1oHa0VRRwGUBa1/jpaWaxgR0OECBgQkUV2EBBBSCA5n98f9+SaQNhyt5PL6zGTyT2fe5bP5+by4nM/53PPMeccIiKSvColugIiIhJbCnoRkSSnoBcRSXIKehGRJKegFxFJcgp6EZEkp6AXEUlyCnoRkSSnoBcRSXKVE10BgHr16rm0tLREV0NEpEL5/PPPv3PO1T/ZeoEI+rS0NHJychJdDRGRCsXMNp7Kehq6ERFJcgp6EZEkp6AXEUlyCnoRkSSnoBcRSXInDXoze9XMdprZyhJldczsQzP72v99rl9uZjbJzNaa2XIz6xzLyouIyMmdSo9+CtDnqLIxwHznXAtgvr8M8Bughf8zEngxOtUUEZHyOuk8eufcx2aWdlRxX6CH//h14CNgtF8+1YXuT/ipmdU2s4bOuW3RqvCZpLBwP5s3v0GRV5DoqohIjNSv15tzzkmP6THK+4WpBiXCezvQwH/cGNhUYr3NftkxQW9mIwn1+jnvvPPKWY3k9t3uj/hm3f/5S5bQuohIbFSr1iCwQR/mnHNmdtp3GHfOvQy8DJCRkaE7lJfBK8oH4OIuH1OjRuME10ZEKqryzrrZYWYNAfzfO/3yLUCTEuul+mVSDp4/ZFMppVqCayIiFVl5g/5dYLj/eDjwTony3/uzby4C9ml8vvyKgz6lkoJeRMrvpEM3Zjad0InXema2GRgPPAm8aWY3AxuBwf7q/wauBNYCB4EbY1DnM4bnHQagUqWqCa6JiFRkpzLrZuhxnupdxroOuCPSSklIcY/eTEEvIuWnb8YGmOcVUKlSNcw040ZEyk9BH2BFftCLiERCQR9gnoJeRKJAQR9gCnoRiQYFfYAp6EUkGhT0AeZ5hxX0IhIxBX2AqUcvItGgoA+wUNBrDr2IREZBH2CeV6DLH4hIxBT0AaahGxGJBgV9gHleAaahGxGJkII+wLwiDd2ISOQU9AHmOU2vFJHIRXyHKYnc/v15rP3maZwrLFV+5MheBb2IREw9+gDYs2cRe/YswvMK8Lwj4Z9atS6kXr1eia6eiFRw6tEHQPENRjp3mk6lSvqTiEh0qUcfAJ5XgFllhbyIxISCPgB0TRsRiSUFfQDoBiMiEksK+gDQNW1EJJYU9AGgSx2ISCwp6ANAQS8isaSgDwAN3YhILCnoA0CzbkQklhT0AaChGxGJJQV9ACjoRSSWFPQBoKAXkVhS0AeAbhkoIrGkoA8AnYwVkVhS0AeAhm5EJJYiCnozu8fMvjSzlWY23cyqm1lTM/vMzNaa2Uwz0wTxk9A8ehGJpXIHvZk1Bu4EMpxz7YAUYAjwFPCsc645sBe4ORoVTWYKehGJpUiHbioDNcysMvAzYBvQC5jlP/86cG2Ex0hqnleIc0UauhGRmCl30DvntgB/Bb4lFPD7gM+B791PNz/dDDSOtJLJ6uDB9SxanAVApZTqCa6NiCSrSIZuzgX6Ak2BRkBNoM9pbD/SzHLMLGfXrl3lrUaFdvDgBgoL99Go4WAaNPhtoqsjIkkqkqGbS4H1zrldzrkjwBygK1DbH8oBSAW2lLWxc+5l51yGcy6jfv36EVSj4vK8AgCaNBlB9Wq/SHBtRCRZRRL03wIXmdnPzMyA3kAesBAY6K8zHHgnsiomr+Kg14lYEYmlSMboPyN00nUpsMLf18vAaODPZrYWqAtMjkI9k9JPQa8TsSISO5VPvsrxOefGA+OPKl4HZEWy3zNFkYJeROJA34xNIPXoRSQeFPQJpKAXkXhQ0CdQKOgr8dMkJRGR6FPQJ1DxxcxCk5ZERGJDQZ9AumqliMSDgj6BvCLdcEREYk9Bn0CedxjTl6VEJMYU9AmkoRsRiQcFfQLpXrEiEg8K+gQK3StWQzciElsK+gTS0I2IxIOCPoE8r4BKKQp6EYktBX0CqUcvIvGg794nQGHhjxw8+A2FhfsV9CIScwr6BFi1+gF27vx/ANSt3CvBtRGRZKegT4Ajh3dTs2YLmv/yfmrVujDR1RGRJKegTwDPK6Ba1QbUq6fevIjEnk7GJoDnHdZsGxGJGwV9AhRpto2IxJGCPgFC0yr1jVgRiQ8FfQJ4XgGVTEEvIvGhoE8AfSNWROJJQZ8AoYuZKehFJD4U9HHmnNOlD0QkrhT0ceZcIeAp6EUkbhT0ceZ5BQAKehGJGwV9nCnoRSTeFPRxVhz0uoWgiMSLgj7O1KMXkXhT0MeZ5x0GwPTNWBGJEwV9nGnoRkTiLaKgN7PaZjbLzFab2Soz62JmdczsQzP72v99brQqmwyKe/QauhGReIm0R/888L5zrhXQAVgFjAHmO+daAPP9ZfFpjF5E4q3cQW9mtYBLgMkAzrnDzrnvgb7A6/5qrwPXRlrJZPJT0GuMXkTiI5IefVNgF/CamS0zs1fMrCbQwDm3zV9nO9CgrI3NbKSZ5ZhZzq5duyKoRsVSpB69iMRZJEFfGegMvOic6wT8yFHDNM45B7iyNnbOveycy3DOZdSvXz+CalQsGroRkXiLJOg3A5udc5/5y7MIBf8OM2sI4P/eGVkVk4uCXkTirdxB75zbDmwys5Z+UW8gD3gXGO6XDQfeiaiGSUZBLyLxVjnC7f8ETDOzqsA64EZC/3m8aWY3AxuBwREeI6noZKyIxFtEQe+cywUyyniqdyT7TWZekXr0IhJf+mZsnHnuMFAJs0g/TImInBoFfZwV313KzBJdFRE5Qyjo40y3ERSReFPQx5lXVKALmolIXCno48zzDqtHLyJxpaCPM88roFKKgl5E4kdBH2eeV0Al0xx6EYkfBX2chU7GKuhFJH4U9HGmMXoRiTcFfZxpjF5E4k1BH2eaRy8i8aagj7MiBb2IxJmCPs7UoxeReNOVtWLAOcc330zk4KGNxzx35MhuBb2IxJWCPgY8L5+N3/6DKlXqUrVqnVLP1aiRRp06XRNUMxE5EynoY8C5QgDSzr+N8867OcG1EZEzncboY8C5IgDMUhJcExERBX1MFPfodXMREQkCBX0MqEcvIkGioI+Bn4JePXoRSTwFfQz8NHSjHr2IJJ6CPgbUoxeRIFHQx4CnHr2IBIiCPgbUoxeRIFHQx4DG6EUkSBT0MaDplSISJAr6GNAXpkQkSBT0MeC84h69Xl4RSTwlUQw4dDJWRIJDQR8DGqMXkSCJOOjNLMXMlpnZe/5yUzP7zMzWmtlMM6saeTUrFo3Ri0iQRKNHfxewqsTyU8CzzrnmwF7gjLsgu3r0IhIkEQW9maUCVwGv+MsG9AJm+au8DlwbyTEqIvXoRSRIIu3RPwfcD3j+cl3ge1ecdLAZaBzhMSoc9ehFJEjKHfRmdjWw0zn3eTm3H2lmOWaWs2vXrvJWI5Ccpx69iARHJD36rsA1ZrYBmEFoyOZ5oLb9lHCpwJayNnbOveycy3DOZdSvXz+CagSPevQiEiTlDnrn3APOuVTnXBowBFjgnBsGLAQG+qsNB96JuJYVjC5qJiJBEot59KOBP5vZWkJj9pNjcIxAC5+MraQevYgkXlS6nM65j4CP/MfrgKxo7LeiUo9eRIJE34yNgeIefSWN0YtIACjoY0A9ehEJEgV9DOjGIyISJAr6GND0ShEJEgV9DOgSCCISJAr6GCju0evlFZEgUBLFgHOFmKUQusabiEhiKehjwLkijc+LSGAo6GMgFPQanxeRYFDQx4B69CISJAr6GFCPXkSCREEfA8UnY0VEgkBBHwPq0YtIkCjoYyC/YBugqZUiEgwK+ijbf2A1e/YsAlyiqyIiAijoo+5wwQ4A0tLuSHBNRERCFPRR5nkFANQ6p0OCayIiEqKgj7IiP+grVaqW4JqIiIQo6KPMeYcBBb2IBIeCPsrUoxeRoFHQR5mnoBeRgFHQR5mnoRsRCRgFfZT91KOvmuCaiIiEKOijzPMKMKuCmV5aEQkGpVGUeV6Bhm1EJFAU9FEWCnoN24hIcCjoo8zzDqtHLyKBoqCPMg3diEjQKOijzPMKSFHQi0iAKOijTD16EQkaBX2UeUUKehEJlnIHvZk1MbOFZpZnZl+a2V1+eR0z+9DMvvZ/nxu96gafevQiEjSR9OgLgXudc22Ai4A7zKwNMAaY75xrAcz3l88YoVk3ml4pIsFR7qB3zm1zzi31H+8HVgGNgb7A6/5qrwPXRlrJiqRIPXoRCZiojNGbWRrQCfgMaOCc2+Y/tR1ocJxtRppZjpnl7Nq1KxrVCAQN3YhI0EQc9GZ2FjAbuNs590PJ55xzjuPcJds597JzLsM5l1G/fv1IqxEYTkM3IhIwEQW9mVUhFPLTnHNz/OIdZtbQf74hsDOyKlYsRV4BlVLUoxeR4Ihk1o0Bk4FVzrlnSjz1LjDcfzwceKf81at4NHQjIkFTOYJtuwK/A1aYWa5fNhZ4EnjTzG4GNgKDI6tixeGcU9CLSOCUO+idc4sBO87Tvcu734rMuULAU9CLSKDom7FRpPvFikgQKeijSLcRFJEgUtBHkXr0IhJECvooUtCLSBAp6KPI8w4DCnoRCRYFfRQV9+h14xERCRIFfRSpRy8iQaSgjyKN0YtIECnoo0jTK0UkiBT0UVSkHr2IBJCCPorUoxeRIFLQR5HG6EUkiBT0UaRZNyISRAr6KNmz539s3x669L6CXkSCREEfJd9uepX9+1dQu1YmKSk/S3R1RETCIrnxiJTgeQWcc046F144I9FVEREpRT36KNGdpUQkqBT0UaKgF5GgUtBHiYJeRIJKQR8lCnoRCSqdjI0SzztcKuiPHDnC5s2byc/PT2CtRCQZVK9endTUVKpUqVKu7RX0UXJ0j37z5s2cffbZpKWlYWYJrJmIVGTOOXbv3s3mzZtp2rRpufahoZso8byCUjccyc/Pp27dugp5EYmImVG3bt2IRgcU9FESGropfTEzhbyIREOkWaKgjwLPK8S5Qp2MFZFAUtBHgXMV82JmixYtom3btnTs2JFDhw5Fbb+PP/54qeWLL744KvudMmUKW7duPe3tXnrpJaZOnVqu440aNeq0t4tEjx49yMnJiekxtm7dysCBA2N6jNzcXP79738f9/knnniC5s2b07JlSz744IOT7q9Pnz7Url2bq6++ulT53//+d5o3b46Z8d1334XLV69eTZcuXahWrRp//etfS20zadIkWrduzbBhw5g4cSIdO3akY8eOtGvXjpSUFPbs2XPcetxzzz3h9S+44AJq164NwEcffXRM3YpNnz6d9u3bk56eTp8+fUrVM26ccwn/ufDCC11FdvjwHjdvfjP37bevhcvy8vISV6ETKCgocAcOHHDOOXfbbbe5N954I+rHqFmzZtT36Zxz3bt3d0uWLCnzucLCwqgf77XXXnN33HFH1Pd7Iidq4+k4cuRIFGpTfid67b788kuXnp7u8vPz3bp161yzZs1O+vebN2+ee/fdd91VV11Vqnzp0qVu/fr17vzzz3e7du0Kl+/YscNlZ2e7sWPHuokTJ5bapmXLlm7Tpk3HHOPdd991PXv2PNUmukmTJrkbb7zROefcwoULj6mbc6G/Q/369cN1u++++9z48eOdc87t2bPnlI/lXNmZAuS4U8hY9eij4GR3llqzZgKfL70+qj9r1kw4rTquWrWKe++9l5YtW7JmzRpeeeUV3nzzTR566CGGDRt2TI9k1KhRTJkyBYC0tDTGjx9P586dad++PatXrwbgwIED3HjjjeHeyuzZsxkzZgyHDh2iY8eODBs2DICzzjoLCHUq7rvvPtq1a0f79u2ZOXMmEOoN9ejRg4EDB9KqVSuGDRtG6D38k1mzZpGTk8OwYcPCn0DS0tIYPXo0nTt35q233uKf//wnmZmZdOjQgQEDBnDw4EEAHnnkkXCvrkePHowePZqsrCwuuOACFi1adMLXbdOmTfTo0YMWLVrwl7/8BYCHH36Y5557LrzOgw8+yPPPP1/m9s45Ro0aRcuWLbn00ku58sormTVr1sn/YMB//vMfunTpQufOnRk0aBAHDhwA4NFHHyUzM5N27doxcuTI8GvVo0cP7r77bjIyMnj++ecZMWIEd955JxdffDHNmjULH3fDhg20a9cOCH1q6d+/P3369KFFixbcf//94eNPnjyZCy64gKysLG699dYyP91kZ2fTpUsXOnXqxMUXX8xXX33F4cOHefjhh5k5cyYdO3YM/52LvfPOOwwZMoRq1arRtGlTmjdvTnZ2NkuWLCE9PZ38/Hx+/PFH2rZty8qVKwHo3bs3Z5999jHH79SpE2lpaceU//znPyczM/OY6Yi3334769at4ze/+Q3PPvtsqeemT5/O0KFDw8tTp04lPT2dDh068Lvf/e6YYxy9/g8//MBVV11Fy5Ytuf322/E8Lxy0P/74I845fvjhBxo1agTAxIkTycrK4h//+Ac//PDDMfuPJgV9FHhFwbzhyI8//shrr71Gt27duPXWW2nTpg3Lly+nU6dO3HLLLVxzzTVMnDiRadOmnXRf9erVY+nSpfzhD38Ih+aECROoVasWK1asYPny5fTq1Ysnn3ySGjVqkJube8x+58yZQ25uLl988QXz5s3jvvvuY9u2bQAsW7aM5557jry8PNatW8cnn3xSatuBAweSkZHBtGnTyM3NpUaNGgDUrVuXpUuXMmTIEPr378+SJUv44osvaN26NZMnTy6zLYWFhWRnZ/Pcc8+Fw/t4srOzmT17NsuXL+ett94iJyeHm266KTwU5HkeM2bM4IYbbihz+7lz5/LVV1+Rl5fH1KlT+d///neSVzrku+++47HHHmPevHksXbqUjIwMnnnmGSD0n/CSJUtYuXIlhw4d4r333gtvd/jwYXJycrj33nsB2LZtG4sXL+a9995jzJgxZR4rNzeXmTNnsmLFCmbOnMmmTZvYunUrEyZM4NNPP+WTTz4J/+d+tFatWrFo0SKWLVvGo48+ytixY6latSqPPvoo1113Hbm5uVx33XWlttmyZQtNmjQJL6emprJlyxYyMzO55pprGDduHPfffz833HBD+D+kaHnppZdo1KgRCxcu5J577gmXHzx4kPfff58BAwYA8OWXX/LYY4+xYMECvvjii2P+I9+4cSPr16+nV69e4bLs7Gz+9re/kZeXxzfffMOcOXOoUqUKL774Iu3bt6dRo0bk5eVx8803A6EhzjfeeIN169bRuXNnbrzxRhYvXhzV9hbTPPooONmdpS644KF4ViesYcOGpKen88orr9CqVauI9tW/f38ALrzwQubMmQPAvHnzmDHjp6t1nnvuuSfcx+LFixk6dCgpKSk0aNCA7t27s2TJEs455xyysrJITU0FoGPHjmzYsIFu3bqdtF4lQ2TlypWMGzeO77//ngMHDnDFFVectC0bNmw44f4vu+wy6tatG95u8eLF3H333dStW5dly5axY8cOOnXqFF7naB9//HG4zY0aNSoVDCfy6aefkpeXR9euXYFQgHfp0gWAhQsX8vTTT3Pw4EH27NlD27Zt+e1vf3vM6wFw7bXXUqlSJdq0acOOHTvKPFbv3r2pVasWAG3atGHjxo189913dO/enTp16gAwaNAg1qxZc8y2+/btY/jw4Xz99deYGUeOHDml9h3Pww8/TGZmJtWrV2fSpEkR7et0/Otf/6Jr167h9i5YsIBBgwZRr149gHB5sRkzZjBw4EBSUlLCZVlZWTRr1gyAoUOHsnjxYvr27cuLL77IsmXLaNasGX/605944oknGDduHAAtW7bkqaee4vHHH2f69OlcddVVDB8+POptj0nQm1kf4HkgBXjFOfdkLI4TFEG9heCsWbOYPHky/fv3Z8iQIQwfPpzzzz+/zHUrV66M53nh5aPn7FarFmpbSkoKhYWFUa9r8f5P9xg1a9YMPx4xYgRvv/02HTp0YMqUKXz00UcnPNapHOfoaW3Fy7fccgtTpkxh+/bt3HTTTadU19PhnOOyyy5j+vTppcrz8/P54x//SE5ODk2aNOGRRx4p9bcq+XpA6df16OGwstY53b/vQw89RM+ePZk7dy4bNmygR48eJ92mcePGbNq0Kby8efNmGjduDMDu3bs5cOAAR44cIT8//5j2xMqMGTNKDcOcyvovvPBCqbKy3iu5ubkA/PKXvwRg8ODBPPnkT3HonGPhwoW8+uqrZGdnc+edd3LLLbeUtxnHFfWhGzNLAV4AfgO0AYaaWZtoHydIvIDOurn88suZOXMmixYtolatWvTt25dLL720zF7s+eefT15eHgUFBXz//ffMnz//pPu/7LLLSr3Z9+7dC0CVKlXK7Nn9+te/ZubMmRQVFbFr1y4+/vhjsrKyTrk9Z599Nvv37z/u8/v376dhw4YcOXLklIajTsWHH37Inj17OHToEG+//Xa4h92vXz/ef/99lixZctxPDgCXXHJJuM3btm1j4cKFp3Tciy66iE8++YS1a9cCoWG4NWvWhEO9Xr16HDhw4JTH+09XZmYm//3vf9m7dy+FhYXMnj27zPX27dsXDuniczpw4r/VNddcw4wZMygoKGD9+vV8/fXX4ffBbbfdxoQJExg2bBijR4+ObqOOY9++ffz3v/+lb9++4bJevXrx1ltvsXv3boBSM3FWr17N3r17w5+wimVnZ7N+/Xo8z2PmzJl069aNxo0bk5eXx65du4DQ+6l169YATJs2jVatWvHCCy9w/fXXs2rVKiZMmHDczlgkYjFGnwWsdc6tc6F5hzOAvifZpkIL6hh9sbp163LXXXeRm5vL448/XurjZrEmTZowePBg2rVrx+DBg+nUqdNJ9ztu3Dj27t1Lu3bt6NChQzjERo4cSXp6evhkbLF+/fqFT2716tWLp59+ml/84hen3I4RI0Zw++23H3c66IQJE/jVr35F165dIx6qKpaVlcWAAQNIT09nwIABZGRkAFC1alV69uzJ4MGDy3w9i/Xr148WLVrQpk0bfv/73x8TDsdTv359pkyZwtChQ0lPT6dLly6sXr2a2rVrc+utt9KuXTuuuOIKMjMzo9LOozVu3JixY8eSlZVF165dSUtLCw/vlHT//ffzwAMP0KlTp1KfBHr27EleXl6ZJ2Pbtm3L4MGDadOmDX369OGFF14gJSWFqVOnUqVKFa6//nrGjBnDkiVLWLBgARDqJAwaNIj58+eTmpoanpI5adIkUlNT2bx5M+np6eHe8Pbt20lNTeWZZ57hscceIzU19bgnPOfOncvll19e6tND27ZtefDBB+nevTsdOnTgz3/+c/i5GTNmMGTIkGN68JmZmYwaNYrWrVvTtGlT+vXrR6NGjRg/fjyXXHIJ6enp5ObmMnbsWCDUuVq8eDGzZ8/myiuvPOH7KFJ2vI9z5d6h2UCgj3PuFn/5d8CvnHPHnZCckZHhyjN3eOvWt/h2U9kn3OKpqPBH8gu2knHhbGrV6giEZrkU/88tycfzvPBsnxYtWpzydiNGjODqq6+O+Tz2aDhw4ABnnXUWhYWF9OvXj5tuuol+/folulpnrLIyxcw+d85lnGzbhJ2MNbORwEiA8847r1z7qFKlNjV/1jya1Sq3OpW7cdZZ0elFSrDl5eVx9dVXh3vryeqRRx5h3rx55Ofnc/nll3PttdcmukpSTrHo0XcBHnHOXeEvPwDgnHvieNuUt0cfZOrRVxwffPDBMePBTZs2Ze7cuae8jxUrVhwz17patWp89tlnZa7fr18/1q9fX6rsqaeeOuF4v5zZIunRxyLoKwNrgN7AFmAJcL1z7svjbaOgFxE5sUAN3TjnCs1sFPABoemVr54o5JOZc05XsBSRiEXaIY/JGL1z7t/A8a9odAaoXr06u3fv1jXpRSQizr/xSPXq1cu9D30zNkaKp3wVz58VESmv4lsJlpeCPkaqVKlS7tt+iYhEky5qJiKS5BT0IiJJTkEvIpLkoj6PvlyVMNsFbCzn5vWABNybK6HU5jOD2nxmiKTN5zvn6p9spUAEfSTMLOdUvjCQTNTmM4PafGaIR5s1dCMikuQU9CIiSS4Zgv7lRFcgAdTmM4PafGaIeZsr/Bi9iIicWDL06EVE5AQqdNCbWR8z+8rM1prZmETXJ1rM7FUz22lmK0uU1TGzD83sa//3uX65mdkk/zVYbmadE1fz8jOzJma20MzyzOxLM7vLL0/adptZdTPLNrMv/Db/xS9vamaf+W2baWZV/fJq/vJa//m0RNa/vMwsxcyWmdl7/nJStxfAzDaY2QozyzWzHL8sbu/tChv0SX4T8ilAn6PKxgDznXMtgPn+MoTa38L/GQm8GKc6RlshcK9zrg1wEXCH//dM5nYXAL2ccx2AjkAfM7sIeAp41jnXHNgL3OyvfzOw1y9/1l+vIroLWFViOdnbW6ync65jiamU8XtvO+cq5A/QBfigxPIDwAOJrlcU25cGrCyx/BXQ0H/cEPjKf/wPYGhZ61XkH+Ad4LIzpd3Az4ClwK8IfXmmsl8efp8TusdDF/9xZX89S3TdT7OdqX6o9QLeAyyZ21ui3RuAekeVxe29XWF79EBjYFOJ5c1+WbJq4Jzb5j/eDjTwHyfd6+B/RO8EfEaSt9sfxsgFdgIfAt8A3zvnCv1VSrYr3Gb/+X1A3fjWOGLPAfcDnr9cl+RubzEH/MfMPvfvlw1xfG/rMsUVkHPOmVlSTpcys7OA2cDdzrkfSt60JRnb7ZwrAjqaWW1gLpC0d5g3s6uBnc65z82sR6LrE2fdnHNbzOznwIdmtrrkk7F+b1fkHv0WoEmJ5VS/LFntMLOGAP7vnX550rwOZlaFUMhPc87N8YuTvt0AzrnvgYWEhi5q+/dehtLtCrfZf74WsDvOVY1EV+AaM9sAzCA0fPM8ydveMOfcFv/3TkL/oWcRx/d2RQ76JUAL/4x9VWAI8G6C6xRL7wLD/cfDCY1hF5f/3j9TfxGwr8THwQrDQl1cUOALAAABAklEQVT3ycAq59wzJZ5K2nabWX2/J4+Z1SB0TmIVocAf6K92dJuLX4uBwALnD+JWBM65B5xzqc65NEL/Xhc454aRpO0tZmY1zezs4sfA5cBK4vneTvRJighPcFwJrCE0rvlgousTxXZNB7YBRwiNz91MaGxyPvA1MA+o469rhGYffQOsADISXf9ytrkboXHM5UCu/3NlMrcbSAeW+W1eCTzslzcDsoG1wFtANb+8ur+81n++WaLbEEHbewDvnQnt9dv3hf/zZXFWxfO9rW/GiogkuYo8dCMiIqdAQS8ikuQU9CIiSU5BLyKS5BT0IiJJTkEvIpLkFPQiIklOQS8ikuT+P4ReBCoE52yUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1124bf4a8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def train_by_q_learning(S, max_iterations, alpha=0.1):\n",
    "    Q, pi = {}, {}\n",
    "    for state in S:\n",
    "        for aindex in range(len(state.actions)):\n",
    "            Q[(state, aindex)] = 0.0\n",
    "            pi[(state, aindex)] = 1.0 / len(state.actions)\n",
    "    perf = []\n",
    "    for _ in range(max_iterations):\n",
    "        state = S[S.index(State(0, 0, 0))]\n",
    "        while not state.is_terminal():\n",
    "            aindex = choose_action_from_stochastic_policy(pi, state)\n",
    "            action = state.actions[aindex]\n",
    "            to_state = action.to_state\n",
    "            _, best_to_q = find_best_action_quality(Q, to_state)\n",
    "            Q[(state, aindex)] += alpha * (action.reward +\n",
    "                                           gamma * best_to_q -\n",
    "                                           Q[(state, aindex)])\n",
    "            update_pi_eps_greedy(pi, Q, state)\n",
    "            state = to_state\n",
    "        perf.append(measure_policy_performance(S, determine_policy(S, Q)))\n",
    "    return perf\n",
    "\n",
    "\n",
    "plot_method_results(S, train_by_q_learning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's try to increase the number of cows around the field and reevaluate the methods."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "cows = [(field_size-5, field_size-5), \n",
    "        (field_size-5, field_size-1), \n",
    "        (field_size-1, field_size-5)]\n",
    "S = build_states(field_size, cows, pickup_reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "measuring with 10 seeds took 0.7505807876586914\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xt0VPW5//H3k4Rc8IYCUiBgOAu8QEi4BASpykUQjx5REYRif9hqsadFqfVUrNcedFVbu9RyjutYqwWsLEDB2+lxabmpqFUSISiCVRYQCHKJ3BRm5/79/TGTkUAgITOTPTP5vNZiJbMze+9nZ8InO9/9zP6acw4REUleKX4XICIisaWgFxFJcgp6EZEkp6AXEUlyCnoRkSSnoBcRSXIKehGRJKegFxFJcgp6EZEkl+Z3AQAdOnRwOTk5fpchIpJQPv7446+dcx0be15cBH1OTg5FRUV+lyEiklDMrKQpz9PQjYhIklPQi4gkOQW9iEiSU9CLiCQ5Bb2ISJJrNOjN7C9mtsfM1h+x7CwzW2pmX4Y+nhlabmY228w2mdknZjYglsWLiEjjmnJGPxcYe9Syu4HlzrlewPLQY4ArgF6hf9OA/4lOmSIi0lyN9tE75941s5yjFo8Dhoc+nwe8DcwMLX/eBecn/NDM2plZZ+fczmgVLMlh//4P2bf/H36XIeK7jh1GcfrpeTHdR3PfMNXpiPDeBXQKfd4V2H7E80pDy44JejObRvCsn+7duzezDElUX3wxi0OH/wmY36WI+Cojo1PcBn2Yc86Z2UnPMO6cewZ4BqCgoEAzlLciztUS8Ero3u1mevW6x+9yRJJec7tudptZZ4DQxz2h5TuAbkc8Lzu0TCSsonIPtbXlZGXpLzmRltDcoH8dmBr6fCrw2hHL/1+o+2YIcFDj83I0LxC8PUdW1jk+VyLSOjQ6dGNmCwheeO1gZqXAg8CjwItmdjNQAkwMPf0N4F+BTUAA+FEMapYE53nBoG/bVkEv0hKa0nUz+ThfGtXAcx3w80iLkuQW8EowSyMjo4vfpYi0CnpnrLQ4z9tGZmY2KSlxcZdskaSnoJcW5wVKNGwj0oIU9NKinHMEvBJ13Ii0IAW9tKiqqr3U1BxSx41IC1LQS4sK1HXcZOX4W4hIK6KglxalHnqRlqeglxbleduAFLKyuvpdikiroaCXFhXwSsjM7EJKSobfpYi0Ggp6aVGeV0JbDduItCgFvbSoQECtlSItTUEvLaaq6gDV1QfI0pulRFqUgl5aTPBCLBq6EWlhCnppMXU99GqtFGlZCnppMd/10GuMXqQlKeilxXheCRkZ3yM1NcvvUkRaFQW9tBjdzEzEHwp6aTGeV6LxeREfKOilRVRXH6Ky8mt13Ij4QEEvLcLztgOoh17EBwp6aRHhCcF1Ri/S4hT00iK+66HXxViRlqaglxbhBbbSps1ZpKWd5ncpIq2Ogl5aREB3rRTxjYJeWoTnlehCrIhPFPQSczU15VRU7CJL88SK+EJBLzHnlQdbKzV0I+IPBb3EXPhmZhq6EfGFgl5i7rseerVWivhBQS8xF/BKSEs7nbS0dn6XItIqKegl5rxA8GZmZuZ3KSKtUkRBb2Z3mNlnZrbezBaYWaaZ9TCzj8xsk5ktMrP0aBUricnztulCrIiPmh30ZtYVuB0ocM7lAqnAJOB3wBPOuZ7AfuDmaBQqiam2thKvvFQXYkV8FOnQTRqQZWZpQFtgJzASWBz6+jzgmgj3IQmsvHwHUKszehEfNTvonXM7gD8A2wgG/EHgY+CAc6469LRSoGukRUri0oTgIv6LZOjmTGAc0APoApwCjD2J9aeZWZGZFZWVlTW3DIlz300IrqAX8UskQzeXAVucc2XOuSrgZWAY0C40lAOQDexoaGXn3DPOuQLnXEHHjh0jKEPiWcArITW1LenpHfwuRaTViiTotwFDzKytBfvmRgEbgJXA9aHnTAVei6xESWSet02tlSI+i2SM/iOCF13XAJ+GtvUMMBP4pZltAtoDz0WhTklQmhBcxH9pjT/l+JxzDwIPHrV4MzA4ku1KcnCuBs/bTscOo/0uRaRV0ztjJWbKy3fiXJWmDxTxmYJeYqbuZmZ6s5SIvxT0EjOB8F0rFfQiflLQS8x4XgkpKelkZHzP71JEWjUFvcTMd3et1I+ZiJ/0P1BiJqDWSpG4oKCXmHCuVrcnFokTCnqJiYrKPdTWlqu1UiQOKOglJnQzM5H4oaCXmPC8bQC0VQ+9iO8U9BITAa8EszQyMrr4XYpIq6egl5jwvBIyM7NJSYnodkoiEgUKeokJL1CiYRuROKGgl6hzzoV66NVxIxIPFPQSdVVVe6mpOaSOG5E4oaCXqAt33GTl+FuIiAAKeomBgHroReKKgl6iLngf+hSysrr6XYqIoKCXGAh4JWRmdiElJcPvUkQEBb3EgKeOG5G4oqCXqAsESnTXSpE4oqCXqKqqOkh19QHNEysSRxT0ElWe5okViTsKeomqugnB1VopEj8U9BJV392HXhdjReKFgl6iyvNKyEjvRGpqlt+liEiIgl6iKuCV6EKsSJxR0EtUed42jc+LxBkFvURNdfVhKivL1HEjEmcU9BI1dXet1NCNSHxR0EvUqIdeJD5FFPRm1s7MFpvZ52a20cyGmtlZZrbUzL4MfTwzWsVKfPuuh16tlSLxJNIz+j8CbzrnzgfygY3A3cBy51wvYHnosbQCXmArbdqcRVraaX6XIiJHaHbQm9kZwCXAcwDOuUrn3AFgHDAv9LR5wDWRFimJwfO2adhGJA5FckbfAygD5pjZWjN71sxOATo553aGnrML6NTQymY2zcyKzKyorKwsgjIkXqiHXiQ+RRL0acAA4H+cc/2Bwxw1TOOcc4BraGXn3DPOuQLnXEHHjh0jKEPiQU1NORUVO8nSPLEicSeSoC8FSp1zH4UeLyYY/LvNrDNA6OOeyEqUROCVbwfUcSMSj5od9M65XcB2MzsvtGgUsAF4HZgaWjYVeC2iCiUh6GZmIvErLcL1bwPmm1k6sBn4EcFfHi+a2c1ACTAxwn1IAgj30GuMXiTuRBT0zrlioKCBL42KZLuSeALeNtLSTictrZ3fpYjIUfTOWImK4ITg52BmfpciIkdR0EtUeJoQXCRuKeglYrW1lXjlpeqhF4lTCnqJWHn5DqBWHTcicUpBLxELhO9ameNvISLSoEjbKxNeVVUVpaWllJeX+11KwqquPoWzzvwTX32Vyc6dG/0uRyTpZGZmkp2dTZs2bZq1fqsP+tLSUk477TRycnLUMdJM5eVfUVXVllNP7a3voUiUOefYu3cvpaWl9OjRo1nbaPVDN+Xl5bRv314BFYHa2kosJV3fQ5EYMDPat28f0ahDqw96QAEVodraSlIs3e8yRJJWpBmloJeIOOeCQZ+S4XcpInIcCvoEsGrVKvr06UO/fv3wPC9q2/3tb39b7/FFF1100ttwrgpwpKR8d0Y/d+5cvvrqq5Pe1tNPP83zzz9/0uvNnTuX6dOnn/R6kdi6dSu5ublR3eYtt9zChg0bgGNfm0gd/Zocua9YKC4u5o033jju1x955BF69uzJeeedx1tvvdXo9saOHUu7du246qqr6i3/7//+b3r27ImZ8fXXX4eXf/755wwdOpSMjAz+8Ic/1Ftn9uzZXHDBBUyZMoXHHnuMfv360a9fP3Jzc0lNTWXfvn0nebQn3t8TTzxBnz59yM3NZfLkyf40fjjnfP83cOBA55cNGzb4tu8TqaiocIcOHXLOOXfrrbe6v/71r1HfxymnnBLxNqqqvnUHD37iqqq+DS+79NJLXWFhYYPPr66ujnifR5szZ477+c9/HvXtnsiWLVtcnz59Yrb95rw2J/renug1iYUTvSafffaZy8vLc+Xl5W7z5s3uX/7lXxr9uVi2bJl7/fXX3ZVXXllv+Zo1a9yWLVvcOeec48rKysLLd+/e7VavXu3uuece99hjj9Vb57zzznPbt28/Zh+vv/66GzFiRFMPsZ7j7a+0tNTl5OS4QCDgnHNuwoQJbs6cOc455/bt23dS+2goq4Ai14SM1Rn9Eb744iE+XvODqP774ouHTqqGjRs3cuedd3LeeefxxRdf8Oyzz/Liiy9y//33M2XKFN5+++16ZzXTp09n7ty5AOTk5PDggw8yYMAA+vbty+effw7AoUOH+NGPfkTfvn3Jy8tjyZIl3H333XieR79+/ZgyZQoAp556KhD85f+rX/2K3Nxc+vbty6JFiwB4++23GT58ONdffz3nn38+U6ZMoaYmeHZSd0a/ePFiioqKmDJlSvgvkJycHGbOnMmAAQN46aWX+POf/8ygQYPIz89n/PjxBAIBAH7zm9+Ez4aGDx/OzJkzGTx4MOeeey6rVq064fdt+/btDB8+nF69evGf//mfADzwwAM8+eST4efce++9/PGPf2xw/UmTJvF///d/4cc33XQTixcvZuvWrVx88cUMGDCAAQMG8MEHHxyz7tF/UVx11VW8/fbbAPz9739n6NChDBgwgAkTJnDo0KHjHsPw4cMpKipq8LV54YUXGDx4MP369ePWW2+lpqYGCL5md955J/n5+fzjH/9g1qxZDBo0iNzcXKZNm4ZzrsHXpG5fAAsWLKBv377k5uYyc+bMcD2nnnoq9957L/n5+QwZMoTdu3cfU/Pq1asZOnQo/fv356KLLuKf//wnlZWVPPDAAyxatIh+/fqFf37qvPbaa0yaNImMjAx69OhBz549Wb16NYWFheTl5VFeXs7hw4fp06cP69evB2DUqFGcdtqxcxH379+fnJycY5afffbZDBo06Jh2xJ/+9Kds3ryZK664gieeeKLe1xYsWMDkyZPDj59//nny8vLIz8/nhz/8IQBlZWWMHz+eQYMGMWjQIN5///0T7g+guroaz/Oorq4mEAjQpUsXAG677TZGjhzJ/PnzY36Wr6CPA4cPH2bOnDl8//vf5yc/+Qm9e/fmk08+oX///txyyy1cffXVPPbYY8yfP7/RbXXo0IE1a9bw7//+7+HQfOihhzjjjDP49NNP+eSTTxg5ciSPPvooWVlZFBcXH7Pdl19+meLiYtatW8eyZcv41a9+xc6dwdkh165dy5NPPsmGDRvYvHkz7733HphhFvwBv/766ykoKGD+/PkUFxeTlZUFQPv27VmzZg2TJk3iuuuuo7CwkHXr1nHBBRfw3HPPNXgs1dXVrF69mieffDIc3sezevVqlixZwieffMJLL71EUVERP/7xj8NDQbW1tSxcuJAbb7yxwfVvuOEGXnzxRQAqKytZvnw5V155JWeffTZLly5lzZo1LFq0iNtvv73R16DO119/zcMPP8yyZctYs2YNBQUFPP74442ud/Rrs3HjRhYtWsT7779PcXExqamp4dfs8OHDXHjhhaxbt47vf//7TJ8+ncLCQtavX4/nefztb3877msC8NVXXzFz5kxWrFhBcXExhYWFvPrqq+FtDxkyhHXr1nHJJZfw5z//+Zhazz//fFatWsXatWuZNWsW99xzD+np6cyaNYsbbriB4uJibrjhhnrr7Nixg27duoUfZ2dns2PHDgYNGsTVV1/Nfffdx1133cWNN94Y9eGxp59+mi5durBy5UruuOOO8PJAIMCbb77J+PHjAfjss894+OGHWbFiBevWrQufIMyYMYM77riDwsJClixZwi233HLC/XXt2pX/+I//oHv37nTu3JkzzjiDMWPGAMFf3o899hgffPABffr04bbbbmPdunVRPd46rb6P/kjnnnu/L/vt3LkzeXl5PPvss5x//vkRbeu6664DYODAgbz88ssALFu2jIULF4afc+aZZ55wG++99x6TJ08mNTWVTp06cemll1JYWMjpp5/O4MGDyc7OBqBfv35s3bqVCy9svH/+yP/s69ev57777uPAgQMcOnSIyy+/vNFj2bp16wm3P3r0aNq3bx9e77333uMXv/gF7du3Z+3atezevZv+/fuHn3O0K664ghkzZlBRUcGbb77JJZdcQlZWFgcPHmT69OnhgP3iiy9OWMeRPvzwQzZs2MCwYcOA4C+QoUOHNnn9OsuXL+fjjz9m0KBBAHiex9lnnw1AampqOJwAVq5cye9//3sCgQD79u2jT58+/Nu//dtxt11YWMjw4cOpm85zypQpvPvuu1xzzTWkp6eH/3ocOHAgS5cuPWb9gwcPMnXqVL788kvMjKqqqpM+viM98MADDBo0iMzMTGbPnh3Rtk7G//7v/zJs2DDOOussAFasWMGECRPo0KEDQHj5smXL6l3b+Oabbzh06FD4r+Gj7d+/n9dee40tW7bQrl07JkyYwAsvvBA+4Rg4cCADBw6kvLycP/3pTwwePJhHHnmEX/7yl1E9PgV9HFi8eDHPPfcc1113HZMmTWLq1Kmcc07DNwhLS0ujtrY2/PjoP/kyMoLdL6mpqVRXV0e91rrt1+2jqqq8SR03p5xySvjzm266iVdffZX8/Hzmzp0bHuY43r6acixH/6Kpe3zLLbcwd+5cdu3axY9//OPjrp+Zmcnw4cN56623WLRoEZMmTQKCF9I6derEunXrqK2tJTMz85h1j/eaOOcYPXo0CxYsOGHtjXHOMXXqVB555JEG605NTQ3v92c/+xlFRUV069aN3/zmNxENCbRp0yb8fTzea3D//fczYsQIXnnlFbZu3crw4cMb3W7Xrl3Zvn17+HFpaSldu3YFYO/evRw6dIiqqirKy8vr/dzE0sKFC+sN2xxPbW0tH374YYM/Bw1ZtmwZPXr0CP8ive666/jggw/CQV9dXc0bb7zBX/7yFzZt2sSsWbOO+1dnJDR0EwfGjBnDokWLWLVqFWeccQbjxo3jsssua/As9pxzzmHDhg1UVFRw4MABli9f3uj2R48ezVNPPRV+vH//fiD4H7mhM7CLL76YRYsWUVNTQ1lZGe+++y6DBw8+5nnOORzV9TpuAE477TS+/fbb49bz7bff0rlzZ6qqqpo0HNUUS5cuZd++fXiex6uvvho+i7722mt58803KSwsPO5fDnVuuOEG5syZw6pVqxg7diwQPGPt3LkzKSkp/PWvfw2PjR8pJyeH4uJiamtr2b59O6tXrwZgyJAhvP/++2zatAkIDoU09S+CI1+bUaNGsXjxYvbsCU6/vG/fPkpKSo5Zpy7UO3TowKFDh1i8eHH4a8d7TQYPHsw777zD119/TU1NDQsWLODSSy9tUo0Q/P7UhXTdtaIT7Q/g6quvZuHChVRUVLBlyxa+/PLL8M/XrbfeykMPPcSUKVPqXS+IpYMHD/LOO+8wbty48LKRI0fy0ksvsXfvXoBwJ86YMWP4r//6r/DziouLT7jt7t278+GHHxIIBHDOsXz5ci644AIAHn/8cc4991yWLFnCnXfeyfr165k5c2b4r7VoUtDHkfbt2zNjxgyKi4v57W9/Gz5TO1K3bt2YOHEiubm5TJw4kf79+ze63fvuu4/9+/eTm5tLfn4+K1euBGDatGnk5eWFL/jVufbaa8MXoUaOHMnvf/97vve97zWwZQeOY4L+pptu4qc//elx20EfeughLrzwQoYNGxbxUFWdwYMHM378ePLy8hg/fjwFBcGJz9LT0xkxYgQTJ05s8Pt5pDFjxvDOO+9w2WWXkZ4ePKaf/exnzJs3j/z8fD7//PMGzzCHDRtGjx496N27N7fffjsDBgwAoGPHjsydO5fJkyeTl5fH0KFDwxfIG3Pka9O7d28efvhhxowZQ15eHqNHjw5fMzlSu3bt+MlPfkJubi6XX355eKgHjv+adO7cmUcffZQRI0aQn5/PwIED6wVeY+666y5+/etf079//3pn/CNGjGDDhg0NXozt06cPEydOpHfv3owdO5annnqK1NRUnn/+edq0acMPfvAD7r77bgoLC1mxYgUQPPmYMGECy5cvJzs7O9ySOXv2bLKzsyktLSUvLy88Zr5r1y6ys7N5/PHHefjhh8nOzuabb75p8BheeeUVxowZU++17dOnD/feey+XXnop+fn54aGU2bNnU1RURF5eHr179+bpp58+4f4uvPBCrr/++nCDRG1tLdOmTQMgLy+P4uJi5s2bx8UXX9zk73lzWLBDx18FBQWurgOgpW3cuDH8G1ZOTnX1YQKBzbRtm0Na2rEdEfGgtrY23O3Tq1cvv8sRabaGssrMPnbONTSdaz06o5dmq62tAI49o48XGzZsoGfPnowaNUohL62aLsZKs9XWVgKGtdB9bt56661jxm179OjBK6+80uDze/fuzebNm+st+/TTT8M90XUyMjL46KOPolvsCVx77bVs2bKl3rLf/e53jV5DEGkuBb00W62rJCWlTYvdFO7yyy+POAz79u3b6AW0WDveLyaRWNHQDcHuETl5rrYibodtRJJJpBnV6oM+MzOTvXv3KuxPktNdK0VahAtNPNLU3v2GtPqhm7rWrLKyMr9LSSjO1VBRsYu0tABpaQf9LkckqdVNJdhcrT7o27Rp0+zpuVqzgwfXUPTxreTnPUuHDoMaX0FEfNPqh26keQKB4Dszs7IavlWDiMQPBb00i+eVAClkZXX1uxQRaYSCXpol4JWQmdlZF2NFEoCCXprF87Zp2EYkQUQc9GaWamZrzexvocc9zOwjM9tkZouspd42KS3K80poq6AXSQjROKOfAWw84vHvgCeccz2B/cDNUdiHxJGqqoNUVe0nq62CXiQRRBT0ZpYNXAk8G3pswEig7kbY84BrItmHxJ/ghVh0Ri+SICI9o38SuAuom16nPXDAOVd3Y+pSQG0ZSSbgqbVSJJE0O+jN7Cpgj3Pu42auP83MisysSO9KTSxeuIe+u8+ViEhTRHJGPwy42sy2AgsJDtn8EWhnZnXvuM0GdjS0snPuGedcgXOuoG4+RUkMnreNjPROpKZm+V2KiDRBs4PeOfdr51y2cy4HmASscM5NAVYC14eeNhV4LeIqJa4EvBJdiBVJILHoo58J/NLMNhEcs38uBvsQH3leicbnRRJIVG5q5px7G3g79PlmYHA0tivxp7r6MJWVZeq4EUkgemesnBTP2wagoRuRBKKgl5Pieeq4EUk0Cno5KXqzlEjiUdDLSQl4JbRpcxZpaaf5XYqINJGCXk6KF9DNzEQSjYJeTop66EUSj4JemqymppyKip1kZeX4XYqInAQFvTSZV74d0IVYkUSjoJcmC/fQq7VSJKEo6KXJ6u5a2VZj9CIJRUEvTRbwSkhLO520tHZ+lyIiJ0FBL01WdzOz4ERiIpIoFPTSZOqhF0lMCnppktraKsordqiHXiQBKeilScrLd+BcjTpuRBKQgl6a5LubmeX4W4iInDQFvTRJIHx7Yg3diCQaBb00iRcoITW1LenpHfwuRUROkoJemiSg1kqRhKWglybRhOAiiUtBL41yrgbPK6WtOm5EEpKCXhpVXr4L5yp1Ri+SoBT00ijP2wqgN0uJJCgFvTQqoAnBRRKagl4a5XklpKSkk5HxPb9LEZFmUNBLo7xAXWulflxEEpH+50qjPG+b7nEjksAU9HJCzrnwm6VEJDEp6OWEKiv3UFtbrguxIglMQS8nFAjoZmYiiU5BLycUvj2xeuhFElazg97MupnZSjPbYGafmdmM0PKzzGypmX0Z+nhm9MqVlhbwSjBLIyOji9+liEgzRXJGXw3c6ZzrDQwBfm5mvYG7geXOuV7A8tBjSVCeV0JmZldSUtL8LkVEmqnZQe+c2+mcWxP6/FtgI9AVGAfMCz1tHnBNpEWKfzxPE4KLJLqojNGbWQ7QH/gI6OSc2xn60i6g03HWmWZmRWZWVFZWFo0yJMqccwQCJbrHjUiCizjozexUYAnwC+fcN0d+zTnnANfQes65Z5xzBc65go4dO0ZahsRAVdU+amoOqeNGJMFFFPRm1oZgyM93zr0cWrzbzDqHvt4Z2BNZieIXTQgukhwi6box4Dlgo3Pu8SO+9DowNfT5VOC15pcnflIPvUhyiKSVYhjwQ+BTMysOLbsHeBR40cxuBkqAiZGVKH7xvG2AkZXV1e9SRCQCzQ5659x7wPFmih7V3O1K/Ai2VnYhJSXD71JEJAJ6Z6wcl25mJpIcFPRyXOqhF0kOCnppUFXVQaqq9quHXiQJKOilQZ7miRVJGgp6aVCw40atlSLJQEEvDQp4dT30mkJQJNEp6KVBXqCEjPROpKZm+V2KiERIQS8NCni6mZlIslDQS4M89dCLJA0FvRyjpiZAZWWZOm5EkoSCXo4RqOu40dCNSFJQ0MsxvIA6bkSSiYJejuF5WwG9WUokWSjo5RgBr4Q2bc4iLe00v0sRkShQ0MsxvIBuZiaSTBT0cgzP26YLsSJJREEv9dTUVFBesZMszRMrkjQU9FJPefl2wNFWHTciSUNBL/V8dzMzDd2IJAsFvdRT10PfVmP0IklDQS/1BLwS0tJOJy2tnd+liEiUKOilnrqbmZmZ36WISJQo6KUeTQguknwU9BJWW1tFefkO3eNGJMko6CWsvHwHztXozVIiSUZBL2FeqLWyrd4sJZJUFPQSph56keSkoJcwz9tGampb0tM7+F2KiESRgl7CvIBaK0WSkYJewgJeiTpuRJJQTILezMaa2T/NbJOZ3R2LfUh0OVeD521XD71IEop60JtZKvAUcAXQG5hsZr2jvR+JrvLyXThXqQuxIkkoFmf0g4FNzrnNzrlKYCEwLgb7kSiqmydWPfQiySctBtvsCmw/4nEpcGEM9sNXX73Etu3PxWLTrU519beAJgQXSUaxCPomMbNpwDSA7t2bdwGwTZt2nNK2ZzTLatUyM7uQkdHZ7zJEJMpiEfQ7gG5HPM4OLavHOfcM8AxAQUGBa86OOnYcTceOo5uzqohIqxGLMfpCoJeZ9TCzdGAS8HoM9iMiIk0Q9TN651y1mU0H3gJSgb845z6L9n5ERKRpYjJG75x7A3gjFtsWEZGTo3fGiogkOQW9iEiSU9CLiCQ5Bb2ISJJT0IuIJDlzrlnvVYpuEWZlQEkzV+8AfB3FcuJNMh+fji1xJfPxJdKxneOc69jYk+Ii6CNhZkXOuQK/64iVZD4+HVviSubjS8Zj09CNiEiSU9CLiCS5ZAj6Z/wuIMaS+fj8mL0EAAAC1UlEQVR0bIkrmY8v6Y4t4cfoRUTkxJLhjF5ERE4goYM+WSchN7NuZrbSzDaY2WdmNsPvmqLNzFLNbK2Z/c3vWqLNzNqZ2WIz+9zMNprZUL9rihYzuyP0M7nezBaYWabfNUXCzP5iZnvMbP0Ry84ys6Vm9mXo45l+1hgNCRv0ST4JeTVwp3OuNzAE+HkSHVudGcBGv4uIkT8CbzrnzgfySZLjNLOuwO1AgXMul+BtyCf5W1XE5gJjj1p2N7DcOdcLWB56nNASNuhJ4knInXM7nXNrQp9/SzAouvpbVfSYWTZwJfCs37VEm5mdAVwCPAfgnKt0zh3wt6qoSgOyzCwNaAt85XM9EXHOvQvsO2rxOGBe6PN5wDUtWlQMJHLQNzQJedKEYR0zywH6Ax/5W0lUPQncBdT6XUgM9ADKgDmhoalnzewUv4uKBufcDuAPwDZgJ3DQOfd3f6uKiU7OuZ2hz3cBnfwsJhoSOeiTnpmdCiwBfuGc+8bveqLBzK4C9jjnPva7lhhJAwYA/+Oc6w8cJgn+9AcIjVWPI/jLrAtwipnd6G9VseWCbYkJ35qYyEHfpEnIE5WZtSEY8vOdcy/7XU8UDQOuNrOtBIfbRprZC/6WFFWlQKlzru4vsMUEgz8ZXAZscc6VOeeqgJeBi3yuKRZ2m1lngNDHPT7XE7FEDvqknYTczIzgGO9G59zjftcTTc65Xzvnsp1zOQRfsxXOuaQ5K3TO7QK2m9l5oUWjgA0+lhRN24AhZtY29DM6iiS50HyU14Gpoc+nAq/5WEtUxGTO2JaQ5JOQDwN+CHxqZsWhZfeE5uKV+HcbMD90ArIZ+JHP9USFc+4jM1sMrCHYGbaWBH8XqZktAIYDHcysFHgQeBR40cxuJnhX3Yn+VRgdemesiEiSS+ShGxERaQIFvYhIklPQi4gkOQW9iEiSU9CLiCQ5Bb2ISJJT0IuIJDkFvYhIkvv/pmOzr+sBA50AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11273d550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_method_results(S, train_by_value_iteration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "measuring with 10 seeds took 55.24452495574951\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XmYVNW18P/v6gYamaeWqYFmVIEegGaSGJlUkpigETFcXgWFl+jVK8k1KkYTzdX40yROGGOc0TxEjRPXe+NrVCJBDXpB4SIyIyjN2IDMdENXrd8fdaqoee6hqtbnefrpOvsMtc851at3rbPP2aKqGGOMyV55DV0BY4wxdcsCvTHGZDkL9MYYk+Us0BtjTJazQG+MMVnOAr0xxmQ5C/TGGJPlLNAbY0yWs0BvjDFZrklDVwCgU6dOWlxc3NDVMMaYjPLpp5/uU9XCWMs1ikBfXFzMihUrGroaxhiTUUTkq3iWs9SNMcZkOQv0xhiT5SzQG2NMlrNAb4wxWc4CvTHGZLmYgV5EeojI+yKyVkS+EJG5TnkHEXlXRDY5v9s75SIi80Vks4isFpGhdb0TxhhjIounRV8L3KSqA4FRwPUiMhCYByxW1f7AYmca4DtAf+dnDvB42mttjDEmbjH70avqLmCX8/qIiKwDugOTgbHOYs8DS4BbnfIX1DNG4cci0k5EujrbyWjV1bvYuesVUEVx0aJFH06c2I7bXU1t7VG6drmUtm3L2bnzVfYfWErzgi6cOnWQguZdA7bTof0Y2rcfwdGjG6itPYLLdYyDhz4DlOPHt9GiRW+OH99Kixa9k6pns6btKSqawb5973L4yBfOtooBAaB5825073ZFagfDGJMxErphSkSKgSHAJ0Bnv+C9G+jsvO4ObPdbrdIpCwj0IjIHT4ufnj17JljthrF79yK2bn0k4vwjRz5n6JCFrFt/a5i54vxWvvlmGRXD/sIn//NdAF9gD08ilEfiGQO4Y8fzWbf+Dk6d2h+0Lc/8MwsvomnTdglu2xiTieIO9CLSCngN+ImqHhY5HYBUVUUkoVHGVfVJ4EmAioqKjBih3K210ee7qtEwy4wY/l+0bj0QgFWrruZU7aHA9dy1dOlyCUeOfMGxY5vIz2+Jy3WM4uIb6NvnpwnVcc+ev7Lmixtxu0+hepL8/Fa4XEdp2XIAo0b+P3bseIn1G27H5Tphgd6YHBFXrxsRaYonyC9U1ded4j0i0tWZ3xXY65TvAHr4rV7klGU+dUefL4KGW0byAl6ruoK260L8ToVIvud3Ep2iRDz/uxU3qm7y8poGzM/LKwDA7a5JeNvGmMwUT68bAZ4B1qnqg36z3gRmOK9nAP/pV36V0/tmFHAoG/Lz4Ame0VMpAoQG+pAgroFfYBQ3OMHdtwwgkkyg96yrWotqLXnSLGB+Xr430J9MeNvGmMwUT+pmDHAl8LmIrHLKfg7cB/xFRGYBXwFTnXlvAd8FNgPHgavTWuOGpG4kXIvcR8LOCwzYghK4jKobCfgH4rxOKdC7UHUh1qI3JufF0+vmQyI3YyeEWV6B61OsV6OkKJ4vQeEDvRA+dSNBrfXgfwaqrpBlPNvLJ1G+1I37lKdFnxfUordAb0zOsTtjE6Fu/C9ChxBveidYYOpGNfjas6Y9dePWU553tkBvTM6zQJ8ATxCPdsgk7AXb4NRN8DcCT4s+XakbT4veG8g919FPy8+zHL0xucYCfSKcHH1kkXL0caRuCNeiTyJ1k+e06J1AH9yiF2faWvTG5I5GMcJUY7N7z3/R4oxetGlTGlB+OkcfXqQcvX/L3HMxNzR1E/jPIPn/v74WvSt8oPe26LdXPs++fX8H4IwWvejQfjTr199B69aDo26/Zcu+IPkcO7ox7PyDh1bQrm1F0vU3cPDQctq1HV5n23e5jnP8xDZat/Lc25GXfwZNm7SmpmZv2OVrXcdokt8ybe9f358Rl+s4+fktqDm5F7e7hjOa94i+guTRqeM49h/4B+r23BdT6zpCkyZtY3exTkKbNmUUFU1P+3b9WaAP44svfgLAhPFbAmfEzNHH0b2SPAjToidgu6kEem+L3pOayQtK3RQUdKFN61KqT1RSfaKSWtdRamsP86Uzv7pmF03yW4Xddq3rCLt2HwEgP78VTZu0CZh/qvYQLtcxTp7cHzLPxOdU7WFcrqOcPHmgzo5hdc1OAE6erALE+R3+nFbX7AbcYeclo9Z1hNraI1RX76SgWcyhTlNWc3IfqifJy2uO210NwImCr6Peo1Jds5Pdu99AtZaCgi64XNXU1h4EoKCga1APudTl57dI6/bCsUCfgNg5eiL0uvG/YSo/zAVbd9jUTTJCcvTBLfr8Mxg+/A3f9PbtC9i46W7fdJ/eP6FHjxmEs+2rJ9iy5TcA9Oo1h97FgZ2rNm66h+3bn6NXz9n07v1vSe9DLvvyy0fYum0+Rd3/hX79wj1KI3WL/94XgLLSp8jLP4NPPpkEQHGvaykuvi5g2Y8+Oo/qmp1hz3cytm37I1u+/C1nnvkdBg96KOXtxfLZZ9P55uDHFBZewJ49/wXAiOFv0qxZx4jrLPlHGS7XUQBGjfwbe/e+7XusyZhz/5HS32dDsRx9IlSjplVU3RH62Pu36EPTO6rukPROsk636MOnbkJq5qRyIk0HzmsWYzmJ6z1NZL47mVP4DMT/XgW+VJ53OoRI5HlJiPqNuC44xzGwIRX92PrXUSQvqM6ZGTIzs9YNJPadse6w3Svj60cflN5JUl5Qiz74ztiQ5RMK9DGCgsP7rcIko/4CYV5es7jPaboCPfXcGg7fVTlWHfIDXwf8/dbzP6o0sUCfiFi9bpTY3SslH+8TJP1XDLg5KqXUTaot+sjL+7f+8tP1h28aTF5eQdyBPl3nu77THr6/vQQ6O0jQt+tUGl6NRebvQT2KlaP3PEgsRupGwnfBDEzdJN9qCM3RN422eMgfd7Q/xHiDgskMoYE+8j/5tKVu6vEbi4c3dROYjokmJNDXQxqtrmX+HtSnGDl6TwfMGKkbQlM3nmXyApZJlve9XEm26ONd1gJ95vMEer/rLvnZnLoJSsdEWydgfl6917kuWKBPQKwcvao7ZupGwqZuqLvUTcwcfeB8DVO308taoK97zvEPudci/fLymgUEwGiflfRdjG38qZvT3649F2KtRZ9rYt4ZqzFTN0RI3QRe6U8ldeM8vdLpR59o6ibeZa1nTeYLfjxGvD2uUnrPhkrdSOKpG9/vLAiTmb8HabC98k9U7Vscc7mYOXrVsC3i0NRNuLtnA5dJljdHv2//P4DQG6aCpTN1U/9/xNmo/o5hcIMiXOrGezNPprfoQ9Ix0dYhsKdOJvabD2b94ICNG+8CwtwJG0xjtbbdIXe9QtA6kkesu2dT6UOdn9+KLl0u4cSJSvLzB9K6TfRHGrRoUUynThM5dmwjbvdJOnUcH3HZli370anjeBQ3LVsOCJlfXHwdJ0/up5sNPJ60oqL/w7Fjm+nVa06dvcfQIQs58M0y33SvXtdx9Oh6WoU5p/363sqOHX8Oe76TU79tS9/fVSKdHXzzQ9fNVDEDvYg8C1wM7FXVwU7Zy8BZziLtgIOqWu4MHr4O2ODM+1hVr013pRtK7DtjI6VuTos0cEnwlf5kiQiDBj7gmz5ydH3U5fPzz6Cs9Im4tt2kSSvKyp6KOL9p0/YMGvRAxPkmtiZNWtf5MWzffhTt24/yTffr+7OIy3bqNI5Oncal7b3rO9+dzCO/gy/gZkPqJp4W/QLg98AL3gJV9TXZROQBwH+06y2qWp6uCjYqMXL0qu6oFzPB+7VQQx9sFvD1MH1f3y2dYhqV+m4dh03dxFhFcjB1o6pLnZZ6CGc82alA5O/7WSSeFn241E0A3wc9MH2TDa0GY2Kp74ZHuNRNPGt5ZE/qJtU9OA/Yo6qb/Mp6i8hKEfmHiJyX4vYbF9Wo+T1VDX+h1Y+3lRCcvsmGLlzGNDaWuvFI9WLsNOBFv+ldQE9V3S8iw4BFIjJIVQ8Hrygic4A5AD179kyxGvUj2Wfd+PN+hQxO3WTD10NjGh2nYZZY6iaoe2UW/G0m/a9KPP34fgi87C1T1RpV3e+8/hTYAoS9XK+qT6pqhapWFBbW/XOp0yLms25ip25OfyMIXi7zWw3GND7JDMuZF/A7G75tp7IHE4H1qlrpLRCRQnH+/YlIH6A/+Ma0yHixRpjyPOsmxh2N3huawqRuTq+bvrsiT18crvs7LY1prJJL3QQG/EwWcw9E5EVgGXCWiFSKyCxn1o8ITNsAfBtYLSKrgFeBa1X1QDor3KBijTAFaEhLPZA33xfa6ybzP0zGNFbJpW4Sz+83VvH0upkWoXxmmLLXgNdSr1bDcLlqWPKPgb7pPXv+SufO30NVWb16Dvv2e8ZYbdas0Df8mr+amt1s3PirqO/h/dD8c9nYwHLyaNLEM4RfvjM+Z6zn1MTD+2HNjzA8oDG5ILHHigSnbjI/R293xvo5fnxzwPSaL26kc+fvAW5fkAcYNPABVv3v1b70S4sWfamp2Y3LdYy8vDPo1GkibVqX0KHjeRw7tilgm96h/VyuE7RvN4pa11FatTqH9u1H067dcHbvfoOiohl89dXjdOt2ecr71LJFP/r2uYkuXS5NeVvGZKwEgnU2pm4s0PuJNDJScJqlQ4cxnH3WPaxbfxtNmrRh9Kh3WL/hF+zY8Wc6tB/NOef8f75l27YpC1jX+3CoNm1KGTp0Ych7FRf/KwB9o9ytmAgR8W3TmFyVSBdJb1//4N43mSzz9yCNgp/md1rohUzvQ57c7lPeEmcb0VsO9nhfY+pfMhdjIT9oOnNZoPcTOY8X2jfeG7BVnccB+1oB0b8k2RB8xjSARIJ1SEs+88Nk5u9BGkW6qzVcj8nTgd7pZeO9McNa9MY0OgmlbkJumMr8MJn5e5BGqrWR5oSUhA7EEF+gFxuww5j6l0CvG9/z6C11k50iP6cmcurG6/R//+ipm9P9ee0GJmMaJd/wg5a6yUqRbnYKd7drsi16Y0zjZqmbbBexRR+5141XvBdjjTGN2+nUTfbcMGWB3s+xY5GGEowd6L3PlIn1ocjPPwOAgoLOiVfQmAyX79z93axph3p5P+/d5rHGTvYneZ7GmrfR5v2dyXeXW/PTz/ET28KWe1M3hYWTKO7lGRkxpM+9820gVou+TZsSzjnnfs4svCi1yhqTgTp1HM/ZZ/2aLl0uqZf369Hjapo2aUth4YW0anU2bndNzHV69pxN8+bd6djBM5xGs2YdGDDgTlq3HlTX1a0zFuj9qO/mp5A5ALRrV0GbNiUASF5+0BLusOXhdOs6JflKGpPBRITu3X9Ub+/XvKCL787wpk1L4lqnTevBtGk9OKCsR9FVaa9bfbLUjZ9Y3Sv9h0ELbrlrnC16Y4ypbxbo/bidu1yDne52eTrQ54Xk4r2BPvMv3BhjsosFej/uGKkb/5suQlv0rrDlxhjT0CzQ+4mdujl9uEJa7hpfrxtjjKlv8Yww9ayI7BWRNX5ld4nIDhFZ5fx812/ebSKyWUQ2iEhGdS1xuyOlbrzdK/1b9EEXY70t+gRGsjHGmPoQT4t+ATApTPlDqlru/LwFICID8QwxOMhZ5w+SQU3cWL1uoqZufDl6+5JkjGlcYkYlVV0KxDvu62TgJVWtUdWtwGZgRAr1q1dujR7o/XvdhBw67wVbC/TGmEYmlah0g4isdlI77Z2y7sB2v2UqnbKMoBECfbheN8HPrm/WrBMA+Xln1EndjDEmWckG+seBvkA5sAt4INENiMgcEVkhIiuqqkIH2m4I6g68GNuu3UjvHM+vKI86LS6+joEDH6Bjx7F1UzljjElSUoFeVfeoqks9Td2nOJ2e2QH08Fu0yCkLt40nVbVCVSsKCwuTqUbahaRugh5yFm3wgqZN29O1yyXk59vAIsaYxiWpQC8iXf0mLwW8PXLeBH4kIgUi0hvoD/xPalWsP8H96L0XWMOlbowxJlPEvLtHRF4ExgKdRKQSuBMYKyLleHIa24AfA6jqFyLyF2AtUAtcr76x9ho/Dboz9nSA96Zu6rc+xhiTDjEDvapOC1P8TJTlfw38OpVKNZSQO2N9/6NCb5gyxphMYZHLT3CvG0vdGGOygQV6P+6gXjehqRsL9MaYzGOB3k9oP/rgXjcW6I0xmccCvZ+QXjdOjr7m5D6nxAK9MSbzWKD3E67XzYFvlvHZZ86IOGFSNyFDChpjTCNjD0/3E/o8ejeHD630TQWnbr415p8hg4QbY0xjk/OB/vQjiEOfR6/qCgz+QQ8sKyjoXKd1M8aYdLDUDdECvTtoeEHL0RtjMo8Fer9AHzrLHTAYifW6McZkopwP9P6pm5B5uAMHI7F+9MaYDJTzgT5ai96To6/xTVuL3hiTiSzQRw307qBHF1ugN8ZkHgv00XL0IakbO1zGmMyTc90ra07uw1V7BFU3LVv2xeWqibhscK8bS90YYzJRzgX6Dz8c6Xs9ftxm1q67OeKyIf3oLdAbYzJQjucilG+++TjKfDcBDzazXjfGmAwUM9CLyLMisldE1viV/VZE1ovIahF5Q0TaOeXFInJCRFY5P3+sy8qnSoPGhA0736/7paVujDGZKJ4W/QJgUlDZu8BgVS0FNgK3+c3boqrlzs+16almXXETq9eNBsy3QG+MyTwxA72qLgUOBJW9o6efF/AxUFQHdasHSvReN66AFr31ujHGZKJ0RK5rgP/nN91bRFaKyD9E5LxIK4nIHBFZISIrqqqq0lCNxKlq9DtjNbDFb6kbY0wmSinQi8jtQC2w0CnaBfRU1SHAvwN/FpE24dZV1SdVtUJVKwoLC1OpRgpitejVN26sMcZkqqQDvYjMBC4GpqvTLFbVGlXd77z+FNgCDEhDPetEcIs9/DIuvylr0RtjMk9SgV5EJgG3AD9Q1eN+5YUiku+87gP0B75MR0XrRqwWfWCgF8vRG2MyUMwbpkTkRWAs0ElEKoE78fSyKQDeFU/f8o+dHjbfBv5DRE7h6dJyraoeCLvhRkGJkqIH4NChT/2mrEVvjMk8MQO9qk4LU/xMhGVfA15LtVL1JVzqJi+vGW73Sdq3G8U3B4NvprJAb4zJPDmeiwhN3bQ4ozcTxm+hY6dxIUtb6sYYk4lyOnJFuxjrXGowxpiMl9OB3pOjjxDowx0ae9aNMSYD5XSg12i9bsKkacIGf2OMaeRyO3IlnLqxFr0xJvPkdqCP8sCysI87sNSNMSYD5XSg939McXCPmnAtenvWjTEmE+V0oA9s0QcdirBdKS3QG2MyT04Hev8eNyEteut1Y4zJEjkd6AMeQeykarxPq2zZsh9NmrQNWNp63RhjMlGORy7/HH1T4HTevk2bUs7/9mcBS9tNVMaYTJTTgd7/YmxenjfQ10ZaHMvRG2MyUU4H+sDUjef5btECvbXojTGZKKcDvf/F2Dxp5pS5Ii1uDzUzxmSkHI9cfi36PG+gj5a6yfHDZYzJSHFFLhF5VkT2isgav7IOIvKuiGxyfrd3ykVE5ovIZhFZLSJD66ryqfIfD/Z0jj7yGLGWujHGZKJ4m6gLgElBZfOAxaraH1jsTAN8B88Qgv2BOcDjqVezjvinbuJo0VvqxhiTieKKXKq6FAgeEnAy8Lzz+nngEr/yF9TjY6CdiHRNR2VTFdpa978Y623RR87Rh79b1hhjGreYQwlG0VlVdzmvdwOdndfdge1+y1U6ZbuoR0ePbmT/gaUIeXToMIZWrc6isvKFgGUOHPjQ9zovnl43WOrGGJN5Ugn0PqqqIhJjmO1AIjIHT2qHnj17pqMaAZavuAS3u8Y3PWH8FjZuujtgmU2b7/W9HjDgTr788kGKiq4MWKaoaAaVlc9765z2ehpjTF1LJRexx5uScX7vdcp3AD38lityygKo6pOqWqGqFYWFhSlUIzz/IB9Lp47jadVqAKWlf6RDhzEB8/r3+7nvtV2MNcZkolQC/ZvADOf1DOA//cqvcnrfjAIO+aV4GiVv18qw8wJa8RbojTGZJ67UjYi8CIwFOolIJXAncB/wFxGZBXwFTHUWfwv4LrAZOA5cneY6p11+XkGUuaf/F1rqxhiTieIK9Ko6LcKsCWGWVeD6VCpV3/KiBHr/4G6pG2NMJrL+gkRP3QSyw2WMyTwWuYg/JWM3TBljMpFFrgRY6sYYk4ks0CfEDpcxJvOk5YapxmTjpnsoaBbaL9/tPhlxnXjveLVeN8aYTJR1gX779ufClh869FnYcoBu3aZGnGeMMZkud3IREfLrxcXX06rVWfVcGWOMqT85E+glwniv9qAyY0y2y51AH6nHjHWZNMZkudyJchECuvWNN8Zku5yJchJhVy11Y4zJdjkT6InQNdK6TBpjsl1WBXrVyGOfRLoYG6k3jjHGZIusCvT+Y8DGyx5rYIzJdlkV6EMH/z5t+YrLwpZHbOkbY0yWyLI7YyMHetVT4WfE0aI/++x7fYOHG2NMpkk6eonIWcDLfkV9gF8C7YD/C1Q55T9X1beSrmECouXoI4nUG8df925XJFMdY4xpFJIO9Kq6ASgHEE+iewfwBp6hAx9S1d+lpYYJidyij8T60Rtjsl26otwEYIuqfpWm7SUlWo4+ErsYa4zJdukK9D8CXvSbvkFEVovIsyLSPk3vEYfEA32WXY82xpgQKUc5EWkG/AB4xSl6HOiLJ62zC3ggwnpzRGSFiKyoqqoKt0jCksrRW+rGGJPl0hHlvgN8pqp7AFR1j6q61JNHeQoYEW4lVX1SVStUtaKwMHSgkORY6sYYY4KlI9BPwy9tIyJd/eZdCqxJw3vEJZkcPdaP3hiT5VLqHC4iLYELgB/7Ff9GRMrx3Ka6LWheHbMWvTHGBEsp0KvqMaBjUNmVKdUoBR9+dG7C61igN8Zku6y6EqnqSnwle3qlMSbLZVWgT4Y9j94Yk+1yPtBbi94Yk+1yPtCr1jZ0FYwxpk5ZoHdboDfGZLecD/TuSI8vNsaYLJHzgd5a9MaYbGeB3lr0xpgsl/OB3lI3xphsl/OBvnlB19gLGWNMBsvqgVDHjV3L9soX2Lz5voDyimGv0KrVIA4fWU37dsMbqHbGGFM/sjrQ5+UV0K5d6FOS27YdCmBB3hiTE7I+dSP2GGJjTI7L+kBvjDG5LvsDvQ0VaIzJcVkfBS11Y4zJdSlfjBWRbcARwAXUqmqFiHQAXgaK8YwyNVVVv0n1vZKsYcO8rTHGNBLpatGPU9VyVa1wpucBi1W1P7DYmW4YlroxxuS4uoqCk4HnndfPA5fU0fv4qGrYckvdGGNyXToCvQLviMinIjLHKeusqruc17uBzsEricgcEVkhIiuqqqrSUI3EBwY3xphckI4bpr6lqjtE5EzgXRFZ7z9TVVVEQprbqvok8CRARUVF+OZ4AlQjBfrAFv2A/r9M9a2MMSajpNyiV9Udzu+9wBvACGCPiHQFcH7vTfV94qhJ2FIJytE3a9ax7qtijDGNSEqBXkRaikhr72vgQmAN8CYww1lsBvCfqbxPPCLl6EN73VjO3hiTW1JN3XQG3hDPANtNgD+r6tsishz4i4jMAr4Cpqb4PnGIL3Vjg4EbY3JNSoFeVb8EysKU7wcmpLLtxOsSPtBLSGC3QG+MyS1Z1Mk8vha9ZNMuG2NMHLIm6sWdo7cGvTEmx2RNoI/Uog/udWM3UBljck3WBPp4+9Fbk94Yk2uyJtCHtui9AT04pWOB3hiTW7Im0Ifm6CMEdOteaYzJMVkT6INb9MG5eS/V2vqojDHGNBpZE+hDc/ThUzdu98l6qY8xxjQWWRPo483Fq/tU3VfFGGMakawJ9MEt+tA7Yj2sRW+MyTVZE+gj97oJWkot0BtjckvWBPrIOfpA1qI3xuSarAn0wTl6b+omuNulWqA3xuSYrAn0e/e+HVRiLXpjjIEsCfTV1TvZ8uXvAsr69PkpAAUFhQHlnQon1lu9jDGmMUjHmLENzuWq9r0uGfwHzjzzIt90fn4LJozfgqo74k1UxhiTzZKOfCLSQ0TeF5G1IvKFiMx1yu8SkR0issr5+W76qhtPvfIjlFuQN8bkplRa9LXATar6mTNu7Kci8q4z7yFV/V2UdetMpEBvjDG5KulAr6q7gF3O6yMisg7onq6KJUskK7JRxhiTNmnJZ4hIMTAE+MQpukFEVovIsyLSPsI6c0RkhYisqKqqSkc1nO1ai94YY/ylHOhFpBXwGvATVT0MPA70BcrxtPgfCLeeqj6pqhWqWlFYWBhukSTrYy16Y4zxl1KgF5GmeIL8QlV9HUBV96iqSz23qj4FjEi9mrGcvilK8qxFb4wx/lLpdSPAM8A6VX3Qr7yr32KXAmuSr168/AK9teiNMSZAKlFxDHAl8LmIrHLKfg5ME5FyPNF3G/DjlGoYB1WX77Xl6I0xJlAqvW4+JPxzBt5KvjrJ1uX0A82sRW+MMYGy5C4i/0BvLXpjjPGXFYHeP3WTZy16Y4wJkCWB3lr0xhgTSVYE+sDUjbXojTHGX1YEemvRG2NMZI22+Xvq1CkqKyuprq6Ouazb3YQO7Z8AYMuWKkQO1HX1jDGmwTRv3pyioqK4l2+0gb6yspLWrVtTXFzsGxYwktraoxw/7mnJt2p1Nnl5jXa3jDEmJarK/v37qaysjHudRpu6qa6upmPHjjGDfLBElzfGmEwiInTs2DGubIdXow30EH/Qrq09XMc1McaYxiPRBm2jDvTxOnlyv9+UteiNMcZfVgT6QPUX6D/44AMGDRpEeXk5J06cSNt277333oDpc889Ny3bXbBgATt37kx4vT/+8Y+88MILSb3fDTfckPB69WHbtm38+c9/bpD3bqjj8vDDD3P8+HHfdKtWrVLaXqLrB3+u/X366aeUlJTQr18/brzxRlQ14rIAv//97+nXrx8iwr59+3zl69evZ/To0RQUFPC73wUOcnfNNddw5plnMnjw4IDy9evXU15ezpAhQ9iyZQsHDx5kypQpnH322ZxzzjksW7Ys7n2cOXMmr776KgBbt25l5MiR9OvXjyuuuIKTJ08C8PXXXzNu3DiGDBlCaWkpb71V90+NybpAX9c5+pMnT3Ls2DEAFi5cyG233caqVas444wz0vYewX8Q//znP9NQFEFCAAATIklEQVSy3WiB3uVyhS0HuPbaa7nqqqvSUofGoqECfW1tbb2/p1dwoK9v0QL9ddddx1NPPcWmTZvYtGkTb7/9dtRtjRkzhvfee49evXoFlHfo0IH58+fzs5/9LGSdmTNnht3uokWLmDJlCitXrqRv377MnTuXSZMmsX79ev73f/+Xc845J849DHTrrbfy05/+lM2bN9O+fXueeeYZAO655x6mTp3KypUreemll/jXf/1XAI4dO8apU6eSeq9YMiLQb9x4N59+9i8Rf9avv933E205/5+NG+9OqA7r1q3jpptu4qyzzmLjxo08/fTT/OUvf+EXv/gF06dPZ8mSJVx88cW+5W+44QYWLFgAQHFxMXfeeSdDhw6lpKSE9evXA3D06FGuvvpqSkpKKC0t5bXXXmPevHmcOHGC8vJypk+fDpxuOakqN998M4MHD6akpISXX34ZgCVLljB27FhfK2T69OkhLaJXX32VFStWMH36dN83kOLiYm699VaGDh3KK6+8wlNPPcXw4cMpKyvjsssu8wWFu+66y9c6Gjt2LLfeeisjRoxgwIABfPDBB1GP2/bt2xk7diz9+/fnV7/6FQC//OUvefjhh33L3H777TzyyCNh11+yZAnnn38+kydPpk+fPsybN4+FCxcyYsQISkpK2LJlC+AJ3OPHj6e0tJQJEybw9ddfA54/7htvvJFzzz2XPn36+Fpb8+bN44MPPqC8vJyHHnoIl8vFzTffzPDhwyktLeWJJ56Iul/3338/JSUllJWVMW/ePICIx2/mzJlce+21jBw5kltuuSVgO5HqHc7MmTO57rrrGDVqFH369GHJkiVcc801nHPOOcycOdO33DvvvMPo0aMZOnQol19+OUePHmX+/Pns3LmTcePGMW7cuIBjX1ZWxqhRo9izZ0/UOm3dupXRo0dTUlLCHXfcEbGel1xyCcOGDWPQoEE8+eSTvuMd/Ln22rVrF4cPH2bUqFGICFdddRWLFi2itraW4cOHs2TJEgBuu+02br/9dgCGDBlCcXFxyHufeeaZDB8+nKZNm4bM+/a3v02HDh0Cyt566y0efvhhHn/8ccaNG8ehQ4dYunQps2bNAqBZs2a0a9cOgM2bNzNx4kTKysoYOnQoW7ZsQVW54YYbOOuss5g4cSJ79+4FPH+rf//735kyZQoAM2bMYNGiRYCnMXr4sOe64qFDh+jWrRsAGzduZMCAAfzsZz9j3bp1EY9vUlS1wX+GDRumwdauXet7vWHDf+iKT6dF/Pnkk+/7fqIt5/+zYcN/hLxnsKNHj+qzzz6rY8aM0TFjxujTTz+thw8f9s2fMWOGvvLKK6qq+v777+v3vvc937zrr79en3vuOVVV7dWrl86fP19VVR977DGdNWuWqqrecsstOnfuXN86Bw4cUFXVli1bBtTDO/3qq6/qxIkTtba2Vnfv3q09evTQnTt36vvvv69t2rTR7du3q8vl0lGjRukHH3wQsj/nn3++Ll++3Dfdq1cvvf/++33T+/bt872+/fbbfXW+88479be//a1vG//+7/+uqqp//etfdcKECRGP33PPPaddunTRffv26fHjx3XQoEG6fPly3bp1qw4ZMkRVVV0ul/bp0yfgvf29//772rZtW925c6dWV1drt27d9Je//KWqqj788MO+43fxxRfrggULVFX1mWee0cmTJ6uq5xxNmTJFXS6XfvHFF9q3b1/fdv3P1xNPPKF33323qqpWV1frsGHD9Msvvwxbp7feektHjx6tx44dU1XV/fv3Rz1+M2bM0O9973taW1vrOy7XX3991HqHM2PGDL3iiivU7XbrokWLtHXr1rp69Wp1uVw6dOhQXblypVZVVel5552nR48eVVXV++67T3/1q1+pqud8V1VV+bYH6JtvvqmqqjfffLNv/yPV6fvf/74+//zzqqr6+9//PuRz6uU9Ht5z7j0ukZZfvnx5wOdo6dKlvnOzZs0aPfvss/Xdd9/V8vJyrampCVg3eJ+8/D+z/rZu3aqDBg2KuOzKlSt1+PDhOmPGDC0vL9dZs2b5juWIESP09ddfV1XVEydO6LFjx/S1117z/U3u2LFD27Ztq6+88opWVVX5Pmuqql9//bXvfXfu3KmDBw/W7t27a7t27XTFihW+5Q4fPqxPPfWUnnvuuTpmzBh99tlnfe8fbO3atQqs0DhibEZ0OB8w4BdR5x8+/LnvdZs2JWl7365du1JaWsrTTz/N2WefndK2fvjDHwIwbNgwXn/9dQDee+89XnrpJd8y7duHHV7X58MPP2TatGnk5+fTuXNnzj//fJYvX06bNm0YMWKE7waK8vJytm3bxre+9a2Y9briiit8r9esWcMdd9zBwYMHOXr0KBdddFHMfdm2bVvU7V9wwQV07NjRt96HH37IT37yEzp27MjKlSvZs2cPQ4YM8S0TzvDhw+na1TOeTd++fbnwwgsBKCkp4f333wdg2bJlvuN65ZVXBrScL7nkEvLy8hg4cKCv1RrsnXfeYfXq1b4W/6FDh9i0aRO9e/cOWfa9997j6quvpkWLFgC+VmK043f55ZeTnx9613a0eofz/e9/HxGhpKSEzp07U1Li+bwPGjSIbdu2UVlZydq1axkzZgzgSTWOHj067LaaNWvm+xY6bNgw3n333ah1+uijj3jttdd85bfeemvY7c6fP5833ngD8Hyj27RpU9TzG82gQYO48sorufjii1m2bBnNmjVLajvxqq2t5bPPPuPRRx9l5MiRzJ07l/vuu49bbrmFHTt2cOmllwKeG5YAli5d6vub7NatG+PHj4/5Hi+++CIzZ87kpptuYtmyZVx55ZWsWbOGvLw8WrduzezZs5k9ezbr1q1j1qxZzJ071/cNIFl1FuhFZBLwCJAPPK2q99XVe9WVV199lWeeeYYf/vCH/OhHP2LGjBkhOUGvJk2a4HaffhRDcB/XgoICAPLz8+skT+vdfqLv0bJlS9/rmTNnsmjRIsrKyliwYIHvK3Ok94rnfYKvmXinZ8+ezYIFC9i9ezfXXHNN1G3471teXp5vOi8vL6799F9fI1zkU1UeffTRiP/c4hHt+Pkf51T473vwcamtrSU/P58LLriAF198Mea2mjZt6jsf8X5mYl0DW7JkCe+99x7Lli2jRYsWjB07NmZ/7+7duwfc/FNZWUn37t19059//jnt2rXzpUXqUlFREUVFRYwcORKAKVOmcN99iYeujh07cvDgQWpra2nSpEnAPj3zzDO+awWjR4+murqaffv2ceaZZwKe1Nnzzz/Piy++SFlZGXfddVfK+1UnOXrxPHDmMeA7wEA8o04NrIv3qksXXnghL7/8Mh988AFt27Zl8uTJTJw4MWwrtlevXqxdu5aamhoOHjzI4sWLY27/ggsu4LHHHvNNf/PNN4DnDzDcRZnzzjuPl19+GZfLRVVVFUuXLmXEiPiH5G3dujVHjhyJOP/IkSN07dqVU6dOsXDhwri3G827777LgQMHOHHiBIsWLfK1NC+99FLefvttli9fnlJw9Tr33HN9344WLlzIeeedF3X54GNx0UUX8fjjj/uO+8aNG30X3YNdcMEFPPfcc74c/IEDnkduJHP8Eq13LKNGjeKjjz5i8+bNgOcC38aNG4HY5z9WncaMGRNQHs6hQ4do3749LVq0YP369Xz88ce+eZE+1127dqVNmzZ8/PHHqCovvPACkydPBuD111/nwIEDLF26lH/7t3/j4MGD8R6KpHTp0oUePXqwYcMGABYvXszAgQNp3bo1RUVFvjx7TU0Nx48f59vf/rbvb3LXrl2+b5giwrhx43zfEJ9//nnfPvXs2dMXH9atW0d1dTWFhYVs27aNiRMncskll9CuXTs++ugjXn75Zd832FTU1cXYEcBmVf1SVU8CLwGT6+i96lzHjh2ZO3cuq1at4t577w37FbxHjx5MnTqVwYMHM3XqVIYMGRJzu3fccQfffPMNgwcPpqyszPchmTNnDqWlpSEXrS699FJKS0spKytj/Pjx/OY3v6FLly5x74f3omCk7qB33303I0eOZMyYMSmnqrxGjBjBZZddRmlpKZdddhkVFRWAJ20wbtw4pk6dGvZ4JurRRx/lueeeo7S0lD/96U8RL+56lZaWkp+fT1lZGQ899BCzZ89m4MCBDB06lMGDB/PjH/84Ygt30qRJ/OAHP6CiooLy8nLfhepkjl+i9Y6lsLCQBQsWMG3aNEpLSxk9erTv4v+cOXOYNGlSwMXYROr0yCOP8Nhjj1FSUsKOHTvCrjtp0iRqa2s555xzmDdvHqNGjfLNi/S5BvjDH/7A7Nmz6devH3379uU73/kO+/btY968eTz99NMMGDCAG264gblz5wKe9FBRURGVlZWUlpYye/ZsAHbv3k1RUREPPvgg99xzD0VFRb60x7Rp0xg9ejQbNmygqKjI1wsm3P5Pnz6d0tJSVq1axc9//nMA/vSnPzF//nxKS0s599xz2b17N5deein9+/dn4MCBXHXVVQFpsvvvv58HH3yQfv36sX//ft8F3gceeICnnnqKsrIypk2bxoIFCxAR8vPzuffee1m1ahVz585NOt0VjkT6KpvSRkWmAJNUdbYzfSUwUlXDdh6uqKjQFStWBJStW7cuZrcml6uaEye+xu2u8ZWlM0dv6o7b7fb19unfv39DV8eYjLNu3ToGDhz4qapWxFq2wbpXisgcEVkhIiuqqqqS3Qp5ec3Jz/fkP884I3z+3DQua9eupV+/fkyYMMGCvDH1oK4uxu4AevhNFzllPqr6JPAkeFr0ybxJfn4BLVr0TLaOJk3+9re/hfTA6N27t6/nRbCBAwfy5ZdfBpR9/vnnXHnllQFlBQUFfPLJJ+mtbAIaqk6//vWveeWVVwLKLr/8cl8fcmMSVVepmybARmACngC/HPgXVf0i3PLJpm6MMSZXJZK6qZMWvarWisgNwN/wdK98NlKQj7Ede+ywMcYESbSBXmf96FX1LSDpp/U0b96c/fv3J/VMemOMyVbqDDzivWkrHo32zlhv16nkL9QaY0x2ypqhBJs2bRr29nNjjDGJyYinVxpjjEmeBXpjjMlyFuiNMSbL1Uk/+oQrIVIFfJXCJjoB+2IulT1ybX/B9jlX2D4nppeqFsZaqFEE+lSJyIp4bhrIFrm2v2D7nCtsn+uGpW6MMSbLWaA3xpgsly2B/smGrkA9y7X9BdvnXGH7XAeyIkdvjDEmsmxp0RtjjIkgowO9iEwSkQ0isllE5jV0fdJFRHqIyPsislZEvhCRuU55BxF5V0Q2Ob/bO+UiIvOd47BaRIY27B4kR0TyRWSliPy3M91bRD5x9utlEWnmlBc405ud+cUNWe9UiEg7EXlVRNaLyDoRGZ0D5/mnzud6jYi8KCLNs+1ci8izIrJXRNb4lSV8XkVkhrP8JhGZkWx9MjbQZ8sA5BHUAjep6kBgFHC9s2/zgMWq2h9Y7EyD5xj0d37mAI/Xf5XTYi6wzm/6fuAhVe0HfAPMcspnAd845Q85y2WqR4C3VfVsoAzP/mfteRaR7sCNQIWqDsbzGPMfkX3negEwKagsofMqIh2AO4GReMbhvtP7zyFhqpqRP8Bo4G9+07cBtzV0vepoX/8TuADYAHR1yroCG5zXTwDT/Jb3LZcpP3hGIVsMjAf+GxA8N5E0CT7feMY5GO28buIsJw29D0nsc1tga3Dds/w8dwe2Ax2cc/ffwEXZeK6BYmBNsucVmAY84VcesFwiPxnbouf0B8ar0inLKs5X1SHAJ0BnVd3lzNoNdHZeZ8OxeBi4BXA70x2Bg6pa60z775Nvf535h5zlM01voAp4zklZPS0iLcni86yqO4DfAV8Du/Ccu0/J/nMNiZ/XtJ3vTA70WU9EWgGvAT9R1cP+89TzLz4rukyJyMXAXlX9tKHrUs+aAEOBx1V1CHCM01/ngew6zwBO6mEynn9y3YCWhKY4sl59n9dMDvQxByDPZCLSFE+QX6iqrzvFe0SkqzO/K7DXKc/0YzEG+IGIbANewpO+eQRo54w/DIH75NtfZ35bYH99VjhNKoFKVfWONv4qnsCfrecZYCKwVVWrVPUU8Dqe85/t5xoSP69pO9+ZHOiXA/2dq/XN8FzQebOB65QW4hk78Rlgnao+6DfrTcB75X0Gnty9t/wq5+r9KOCQ31fERk9Vb1PVIlUtxnMe/66q04H3gSnOYsH76z0OU5zlM67Vq6q7ge0icpZTNAFYS5aeZ8fXwCgRaeF8zr37nNXn2pHoef0bcKGItHe+CV3olCWuoS9YpHix47vARmALcHtD1yeN+/UtPF/rVgOrnJ/v4slNLgY2Ae8BHZzlBU8PpC3A53h6NDT4fiS572OB/3Ze9wH+B9gMvAIUOOXNnenNzvw+DV3vFPa3HFjhnOtFQPtsP8/Ar4D1wBrgT0BBtp1r4EU81yBO4fnmNiuZ8wpc4+z7ZuDqZOtjd8YaY0yWy+TUjTHGmDhYoDfGmCxngd4YY7KcBXpjjMlyFuiNMSbLWaA3xpgsZ4HeGGOynAV6Y4zJcv8/4eu7koKBR5kAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11238c4a8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_method_results(S, train_by_monte_carlo_method)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "measuring with 10 seeds took 53.62889814376831\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xt4FPW9x/H3l4SAIHKJgJBAgxVBQ24YEEUQRS6nIIJSKqICiuh5ULGnRxFR0WotWjxqH2oroqX0UUEQi603FKFIbUHAKBhEULAEuVpAuSgk+zt/7GQJt8Bekt1ZPq/nyZOd2bn8Zif57Hd/MztjzjlERCR51Yh3A0REpGop6EVEkpyCXkQkySnoRUSSnIJeRCTJKehFRJKcgl5EJMkp6EVEkpyCXkQkyaXGuwEAp59+usvKyop3M0REfGXZsmXbnXONjzddQgR9VlYWS5cujXczRER8xcy+OpHp1HUjIpLkFPQiIklOQS8ikuQU9CIiSU5BLyKS5I4b9Gb2vJltNbOVFcY1MrN3zGyN97uhN97M7LdmttbMPjGz9lXZeBEROb4TqeinAr0PG3c3MM851xqY5w0D/BfQ2vsZCfw+Ns0UEZFIHfc8eufcQjPLOmz0FUA37/GfgAXAGG/8NBe8P+G/zKyBmTVzzm2KVYP9bvOWv7Jnz9p4N0NEEkTj07tz2mm5VbqOSL8w1bRCeG8GmnqPM4ANFaYr8cYdEfRmNpJg1U/Lli0jbIb/FBffhXP7AYt3U0QkAdSq1TRhgz7EOefMLOw7jDvnJgOTAQoLC0+KO5Q753BuP62ybuPMM++Id3NE5CQR6Vk3W8ysGYD3e6s3fiPQosJ0md44ASAAgFlKnNshIieTSIP+NWCo93goMKfC+Ou9s286AbvUP3+Qc6UAmCXEJYZE5CRx3MQxs5cIHng93cxKgPHABOBlM7sR+AoY5E3+BvATYC2wFxheBW32LefKAFX0IlK9TuSsm8HHeKr7UaZ1wKhoG5WsDga9KnoRqT76Zmw1Oth1o4peRKqPgr4aBVTRi0gcKOirkSp6EYkHBX01cgFV9CJS/RT01UgVvYjEg4K+Gun0ShGJBwV9NQpV9DXUdSMi1UdBX41U0YtIPCjoq5EugSAi8aCgr0aq6EUkHhT01UgVvYjEg4K+GqmiF5F4UNBXo1BFj4JeRKqP+hCq0IEDOyleNYaysj3e8C4ArIaCXkSqjyr6KrR792ds3/4uB/b/h0DgACkpdUhP70bdOq3j3TQROYmooq9C5X3ybdr8kgYNCuPcGhE5Wamir0K6to2IJAIFfRXSWTYikggU9FVI582LSCJQ0FehgCp6EUkACvoqpIpeRBKBgr4KqY9eRBKBgr4KqaIXkUSgoK9CquhFJBEo6KvQwaBXRS8i8aOgr0L6wpSIJAIFfRVSRS8iiUBBX4VU0YtIIlDQVyEXUEUvIvGnoK9CquhFJBFEFfRm9nMz+9TMVprZS2ZW28xamdliM1trZjPMLC1WjfUbnV4pIokg4qA3swzgdqDQOdcOSAGuBh4FnnDOnQXsAG6MRUP9KFjR18DM4t0UETmJRdt1kwqcYsFO6DrAJuBSYJb3/J+A/lGuw7ccAfXPi0jcRZxCzrmNZjYR+DewD5gLLAN2uvLOaSgBMqJupc8s+fAKdu/+HOdKSUk5Jd7NEZGTXMRBb2YNgSuAVsBOYCbQO4z5RwIjAVq2bBlpMxKOc47vvltJ/frn0aBBB06t2ybeTRKRk1w0/QqXAeucc9sAzGw20BloYGapXlWfCWw82szOucnAZIDCwkIXRTsSTACARo26cGar2+LcFhGR6Pro/w10MrM6Fjza2B0oBuYDA71phgJzomuiv5T3WtXQmTYikiAiDnrn3GKCB12XAyu8ZU0GxgD/Y2ZrgXTguRi00zd0SqWIJJqoTglxzo0Hxh82+kugYzTL9TNd30ZEEo2+GRtj+jasiCQaBX2MBVTRi0iCUdDHmCp6EUk0CvoY0xUrRSTRKOhjTBW9iCQaBX2M6awbEUk0CvoYU0UvIolGQR9jquhFJNEo6GNMFb2IJBoFfYzpEggikmgU9DF2sKJX142IJAYFfYw5F7xMsSp6EUkUCvoYU0UvIolGaRQjzgXYvXs1e/auBcBqqKIXkcSgoI+RzVteo7j4F6Hh1JRT49gaEZGDFPQxcuDADgCys5+kVq0zqFv37Di3SEQkSEEfI+WnVZ6efgmpqarmRSRx6GBsjOgbsSKSqBT0MaJvxIpIolLQx4i+ESsiiUpBHyPBir4GZnpJRSSxKJVixLky9c+LSEJS0MeIc6XqthGRhKSgj5FgRa+gF5HEo6CPkWBFr64bEUk8CvoYUUUvIolKQR8jLlBKDVX0IpKAFPQxoopeRBKVgj5GdHqliCQqBX2MOFeqa9CLSEJS0MeIKnoRSVRRBb2ZNTCzWWb2mZmtMrMLzKyRmb1jZmu83w1j1dhEpi9MiUiiiraifwp4yznXFsgDVgF3A/Occ62Bed5w0tPBWBFJVBH3NZhZfaArMAzAObcf2G9mVwDdvMn+BCwAxkTTyERTVvYDmzfPpizwfWjc3n1f6YYjIpKQoulUbgVsA/5oZnnAMmA00NQ5t8mbZjPQ9Ggzm9lIYCRAy5Yto2hG9du58198tvreI8Y3adInDq0REalcNEGfCrQHbnPOLTazpzism8Y558zMHW1m59xkYDJAYWHhUadJVIHADwC0bz+dU+u2CY1XRS8iiSiaPvoSoMQ5t9gbnkUw+LeYWTMA7/fW6JqYeALeTUZqpp5GzZoHf3QtehFJRBEnk3NuM7DBzMpL2u5AMfAaMNQbNxSYE1ULE9DB2wbqdEoRSXzRJtVtwAtmlgZ8CQwn+ObxspndCHwFDIpyHQlHtw0UET+JKuidc0VA4VGe6h7NchOdKnoR8RN1KkdAFb2I+ImCPgIHg14VvYgkPgV9BA523aiiF5HEp6CPgCp6EfETBX0EVNGLiJ8o6CPgAqroRcQ/FPQRUEUvIn6ioI+ATq8UET9R0EcgWNHXwMzi3RQRkeNS0EfAEVD/vIj4hoI+ArptoIj4iYI+ArptoIj4iYI+AsGKXl03IuIPCvoIbN36tg7EiohvqCyNQCCwP3QuvYhIolNFH6FmzQbEuwkiIidEQR8B9dGLiJ8o6COgs25ExE8U9BEIBr0qehHxBwV9BPSFKRHxEwV9mJwLgC6BICI+oqAPk65cKSJ+o6APk24jKCJ+o6APk246IiJ+o6APk7puRMRvFPRhUkUvIn6joA+T+uhFxG8U9GFSRS8ifqOgD1PwPHoFvYj4h4I+TAcrenXdiIg/RB30ZpZiZh+Z2d+84VZmttjM1prZDDNLi76ZiUNn3YiI38Sioh8NrKow/CjwhHPuLGAHcGMM1pEwVNGLiN9EFfRmlgn0AaZ4wwZcCszyJvkT0D+adSSa8oq+hip6EfGJaCv6J4G7gIA3nA7sdAfvs1cCZES5joSy9ovHAFX0IuIfEQe9mfUFtjrnlkU4/0gzW2pmS7dt2xZpM6rdDz9sBqBBgw5xbomIyImJpqLvDPQzs/XAdIJdNk8BDexguZsJbDzazM65yc65QudcYePGjaNoRvVyrowmTfqQmlov3k0RETkhEQe9c26scy7TOZcFXA2855wbAswHBnqTDQXmRN3KBOJcKTXUbSMiPlIV59GPAf7HzNYS7LN/rgrWETe6X6yI+E1MSlPn3AJggff4S6BjLJabiHS/WBHxG30zNky6X6yI+I2CPkyq6EXEbxT0YVJFLyJ+o6APkw7GiojfKOjDFKzo1XUjIv6hoA9TsKLXyyYi/qHECoNzTgdjRcR3FPRh0LXoRcSPFPRh0Y3BRcR/FPRhUEUvIn6koA/DwaBXRS8i/qGgD8PB2wiqohcR/1DQhyGgil5EfEhBH4ZA2T5AFb2I+IuCPgyrV98PQEpKnTi3RETkxCnow1AW+AGAxo17xrklIiInTkEfBufKaNjwAlJSase7KSIiJ0xBHwZd/kBE/EhBHwZdi15E/EhBHwZV9CLiRwr6MKiiFxE/UtCHQXeXEhE/UtCHQXeXEhE/UtCHQXeXEhE/UmqFQRW9iPiRgj4MzgXURy8ivqOgD4MqehHxIwV9GHTWjYj4kYI+DPrClIj4kYI+DPrClIj4kYI+DKroRcSPIg56M2thZvPNrNjMPjWz0d74Rmb2jpmt8X43jF1z40sVvYj4UTQVfSnwC+fcuUAnYJSZnQvcDcxzzrUG5nnDvudcAHCq6EXEdyJOLefcJmCT9/g7M1sFZABXAN28yf4ELADGRNXKOCkr28e27e/iAgdw3o3Ba6iiFxGfiUl5amZZQAGwGGjqvQkAbAaaHmOekcBIgJYtW8aiGTG3detbFK/630PGpaU1jlNrREQiE3XQm9mpwCvAHc65b80s9JxzzpmZO9p8zrnJwGSAwsLCo04Tb2WBfQAUnvcKaWmNMEulVq1mcW6ViEh4ogp6M6tJMORfcM7N9kZvMbNmzrlNZtYM2BptI+PFuVIATjmlJWlpjeLcGhGRyERz1o0BzwGrnHP/V+Gp14Ch3uOhwJzImxdf5f3yOgArIn4WTYJ1Bq4DVphZkTfuHmAC8LKZ3Qh8BQyKronxU17R65RKEfGzaM66WQTYMZ7uHulyE4kLqKIXEf/TN2MroYpeRJKBgr4SB/voFfQi4l8K+kqUX/Kg4imjIiJ+o6CvRLCiVzUvIv6moK+EQzcaERH/U9BXQneUEpFkoKCvhO4RKyLJQEFfCVX0IpIMFPSVcIFSaqiiFxGfU9BXQhW9iCQDBX0ldI9YEUkGCvpKOFeK1VBFLyL+pqCvhCp6EUkGSrEKNpT8md27V4WGv/3uE1JT68WxRSIi0VPQV7B27QTMapCScmpoXMMGneLYIhGR6CnoK3CulJYtbuLHP/7f408sIuIT6qP3OOf0TVgRSUoK+pAAoGvPi0jyUdB7Dt5NShW9iCQXBb1Hd5MSkWSl8tVzMOgT/yU5cOAAJSUlfP/99/FuiohUg9q1a5OZmUnNmjUjmj/xU62aHOy6SfwPOSUlJdSrV4+srCzd5lAkyTnn+OabbygpKaFVq1YRLSPxU62a+Kmi//7770lPT1fIi5wEzIz09PSoPsEr6D1+66NXyIucPKL9f1fQe/xU0YuIhENB7znYR++Pij7Rvf/++2RnZ5Ofn8++fftittxHHnnkkOELL7wwJsudOnUqX3/9ddjz/eEPf2DatGkRre/WW28Ne75EtnPnTp5++uljPv/WW2/Rpk0bzjrrLCZMmHDc5Y0bN44WLVpw6qmnHjJ+4cKFtG/fntTUVGbNmnXIc71796ZBgwb07ds3rLY//vjjmBnbt28/6vNZWVmh57KyssjJySE/P5/CwsLQNB9//DEXXHABOTk5XH755Xz77bdA8OSJoUOHkpOTwznnnMOvf/3rsNoWCwp6jyr66O3fv589e/YA8MILLzB27FiKioo45ZRTYraOw4P+gw8+iMlyKwv6srKyY853yy23cP3118ekDdWpsm2KVGVBX1ZWxqhRo3jzzTcpLi7mpZdeori4uNLlXX755SxZsuSI8S1btmTq1Klcc801Rzx355138uc//zmsdm/YsIG5c+fSsmXLE55n/vz5FBUVsXTp0tC4ESNGMGHCBFasWMGAAQP4zW9+A8DMmTP54YcfWLFiBcuWLeOZZ55h/fr1AOzYsSOstkZKQe8J+LSi//zzh1i2/JqY/nz++UNhtWHVqlX84he/oE2bNnz++edMmTKFl19+mfvuu48hQ4awYMGCQyqsW2+9lalTpwLB6mj8+PG0b9+enJwcPvvsMwB2797N8OHDycnJITc3l1deeYW7776bffv2kZ+fz5AhQwBC1Z5zjjvvvJN27dqRk5PDjBkzAFiwYAHdunVj4MCBtG3bliFDhuCcO6T9s2bNYunSpQwZMiT0CSQrK4sxY8bQvn17Zs6cybPPPkuHDh3Iy8vjqquuYu/evQA88MADTJw4EYBu3boxZswYOnbsyNlnn837779f6eu2YcMGunXrRuvWrXnwwQcBuP/++3nyySdD04wbN46nnnrqqPNv2rSJrl27kp+fT7t27ULr++///m8KCwvJzs5m/PjxoekP36bf/va3nHvuueTm5nL11VcDsGTJEi644AIKCgq48MILWb169RHr3b17N927dw/tszlz5gBw991388UXX5Cfn8+dd955yDxLlizhrLPO4swzzyQtLY2rr76aOXPmsGvXLtq0aRNaz+DBg3n22WcB6NSpE82aNTti/VlZWeTm5lKjxpHx1b17d+rVO/KKs8uWLePiiy/mvPPOo1evXmzatCn03M9//nMee+yxQ/rBv/nmG3r27El2djYjRow44m/maD7//HO6du0KQI8ePXjllVeAYP/6nj17KC0tZd++faSlpXHaaacBUFhYyJAhQ3jvvfdOaB2RUvnqUUUfnj179vDyyy/z3HPPATB8+HAeeOAB6tWrR0FBAYsWLaJv374MHDiQBQsWVLqs008/neXLl/P0008zceJEpkyZwkMPPUT9+vVZsWIFEKx8rrrqKiZNmkRRUdERy5g9ezZFRUV8/PHHbN++nQ4dOoT+6T766CM+/fRTmjdvTufOnfnHP/7BRRddFJp34MCBTJo0iYkTJx7yUTw9PZ3ly5cDwX/8m266CYB7772X5557jttuu+2IdpSWlrJkyRLeeOMNHnzwQd59991jbveSJUtYuXIlderUoUOHDvTp04cbbriBK6+8kjvuuINAIMD06dOPWtUCvPjii/Tq1Ytx48ZRVlYWevP51a9+RaNGjSgrK6N79+588skn5ObmHrFNzZs3Z926ddSqVYudO3cC0LZtW95//31SU1N59913ueeee0KBVa527dq8+uqrnHbaaWzfvp1OnTrRr18/JkyYwMqVK4+6fzZu3EiLFi1Cw5mZmSxevJj69eszadIkhg0bxujRo9mxY0fodY6VAwcOcNtttzFnzhwaN27MjBkzGDduHM8//zxz5swhIyODvLy8Q+Z58MEHueiii7j//vt5/fXXQ3/nEAzunj17YmbcfPPNjBw5EoDs7GzmzJlD//79mTlzJhs2bACCf19z5syhWbNm7N27lyeeeIJGjRoBwTeHN998k0mTJjFq1Ciuu+46hg0bRvPmzWP6GijVPH7toz/77Pvist5mzZqRm5vLlClTaNu2bVTLuvLKKwE477zzmD17NgDvvvsu06dPD03TsGHDSpexaNEiBg8eTEpKCk2bNuXiiy/mww8/5LTTTqNjx45kZmYCkJ+fz/r16w8J+mP52c9+Fnq8cuVK7r33Xnbu3Mnu3bvp1avXcbel/OP5sfTo0YP09PTQfIsWLeKOO+4gPT2djz76iC1btlBQUBCa5nAdOnTghhtu4MCBA/Tv35/8/HwAXn75ZSZPnkxpaSmbNm2iuLg4FPQVtyk3N5chQ4bQv39/+vfvD8CuXbsYOnQoa9aswcw4cODAEet1znHPPfewcOFCatSowcaNG9myZUul23q812HmzJmMGjWKjz/+OOLlHMvq1atZuXIlPXr0AILdSOWh+8gjjzB37twj5lm4cGHob7FPnz6H/P0tWrSIjIwMtm7dSo8ePWjbti1du3bl+eef5/bbb+ehhx6iX79+pKWlAcE39JSUFL7++mt27NhBly5duOyyyzjzzDNJSUmhb9++9O3bl23btjF27FhatmzJBx98QMeOHWP2GlRJ142Z9Taz1Wa21szurop1xJrfTq+Mt1mzZpGRkcGVV17JL3/5S7766qtjTpuamkogEAgNH34+cK1atQBISUmhtLQ05m0tX36466hbt27o8bBhw5g0aRIrVqxg/PjxxzynOZxtOfyUufLhESNGMHXqVP74xz9yww03HHP+rl27snDhQjIyMhg2bBjTpk1j3bp1TJw4kXnz5vHJJ5/Qp0+fQ9pacZtef/11Ro0axfLly+nQoQOlpaXcd999XHLJJaxcuZK//vWvR93OF154gW3btrFs2TKKiopo2rTpcc/xzsjICFW4EPzSX0ZGBgCBQIBVq1ZRp06dKumzds6RnZ1NUVERRUVFrFixgrlz5/LFF1+wbt068vLyyMrKoqSkhPbt27N58+bjbgtAkyZNGDBgQOgTV9u2bZk7dy7Lli1j8ODB/PjHPwaCn7x69+5NzZo1adKkCZ07dz6kb3/Xrl0888wz9OvXjzVr1vD888+H3phjJeZBb8Gk/B3wX8C5wGAzOzfW64k1XdQsPD179mTGjBm8//771K9fnyuuuILLLrvsqFXsj370I4qLi/nhhx/YuXMn8+bNO+7ye/Towe9+97vQcHkA1KxZ86hVZpcuXZgxYwZlZWVs27aNhQsXhlUR1atXj+++++6Yz3/33Xc0a9aMAwcO8MILL5zwcivzzjvv8J///Id9+/bxl7/8hc6dOwMwYMAA3nrrLT788MNjfnIA+Oqrr2jatCk33XQTI0aMYPny5Xz77bfUrVuX+vXrs2XLFt58882jzhsIBNiwYQOXXHIJjz76KLt27WL37t3s2rUrFGTlx1EOt2vXLpo0aULNmjWZP39+6E2+stewQ4cOrFmzhnXr1rF//36mT59Ov379AHjiiSc455xzePHFFxk+fPhR92802rRpw7Zt2/jnP/8JBLtyPv30U3Jycti6dSvr169n/fr1ZGZmsnz5cs444wy6du3Kiy++CMCbb74Z+vvbs2dPaBv37NnD3LlzadeuHQBbt24Fgq/tww8/zC233AIEDx6/9957oXn+9a9/hT4FX3vttbRv355169Yxbdo0/v73v3P99ddTu3btmL4GVVHRdwTWOue+dM7tB6YDV1TBemLKBVTRRyI9PZ3Ro0dTVFTEI488QkrKka9fixYtGDRoEO3atWPQoEEUFBQcd7n33nsvO3bsoF27duTl5TF//nwARo4cGepyqGjAgAHk5uaSl5fHpZdeymOPPcYZZ5xxwtsxbNgwbrnllmOeDvrQQw9x/vnn07lz56i7qsp17NiRq666itzcXK666qrQ8YG0tDQuueQSBg0adNTXs9yCBQvIy8ujoKCAGTNmMHr06NBw27Ztueaaa0JvHocrKyvj2muvJScnh4KCAm6//XYaNGjAXXfdxdixYykoKDjmJ5IhQ4awdOlScnJymDZtWuj1SE9Pp3PnzrRr1+6Ig7GpqalMmjSJXr16cc455zBo0CCys7NZvXo1U6ZM4fHHH6dLly507dqVhx9+GIC77rqLzMxM9u7dS2ZmJg888AAAH374IZmZmcycOZObb76Z7Ozs0Hq6dOnCT3/6U+bNm0dmZiZvv/02aWlpzJo1izFjxpCXl0d+fv5xz9YaP348CxcuJDs7m9mzZ4fOyNmyZQsXXXQReXl5dOzYkT59+tC7d28AXnrpJc4++2zatm1L8+bNGT58OACjRo1i9+7dZGdn06FDB4YPHx6q2AcNGsTq1auZMGECrVu3rrRN0bBYH+k1s4FAb+fcCG/4OuB859wxTxouLCx0FT/KnKivv57Jvzc8d/wJT0BZ2V6+/34j7QtepGHD82OyzKqyatUqzjnnnHg3Q6pIIBAInRlTlf/84i9H+783s2XOucJjzBISt34KMxsJjATCOn+1opo1G1C3zlkxa1PDhhdSr1728ScUqSLFxcX07duXAQMGKOQlZqoi6DcCLSoMZ3rjDuGcmwxMhmBFH8mKGjfuQePGPSKZVaRavP3224wZM+aQca1ateLVV1896vTnnnsuX3755SHjVqxYwXXXXXfIuFq1arF48eLYNlaSVlUE/YdAazNrRTDgrwaO/AqbyEmgV69elR5QPRE5OTlHPTdd5ETFPOidc6VmdivwNpACPO+c+zTW6znZOed0BUuRk0S0x1KrpI/eOfcG8EZVLFuC30z85ptvdE16kZNA+Y1HojnlUieN+1BmZiYlJSVs27Yt3k0RkWpQfivBSCnofahmzZoR31JMRE4+unqliEiSU9CLiCQ5Bb2ISJKL+SUQImqE2Tbg2Jc/rNzpwNHv/5W8tM0nB23zySGabf6Rc67x8SZKiKCPhpktPZFrPSQTbfPJQdt8cqiObVbXjYhIklPQi4gkuWQI+snxbkAcaJtPDtrmk0OVb7Pv++hFRKRyyVDRi4hIJXwd9H68CfmJMLMWZjbfzIrN7FMzG+2Nb2Rm75jZGu93Q2+8mdlvvdfhEzNrH98tiIyZpZjZR2b2N2+4lZkt9rZrhpmleeNrecNrveez4tnuSJlZAzObZWafmdkqM7vgJNjHP/f+plea2UtmVjsZ97OZPW9mW81sZYVxYe9bMxvqTb/GzIZG2h7fBr1fb0J+gkqBXzjnzgU6AaO8bbsbmOecaw3M84Yh+Bq09n5GAr+v/ibHxGhgVYXhR4EnnHNnATuAG73xNwI7vPFPeNP50VPAW865tkAewW1P2n1sZhnA7UChc64dwcuYX01y7uepQO/DxoW1b82sETAeOJ/gvbjHl785hM0558sf4ALg7QrDY4Gx8W5XFW3rHKAHsBpo5o1rBqz2Hj8DDK4wfWg6v/wQvBPZPOBS4G+AEfwSSerh+5vgvQ4u8B6netNZvLchzO2tD6w7vN1Jvo8zgA1AI2+//Q3olaz7GcgCVka6b4HBwDMVxh8yXTg/vq3oOfhHU67EG5dUvI+rBcBioKlzbpP31Gagqfc4GV6LJ4G7gIA3nA7sdM6VesMVtym0vd7zu7zp/aQVsA34o9ddNcXM6pLE+9g5txGYCPwb2ERwvy0jufdzReHu25jtcz8HfdIzs1OBV4A7nHPfVnzOBd/ik+KUKTPrC2x1zi2Ld1uqUSrQHvi9c64A2MPBj/JAcu1jAK/b4QqCb3LNgboc2b1xUqjufevnoD+hm5D7lZnVJBjyLzjnZnujt5hZM+/5ZsBWb7zfX4vOQD8zWw9MJ9h98xTQwMzK75lQcZtC2+s9Xx/4pjobHAMlQIlzrvwO37MIBn+y7mOAy4B1zrltzrkDwGyC+z6Z93NF4e7bmO1zPwd96Cbk3lH6q4HX4tymmLDg/QGfA1Y55/6vwlOvAeVH3ocS7LsvH3+9d/S+E7CrwkcyKaVqAAABFUlEQVTEhOecG+ucy3TOZRHcj+8554YA84GB3mSHb2/56zDQm95Xla9zbjOwwczaeKO6A8Uk6T72/BvoZGZ1vL/x8m1O2v18mHD37dtATzNr6H0a6umNC1+8D1hEebDjJ8DnwBfAuHi3J4bbdRHBj3WfAEXez08I9k/OA9YA7wKNvOmN4BlIXwArCJ7VEPftiHDbuwF/8x6fCSwB1gIzgVre+Nre8Frv+TPj3e4ItzUfWOrt578ADZN9HwMPAp8BK4E/A7WScT8DLxE8DnGA4Ke3GyPZt8AN3vavBYZH2h59M1ZEJMn5uetGREROgIJeRCTJKehFRJKcgl5EJMkp6EVEkpyCXkQkySnoRUSSnIJeRCTJ/T8p4IJxSj9aIgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x111ac49e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_method_results(S, train_by_sarsa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "measuring with 10 seeds took 52.82578778266907\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl0VPUd9/H3lxBAUUGWUiBooGyyhMUkFbGC4ELViiAiiBZwQVupS30URBQqHtceF1qP1ooiHg5EWdR6fFxYVLCPhggRMKxlkbAZWY2sIb/nj7kZAiSZMDNx5g6f1zk5mbv/7tycT77zu3fuNeccIiKSuKrFugEiIlK1FPQiIglOQS8ikuAU9CIiCU5BLyKS4BT0IiIJTkEvIpLgFPQiIglOQS8ikuCqx7oBAA0aNHCpqamxboaIiK988803PzrnGoaaLy6CPjU1lZycnFg3Q0TEV8xsY2XmU9eNiEiCU9CLiCQ4Bb2ISIJT0IuIJDgFvYhIggsZ9Gb2upn9YGbLS42rZ2afmtka7/fZ3ngzs4lmttbMlppZ16psvIiIhFaZin4y0Oe4caOBuc65VsBcbxjg90Ar72cE8HJ0mikiIuEKeR29c+4LM0s9bnRfoKf3+k3gM2CUN36KCzyf8Cszq2tmjZ1zW6PVYL8pLFzF9h8+jHUzRCRONWzQm7POSqvSbYT7halGpcJ7G9DIe90U2FRqvnxv3AlBb2YjCFT9nHPOOWE2I/5t/P7fbNs2G7BYN0VE4lDNmo3iNuiDnHPOzE76CePOuVeBVwHS09MT9gnlxcWHOP305nS7YE6smyIip6hwr7rZbmaNAbzfP3jjNwPNSs2X4o07dblizOLiThMicooKN+jfB4Z6r4cC75Ua/0fv6psLgD2ncv88gHNFmCXFuhkicgoLWWqa2TQCJ14bmFk+MA54CnjbzG4FNgIDvdk/BK4E1gL7gOFV0GZfce6Igl5EYqoyV90MLmdS7zLmdcBdkTYqkQQqenXdiEjs6JuxVUwVvYjEmoK+ihWroheRGFPQVzFV9CISawr6KhYIelX0IhI7CvoqpssrRSTWFPRVTBW9iMSagr6KqaIXkVhT0FcxnYwVkVhT0FcxfWFKRGJNQV/FVNGLSKwp6KuYKnoRiTUFfRVTRS8isaagr2IKehGJNfUpREFBwSdsyp9S5rSior0KehGJKVX0UbD9h//Lnj2LKS4+fMJPnbO60KB+r1g3UUROYaroo8C5ImrVSiH9/KxYN0VE5ASq6KNA/fAiEs8U9FGg+9mISDxT0EeB7mcjIvFMQR8FquhFJJ4p6KNAFb2IxDMFfRToZKyIxDMFfRQ4V0Q1dd2ISJxS0EeBKnoRiWcK+ijQHSpFJJ4p6KNAFb2IxDMFfRTo8koRiWcK+ijQ5ZUiEs8U9FHgilXRi0j8UtBHgSp6EYlnEQW9md1nZt+Z2XIzm2ZmtcysuZl9bWZrzSzLzGpEq7HxSidjRSSehR30ZtYUuBtId851AJKAQcDTwPPOuZbALuDWaDQ0nhW7Iqyaum5EJD5F2nVTHTjNAh3UpwNbgV7ADG/6m8C1EW4j7qmiF5F4FnYZ6pzbbGZ/B74H9gOfAN8Au51zRd5s+UDTiFsZp1aueoQtW2bg3CGqJX4PlYj4VCRdN2cDfYHmQBOgNtDnJJYfYWY5ZpZTUFAQbjNi6qef8qhZsxGp5/6Jpk0Hx7o5IiJliqTr5lJgvXOuwDl3GJgFdAfq2tFrDVOAzWUt7Jx71TmX7pxLb9iwYQTNiB3niqhduyW/+c3/4fTTm8e6OSIiZYok6L8HLjCz083MgN5AHjAfGODNMxR4L7Imxi/1zYuIH4Qd9M65rwmcdF0MLPPW9SowCvirma0F6gOTotDOuKSbmYmIH0SUUs65ccC440avAzIjWa9fqKIXET/QN2MjoAeOiIgfKOgjoIpeRPxAQR8B3Z5YRPxAQR8B3cxMRPxAQR8BVfQi4gcK+gioohcRP1DQR0AnY0XEDxT0EdAXpkTEDxT0EVBFLyJ+oKAPk3NOJ2NFxBcU9GFy7giAKnoRiXsK+jAp6EXELxT0YSp5iJaCXkTinYI+TEcrevXRi0h8U9CHSRW9iPiFgj5MquhFxC8U9GFSRS8ifqGgD5MqehHxCwV9mFTRi4hfKOjDpOvoRcQvFPRhClb01dR1IyLxTUEfJlX0IuIXCvowHe2jV0UvIvFNQR8mVfQi4hcK+jDp8koR8QsFfZh0eaWI+IWCPkyq6EXELxT0YSoJ+mqq6EUkzinow6SKXkT8QkEfppI+ekxvoYjEN6VUmFTRi4hfRBT0ZlbXzGaY2UozW2Fm3cysnpl9amZrvN9nR6ux8URX3YiIX0Ra0b8IfOScawt0AlYAo4G5zrlWwFxvOOEcPRmril5E4lvYKWVmdYCLgWEAzrlDwCEz6wv09GZ7E/gMGBVJI+NFQcEn7D+wGYC9e5cCquhFJP5FUo42BwqAN8ysE/ANcA/QyDm31ZtnG9CorIXNbAQwAuCcc86JoBm/jCNHDrJ02Z8BFxyXlHQGyckJ2TMlIgkkkqCvDnQF/uKc+9rMXuS4bhrnnDMzV9bCzrlXgVcB0tPTy5wnngQ+sDhatPgrKU1vBqBatZokJdWMbcNEREKIpI8+H8h3zn3tDc8gEPzbzawxgPf7h8iaGB9K+uSrJ9UmOfkskpPPUsiLiC+EHfTOuW3AJjNr443qDeQB7wNDvXFDgfciamGc0G2JRcSvIk2tvwBTzawGsA4YTuCfx9tmdiuwERgY4Tbigm5LLCJ+FVHQO+dygfQyJvWOZL3xSF+QEhG/0jdjK0lfkBIRv1LQV5IqehHxKwV9JRWrohcRn1LQV5IqehHxKwV9JamPXkT8SkFfSbq8UkT8SkFfSfrClIj4lYK+klxxSUWvt0xE/EWpVUkOnYwVEX9S0FeS+uhFxK8U9JWkPnoR8SsFfSWpohcRv1LQV5IqehHxK6VWCMXFh1i1ejyFhasAVfQi4j+q6EPYt289W7ZkcfDgNurWzaRWraaxbpKIyElRRR9CSZdNm9bjadjwshi3RkTk5KmiD0E3MxMRv1PQh6CbmYmI3ynoQyjWZZUi4nMK+hB0WaWI+J2CPgR9UUpE/E5BH4IqehHxOwV9CKroRcTvFPQhqKIXEb9T0Iegil5E/E5BH4IrVkUvIv6moA9BFb2I+J2CPgQFvYj4nYI+hKPPilXQi4g/KehD0E3NRMTvIg56M0sysyVm9oE33NzMvjaztWaWZWY1Im9m7OimZiLid9Go6O8BVpQafhp43jnXEtgF3BqFbcSMKnoR8buIgt7MUoCrgNe8YQN6ATO8Wd4Ero1kG7Gmil5E/C7Siv4F4EGg2BuuD+x2JekI+YCvn73nilXRi4i/hR30ZnY18INz7pswlx9hZjlmllNQUBBuM6qcKnoR8btIKvruwDVmtgGYTqDL5kWgrh0tf1OAzWUt7Jx71TmX7pxLb9iwYQTNqFq6jl5E/C7soHfOPeScS3HOpQKDgHnOuSHAfGCAN9tQ4L2IWxlDzhVhlkTg9IOIiP9UxXX0o4C/mtlaAn32k6pgG78Y546omhcRX4vKGUbn3GfAZ97rdUBmNNYbDwIVvU7Eioh/6ZuxIaiiFxG/U9CHEAh6VfQi4l8K+hBKTsaKiPiVgj4EVfQi4ncK+hBU0YuI3ynoQ3DuCIaCXkT8S0EfQrErwqop6EXEvxT0IaiPXkT8TkEfiq6jFxGfU9CHoC9MiYjfKehD0C0QRMTvFPQhqKIXEb9T0IdQrIpeRHxOQR9C0eHdquhFxNcU9BU4cmQfhT+vorj4UKybIiISNgV9BYqKCgGoWyc9xi0REQmfgr4CJQ8GP/301Ng2REQkAgr6CujB4CKSCBT0FSip6HXVjYj4mYK+AqroRSQRKOgrcDToVdGLiH8p6CtwtOtGFb2I+JeCvgKq6EUkESjoK3C0otfbJCL+pQSrQLEqehFJAAr6CqiPXkQSgYK+Iq4YUEUvIv6moK+AKnoRSQQK+groC1MikggU9BXQLRBEJBEo6CtQrIpeRBJA2EFvZs3MbL6Z5ZnZd2Z2jze+npl9amZrvN9nR6+5vyxV9CKSCCKp6IuA+51z7YALgLvMrB0wGpjrnGsFzPWGfUl99CKSCMIOeufcVufcYu/1T8AKoCnQF3jTm+1N4NpIGxkLxcWH2b79A0AVvYj4W1T66M0sFegCfA00cs5t9SZtAxqVs8wIM8sxs5yCgoJoNCOqdu36f+zYMR+A6tXPinFrRETCF3HQm9kZwEzgXufc3tLTnHMOcGUt55x71TmX7pxLb9iwYaTNiLojR/YD0LXLVJKTFfQi4l8RBb2ZJRMI+anOuVne6O1m1tib3hj4IbImxkbJidjkGvVj3BIRkchEctWNAZOAFc6550pNeh8Y6r0eCrwXfvNip+REbDX1z4uIz0WSYt2Bm4FlZpbrjRsDPAW8bWa3AhuBgZE1MTZ0+wMRSRRhB71zbiFg5UzuHe5644UeOiIiiULfjC2HKnoRSRQK+nLoy1IikigU9OVQRS8iiUJBXw710YtIolDQl0NdNyKSKBT05VDQi0iiUNCXQ7coFpFEoaAvR0lFr7dIRPxOKVYO54owq07gTg8iIv6loC+Hc0fUPy8iCUFBX45A0Kt/XkT8T0FfjmJXpIpeRBKCgr4cquhFJFEoyUo5dOhH1m94ieLig+zenaOKXkQSgoK+lJ07/0t+/hSSk+thVp16Z18Y6yaJiERMQV+Kc4cByEifxWmnNYtxa0REokN99KXotgcikogU9KUU67YHIpKAFPSlqKIXkUSkoC9FNzITkUSkRCslmhX94cOHyc/P58CBAxGvS0RObbVq1SIlJYXk5OSwllfQlxLNoM/Pz+fMM88kNTVVN0YTkbA559ixYwf5+fk0b948rHWo66a0KAb9gQMHqF+/vkJeRCJiZtSvXz+i3gEFfSnFUX5OrEJeRKIh0ixR0JcSOBlrmOltEZHEoUQr5VS7kdmCBQto3749nTt3Zv/+/VFb7xNPPHHM8IUXRudWEpMnT2bLli0nvdwrr7zClClTwtreyJEjT3q5SPTs2ZOcnJwq3caWLVsYMGBAlW4jNzeXDz/8sNzpTz75JC1btqRNmzZ8/PHHIdfXp08f6taty9VXX33M+H/+85+0bNkSM+PHH38Mjl+5ciXdunWjZs2a/P3vfz9mmYkTJ3LeeecxZMgQnn32WTp37kznzp3p0KEDSUlJ7Ny5s9x23HfffcH5W7duTd26dQH47LPPTmhbiWnTptGxY0fS0tLo06fPMe38xTjnYv5z/vnnu3iwes2Tbt78dlFZV15eXlTWE20HDx50hYWFzjnn7rjjDvfWW29FfRu1a9eO+jqdc65Hjx5u0aJFZU4rKiqK+vbeeOMNd9ddd0V9vRWpaB9PxuHDh6PQmvBV9N599913Li0tzR04cMCtW7fOtWjRIuTxmzNnjnv//ffdVVdddcz4xYsXu/Xr17tzzz3XFRQUBMdv377dZWdnuzFjxrhnn332mGXatGnjNm3adMI23n//fXfJJZdUdhfdxIkT3fDhw51zzs2fP/+EtjkXOA4NGzYMtu2BBx5w48aNc845t3Pnzkpvy7myMwXIcZXIWFX0pVRVRb969QS+WXxjVH9Wr55wUm1YsWIF999/P23atGH16tW89tprvP322zzyyCMMGTLkhIpk5MiRTJ48GYDU1FTGjRtH165d6dixIytXrgSgsLCQ4cOHB6uVmTNnMnr0aPbv30/nzp0ZMmQIAGeccQYQKCoeeOABOnToQMeOHcnKygIC1VDPnj0ZMGAAbdu2ZciQIQT+ho+aMWMGOTk5DBkyJPgJJDU1lVGjRtG1a1feeecd/v3vf5ORkUGnTp247rrr2LdvHwDjx48PVnU9e/Zk1KhRZGZm0rp1axYsWFDh+7Zp0yZ69uxJq1at+Nvf/gbAo48+ygsvvBCc5+GHH+bFF18sc3nnHCNHjqRNmzZceumlXHnllcyYMSP0AQM++eQTunXrRteuXbn++uspLCwE4LHHHiMjI4MOHTowYsSI4HvVs2dP7r33XtLT03nxxRcZNmwYd999NxdeeCEtWrQIbnfDhg106NABCHxq6d+/P3369KFVq1Y8+OCDwe1PmjSJ1q1bk5mZye23317mp5vs7Gy6detGly5duPDCC1m1ahWHDh3i0UcfJSsri86dOwePc4n33nuPQYMGUbNmTZo3b07Lli3Jzs5m0aJFpKWlceDAAX7++Wfat2/P8uXLAejduzdnnnnmCdvv0qULqampJ4z/1a9+RUZGxgmXI955552sW7eO3//+9zz//PPHTJs2bRqDBw8ODk+ZMoW0tDQ6derEzTfffMI2jp9/7969XHXVVbRp04Y777yT4uLiYND+/PPPOOfYu3cvTZo0AeDZZ58lMzOTf/3rX+zdu/eE9UeTgr4Ul2APG/n555954403uOiii7j99ttp164dS5cupUuXLtx2221cc801PPvss0ydOjXkuho0aMDixYv505/+FAzNCRMmUKdOHZYtW8bSpUvp1asXTz31FKeddhq5ubknrHfWrFnk5uby7bffMmfOHB544AG2bt0KwJIlS3jhhRfIy8tj3bp1fPnll8csO2DAANLT05k6dSq5ubmcdtppANSvX5/FixczaNAg+vfvz6JFi/j2228577zzmDRpUpn7UlRURHZ2Ni+88EIwvMuTnZ3NzJkzWbp0Ke+88w45OTnccsstwa6g4uJipk+fzk033VTm8rNnz2bVqlXk5eUxZcoU/vvf/4Z4pwN+/PFHHn/8cebMmcPixYtJT0/nueeeAwL/hBctWsTy5cvZv38/H3zwQXC5Q4cOkZOTw/333w/A1q1bWbhwIR988AGjR48uc1u5ublkZWWxbNkysrKy2LRpE1u2bGHChAl89dVXfPnll8F/7sdr27YtCxYsYMmSJTz22GOMGTOGGjVq8Nhjj3HDDTeQm5vLDTfccMwymzdvplmzozcNTElJYfPmzWRkZHDNNdcwduxYHnzwQW666abgP6RoeeWVV2jSpAnz58/nvvvuC47ft28fH330Eddddx0A3333HY8//jjz5s3j22+/PeEf+caNG1m/fj29evUKjsvOzuYf//gHeXl5/O9//2PWrFkkJyfz8ssv07FjR5o0aUJeXh633norEOjifOutt1i3bh1du3Zl+PDhLFy4MKr7W+LU6ZCuhKp6Tmzr1o9EfZ2V0bhxY9LS0njttddo27ZtROvq378/AOeffz6zZs0CYM6cOUyfPj04z9lnn13hOhYuXMjgwYNJSkqiUaNG9OjRg0WLFnHWWWeRmZlJSkoKAJ07d2bDhg1cdNFFIdtVOkSWL1/O2LFj2b17N4WFhVxxxRUh92XDhg0Vrv+yyy6jfv36weUWLlzIvffeS/369VmyZAnbt2+nS5cuwXmO98UXXwT3uUmTJscEQ0W++uor8vLy6N69OxAI8G7dugEwf/58nnnmGfbt28fOnTtp3749f/jDH054PwCuvfZaqlWrRrt27di+fXuZ2+rduzd16tQBoF27dmzcuJEff/yRHj16UK9ePQCuv/56Vq9efcKye/bsYejQoaxZswYz4/Dhw5Xav/I8+uijZGRkUKtWLSZOnBjRuk7Gf/7zH7p37x7c33nz5nH99dfToEEDgOD4EtOnT2fAgAEkJR3Ni8zMTFq0aAHA4MGDWbhwIX379uXll19myZIltGjRgr/85S88+eSTjB07FoA2bdrw9NNP88QTTzBt2jSuuuoqhg4dGvV9r5KgN7M+wItAEvCac+6pqthOtAUq+sT53zdjxgwmTZpE//79GTRoEEOHDuXcc88tc97q1atTXFwcHD7+mt2aNWsCkJSURFFRUdTbWrL+k91G7dq1g6+HDRvGu+++S6dOnZg8eTKfffZZhduqzHaOv6ytZPi2225j8uTJbNu2jVtuuaVSbT0Zzjkuu+wypk2bdsz4AwcO8Oc//5mcnByaNWvG+PHjjzlWpd8POPZ9Pb47rKx5Tvb4PvLII1xyySXMnj2bDRs20LNnz5DLNG3alE2bNgWH8/Pzadq0KQA7duygsLCQw4cPc+DAgRP2p6pMnz79mG6Yysz/0ksvHTOurL+V3NxcAH7zm98AMHDgQJ566mgcOueYP38+r7/+OtnZ2dx9993cdttt4e5GuaLedWOBkvgl4PdAO2CwmbWL9naqQlVV9LFy+eWXk5WVxYIFC6hTpw59+/bl0ksvLbOKPffcc8nLy+PgwYPs3r2buXPnhlz/ZZdddswf+65duwBITk4us7L73e9+R1ZWFkeOHKGgoIAvvviCzMzMSu/PmWeeyU8//VTu9J9++onGjRtz+PDhSnVHVcann37Kzp072b9/P++++26wwu7Xrx8fffQRixYtKveTA8DFF18c3OetW7cyf/78Sm33ggsu4Msvv2Tt2rVAoBtu9erVwVBv0KABhYWFle7vP1kZGRl8/vnn7Nq1i6KiImbOnFnmfHv27AmGdMk5Haj4WF1zzTVMnz6dgwcPsn79etasWRP8O7jjjjuYMGECQ4YMYdSoUdHdqXLs2bOHzz//nL59+wbH9erVi3feeYcdO3YAHHMlzsqVK9m1a1fwE1aJ7Oxs1q9fT3FxMVlZWVx00UU0bdqUvLw8CgoKgMDf03nnnQfA1KlTadu2LS+99BI33ngjK1asYMKECeUWY5Goij76TGCtc26dc+4QMB3oG2KZuJBoFX2J+vXrc88995Cbm8sTTzxxzMfNEs2aNWPgwIF06NCBgQMH0qVLl5DrHTt2LLt27aJDhw506tQpGGIjRowgLS0teDK2RL9+/YInt3r16sUzzzzDr3/960rvx7Bhw7jzzjvLvRx0woQJ/Pa3v6V79+4Rd1WVyMzM5LrrriMtLY3rrruO9PR0AGrUqMEll1zCwIEDy3w/S/Tr149WrVrRrl07/vjHP54QDuVp2LAhkydPZvDgwaSlpdGtWzdWrlxJ3bp1uf322+nQoQNXXHEFGRkZUdnP4zVt2pQxY8aQmZlJ9+7dSU1NDXbvlPbggw/y0EMP0aVLl2M+CVxyySXk5eWVeTK2ffv2DBw4kHbt2tGnTx9eeuklkpKSmDJlCsnJydx4442MHj2aRYsWMW/ePCBQJFx//fXMnTuXlJSU4CWZEydOJCUlhfz8fNLS0oLV8LZt20hJSeG5557j8ccfJyUlpdwTnrNnz+byyy8/5tND+/btefjhh+nRowedOnXir3/9a3Da9OnTGTRo0AkVfEZGBiNHjuS8886jefPm9OvXjyZNmjBu3Dguvvhi0tLSyM3NZcyYMUCguFq4cCEzZ87kyiuvrPDvKFJW3se5sFdoNgDo45y7zRu+Gfitc67cC5LT09NdONcOb9nyDt9vKvuEWzgOHNhKjRr1ubDbvIjXtWLFiuB/bkk8xcXFwat9WrVqVenlhg0bxtVXX13l17FHQ2FhIWeccQZFRUX069ePW265hX79+sW6WaessjLFzL5xzqWHWjZm5auZjQBGAJxzzjlhrSM5uS61T28ZtTbVPr0lZ9fTc2KlYnl5eVx99dXBaj1RjR8/njlz5nDgwAEuv/xyrr322lg3ScJUFRV9N2C8c+4Kb/ghAOfck+UtE25FH89U0fvHxx9/fEJ/cPPmzZk9e3al17Fs2bITrrWuWbMmX3/9dZnz9+vXj/Xr1x8z7umnn66wv19ObZFU9FUR9NWB1UBvYDOwCLjROfddecso6EVEKhZXXTfOuSIzGwl8TODyytcrCvlE5pzTHSxFJGKRFuRV0kfvnPsQKP+ORqeAWrVqsWPHDt2TXkQi4rwHj9SqVSvsdSTetYRxouSSr5LrZ0VEwlXyKMFwKeirSHJyctiP/RIRiSbd1ExEJMEp6EVEEpyCXkQkwUX9OvqwGmFWAGwMc/EGQAyezRVT2udTg/b51BDJPp/rnGsYaqa4CPpImFlOZb4wkEi0z6cG7fOp4ZfYZ3XdiIgkOAW9iEiCS4SgfzXWDYgB7fOpQft8aqjyffZ9H72IiFQsESp6ERGpgK+D3sz6mNkqM1trZqNj3Z5oMbNmZjbfzPLM7Dszu8cbX8/MPjWzNd7vs73xZmYTvfdhqZl1je0ehMfMksxsiZl94A03N7Ovvf3KMrMa3via3vBab3pqLNsdLjOra2YzzGylma0ws26nwDG+z/ubXm5m08ysViIeZzN73cx+MLPlpcad9LE1s6He/GvMbGi47fFt0Pv5IeSVUATc75xrB1wA3OXt22hgrnOuFTDXG4bAe9DK+xkBvPzLNzkq7gFWlBp+GnjeOdcS2AXc6o2/FdjljX/em8+PXgQ+cs61BToR2PeEPcZm1hS4G0h3znUgcBvzQSTmcZ4M9Dlu3EkdWzOrB4wDfkvgWdzjSv45nDTnnC9/gG7Ax6WGHwIeinW7qmhf3wMuA1YBjb1xjYFV3ut/AYNLzR+czy8/QIr3x98L+AAwAl8iqX788SbwrINu3uvq3nwW6304yf2tA6w/vt0JfoybApuAet5x+wC4IlGPM5AKLA/32AKDgX+VGn/MfCfz49uKnqN/NCXyvXEJxfu42gX4GmjknNvqTdoGNPJeJ8J78QLwIFDsDdcHdjvnirzh0vsU3F9v+h5vfj9pDhQAb3jdVa+ZWW0S+Bg75zYDfwe+B7YSOG7fkNjHubSTPbZRO+Z+DvqEZ2ZnADOBe51ze0tPc4F/8QlxyZSZXQ384Jz7JtZt+QVVB7oCLzvnugA/c/SjPJBYxxjA63boS+CfXBOgNid2b5wSfulj6+eg3ww0KzWc4o1LCGaWTCDkpzrnZnmjt5tZY296Y+AHb7zf34vuwDVmtgGYTqD75kWgrvcMYjh2n4L7602vA+z4JRscBflAvnOu5OnhMwgEf6IeY4BLgfXOuQLn3GFgFoFjn8jHubSTPbZRO+Z+DvpFQCvvjH0NAid13o9xm6LCAs8enASscM49V2rS+0DJmfehBPruS8b/0Tt7fwGwp9RHxLjnnHvIOZfinEslcBznOeeGAPOBAd5sx+9vyfswwJvfV5Wvc24bsMnM2nijegN5JOgx9nzfkdcgAAAA4ElEQVQPXGBmp3t/4yX7nLDH+Tgne2w/Bi43s7O9T0OXe+NOXqxPWER4suNKYDXwP+DhWLcnivt1EYGPdUuBXO/nSgL9k3OBNcAcoJ43vxG4Aul/wDICVzXEfD/C3PeewAfe6xZANrAWeAeo6Y2v5Q2v9aa3iHW7w9zXzkCOd5zfBc5O9GMM/A1YCSwH3gJqJuJxBqYROA9xmMCnt1vDObbALd7+rwWGh9sefTNWRCTB+bnrRkREKkFBLyKS4BT0IiIJTkEvIpLgFPQiIglOQS8ikuAU9CIiCU5BLyKS4P4//ZXjZb1jItMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x111f2c710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_method_results(S, train_by_q_learning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can clearly see the trend: \n",
    "- value iteration has no exploration and that's why we end up exploting one reward;\n",
    "- Q and SARSA learning use an update rule with the immediate reward and that's why it is hard to propogate more than one deferred reward;\n",
    "- Monte Carlo method performs best: since it utilizes exploration and updates the Q function based on the final reward."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}