1b_Breakout-v0-Evaluate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7d1fa20c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys\n",
    "import gc\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65a74373",
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.append('../')\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c8c767df",
   "metadata": {},
   "outputs": [],
   "source": [
    "from get_algos import get_all_algos , run_experiment ,call_paths, plot_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c9fc3531",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "from gym.wrappers import Monitor\n",
    "from stable_baselines3 import PPO, A2C, DDPG, DQN, SAC, TD3\n",
    "from stable_baselines3.common.vec_env import VecFrameStack\n",
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "from stable_baselines3.common.env_util import make_atari_env"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1d33e3c",
   "metadata": {},
   "source": [
    "### 1. Explore breakout environment\n",
    "<ul>\n",
    "    <li>Explore the environment</li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1aad3196",
   "metadata": {},
   "outputs": [],
   "source": [
    "env_name='Breakout-v0'\n",
    "env=make_atari_env(env_name)\n",
    "env=VecFrameStack(env,n_stack=6)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54c5cdd5",
   "metadata": {},
   "source": [
    "### 2. Evaluation\n",
    "<ol>\n",
    "    <li>Create the environment</li>\n",
    "    <li>For each algorithm:</li>\n",
    "    <ol><li>Load the model</li>\n",
    "        <li>Evaluate the model for 5 sample iterations </li>\n",
    "        <li>Output the score  for each algo and each iteration</li>\n",
    "        <li>Assess stability of the model based on the variance / standard deviation </li>\n",
    "    </ol>\n",
    "    <li>Plot the corresponding scores</li>\n",
    "</ol> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "65fae03b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['PPO', 'A2C', 'DQN']\n"
     ]
    }
   ],
   "source": [
    "algo_list=get_all_algos(gym.make(env_name))\n",
    "print(algo_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "52178f03",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_experiment(algo_list, env_name,n_steps=200000,n_episodes=100):\n",
    "    rewards_mean={}\n",
    "    rewards_std={}\n",
    "    for algo_name in reversed(algo_list):\n",
    "        env=make_atari_env(env_name)\n",
    "        env=VecFrameStack(env, n_stack=6)\n",
    "        print('******* Evaluating', algo_name,' *******')\n",
    "        device='cpu' if algo_name=='DQN' else 'cuda'\n",
    "        log_path, render_path, model_path=call_paths(algo_name,env_name,n_steps)\n",
    "        model_name=env_name+\"_\"+algo_name+\"_model\"\n",
    "        model=eval(algo_name).load(os.path.join(model_path,model_name),env,device=device)\n",
    "        mean_rew, mean_std=evaluate_policy(model,env, n_eval_episodes=n_episodes)\n",
    "        rewards_mean[algo_name]=mean_rew\n",
    "        rewards_std[algo_name]=mean_std\n",
    "        print('Mean rewards for ',n_episodes, ' episodes of ',algo_name,' is ',mean_rew,'\\n')\n",
    "        del model\n",
    "        env.close()\n",
    "    return rewards_mean,rewards_std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "66a2bd41",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "******* Evaluating DQN  *******\n",
      "Wrapping the env in a VecTransposeImage.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\ai_gym\\lib\\site-packages\\stable_baselines3\\common\\buffers.py:229: UserWarning: This system does not have apparently enough memory to store the complete replay buffer 84.69GB > 11.52GB\n",
      "  \"This system does not have apparently enough memory to store the complete \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean rewards for  100  episodes of  DQN  is  9.13 \n",
      "\n",
      "******* Evaluating A2C  *******\n",
      "Wrapping the env in a VecTransposeImage.\n",
      "Mean rewards for  100  episodes of  A2C  is  8.62 \n",
      "\n",
      "******* Evaluating PPO  *******\n",
      "Wrapping the env in a VecTransposeImage.\n",
      "Mean rewards for  100  episodes of  PPO  is  7.93 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "rewards=evaluate_experiment(algo_list, env_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0a4d7aa0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEPCAYAAABIut/fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWjUlEQVR4nO3de5hddX3v8feXJGIAYzBMkFtIoEBM0jAkYwgS8AFblUuRWEgNlYvHmpNjWrFV+1BtRbw9Wk+VFj3FSApVEOUoHDi1RCAYLSLEBAfJBRLkJoQjA02gJIRJhu/5Y6+EyX2FmTWXNe/X8+xn7/1bt+/OznxmzW//9m9FZiJJqp+9ersASVI1DHhJqikDXpJqyoCXpJoy4CWppgx4SaopA74XRcTFEbE0IpZFxEd3sHxsRPwiIl6OiI93an99RCyKiPuLbS/r0cIl9QuDe7uAgSoiJgAfAqYA7cD8iPi3zHy402r/CXwEOHubzV8GTs3MFyNiCHBXRNyamff0QOmS+gnP4HvPW4B7M3N9Zm4Cfgq8t/MKmflMZv4S2LhNe2bmi8XTIcXNb6xJ2ooB33uWAidFxIiI2Ac4HTis7MYRMSgiWoFngNsz895qypTUXxnwvSQzVwBfBm4D5gOtQMcebN+Rmc3AocCUostHkrYw4HtRZs7LzMmZeTKwBlj5GvaxFvgJ8O5uLk9SP2fA96KIGFncj6LR//7dkts1RcTw4vFQ4A+BBysqU1I/Fc4m2Xsi4j+AETQ+RP2rzFwQEbMBMvPKiHgzsBgYBrwCvAiMA0YD/woMovFL+obM/GzPvwJJfVmfCvgDDjggR48e3dtlSFK/sWTJkmczs2lHy/rUOPjRo0ezePHi3i5DkvqNiHh8Z8vsg5ekmjLgJammDHhpD3zta19j/PjxTJgwgZkzZ7Jhw4bt1rnhhhsYN24c48eP57zzzgOgtbWVE044gfHjxzNx4kS+//3v93TpGoD61IesLS0taR+8+qqnnnqKadOmsXz5coYOHcqMGTM4/fTTueiii7ass2rVKmbMmMGdd97J/vvvzzPPPMPIkSNZuXIlEcFRRx3F6tWrmTx5MitWrGD48OG99npUDxGxJDNbdrSsT33IKvV1mzZt4qWXXmLIkCGsX7+egw8+eKvl3/rWt5gzZw77778/ACNHjgTg6KOP3rLOwQcfzMiRI2lrazPgVSm7aKSSDjnkED7+8Y8zatQoDjroIN74xjfyzne+c6t1Vq5cycqVKznxxBOZOnUq8+fP324/ixYtor29nSOPPLKnStcAZcBLJa1Zs4abb76ZRx99lNWrV7Nu3TquvfbardbZtGkTq1atYuHChVx//fV86EMfYu3atVuWP/3005x//vlcffXV7LWXP36qlv/DpJLuuOMOxowZQ1NTE0OGDOG9730vd99991brHHrooZx11lkMGTKEMWPGcPTRR7Nq1SoAXnjhBc444wy+8IUvMHXq1N54CRpgDHippFGjRnHPPfewfv16MpMFCxbwlre8Zat1zj77bBYuXAjAs88+y8qVKzniiCNob29n+vTpXHDBBZxzzjm9UL0Gotp8yDr6kh/1dgm19diXzujtEvqE448/nnPOOYdJkyYxePBgjjvuOGbNmsWnP/1pWlpaOOuss3jXu97Fbbfdxrhx4xg0aBBf+cpXGDFiBNdeey0/+9nPeO6557jmmmsAuOaaa2hubu7V16R6q80wSQO+Oga81HftapikXTSSVFMGvCTVlAEvSTVlwEtSTRnwklRTBrwk1ZQBL6mWHnroIZqbm7fchg0bxuWXX77VOmvWrGH69OlMnDiRKVOmsHTpUgA2bNjAlClTOPbYYxk/fjyXXnppL7yCrqvNF50kqbNjjjmG1tZWADo6OjjkkEOYPn36Vut88YtfpLm5mZtuuokHH3yQOXPmsGDBAvbee2/uvPNO9ttvPzZu3Mi0adM47bTT+t0UE57BS6q9BQsWcOSRR3L44Ydv1b58+XJOPfVUAMaOHctjjz3G7373OyKC/fbbD4CNGzeyceNGIqLH6+4qz+DVK/zmcXX85vH2vve97zFz5szt2o899lhuvPFGTjrpJBYtWsTjjz/Ok08+yYEHHkhHRweTJ0/m4YcfZs6cORx//PG9UHnXeAYvqdba29u55ZZbOPfcc7dbdskll7B27Vqam5u54oorOO644xg0aBAAgwYNorW1lSeffJJFixZt6Z/vTzyDl1Rrt956K5MmTeLAAw/cbtmwYcO4+uqrAchMxowZwxFHHLHVOsOHD+eUU05h/vz5TJgwoUdq7i6ewUuqteuvv36H3TMAa9eupb29HYCrrrqKk08+mWHDhtHW1rblQi0vvfQSt99+O2PHju2pkruNZ/CSamvdunXcfvvtfPOb39zSduWVVwIwe/ZsVqxYwYUXXkhEMH78eObNmwc0rrx14YUX0tHRwSuvvMKMGTM488wze+U1dIUBL6m29t13X5577rmt2mbPnr3l8QknnMDKlSu3227ixIn86le/qry+qtlFI0k1ZcBLUk0Z8JJUUwa8JNWUAS9JNVVpwEfEX0bEsohYGhHXR8TrqzyeJOlVlQV8RBwCfARoycwJwCDgfVUdT5K0taq7aAYDQyNiMLAPsLri40mSCpUFfGY+BfxP4AngaeD5zLxt2/UiYlZELI6IxW1tbVWVI0kDTpVdNPsD7wHGAAcD+0bE+7ddLzPnZmZLZrY0NTVVVY4kDThVTlXwB8CjmdkGEBE3Am8Drq3wmJIq4hz+1alqDv8q++CfAKZGxD7RuBTKO4AVFR5PktRJlX3w9wI/AO4DHiiONbeq40mStlbpbJKZeSnQPy9HLkn9nN9klaSaMuAlqaYMeEmqKQNekmrKgJekmjLgJammDHhJqikDXpJqyoCXpJoy4CWppgx4SaopA16SasqAl6SaMuAlqaYMeEmqKQNekmrKgJekmjLgJammDHhJqikDXpJqyoCXpJoy4CWppgx4SaopA16SasqAl6SaMuAlqaYMeEmqKQNekmrKgJekmjLgJammDHhJqikDXpJqyoCXpJoy4CWppnYb8BFxbkS8oXj8txFxY0RMqr40SVJXlDmD/7vM/K+ImAb8ATAP+Odqy5IkdVWZgO8o7s8A5mbmj4DXVVeSJKk7lAn4pyLim8CfAP8eEXuX3I6IGB4RP4iIByNiRUSc0JViJUnllQnqGcCPgXdl5lrgTcAnSu7/H4H5mTkWOBZY8VqKlCTtucE7WxARb+r0dGGntpeBxbvbcUS8ETgZuAggM9uB9tdeqiRpT+w04IElQAIBjALWFI+HA08AY3az7zFAG3B1RBxb7O/izFzXeaWImAXMAhg1atSevwJJ0g7ttIsmM8dk5hHAHcAfZeYBmTkCOBO4rcS+BwOTgH/OzOOAdcAlOzjO3MxsycyWpqam1/QiJEnbK9MHPzUz/33zk8y8FXhbie2eBJ7MzHuL5z+gEfiSpB5QJuBXF19wGl3cPgWs3t1Gmfn/gN9GxDFF0zuA5V2oVZK0B3bVB7/ZTOBS4CYaffI/K9rK+Avguoh4HfAI8IHXUqQkac/tMuAjYhBwRWb+6WvZeWa2Ai2vZVtJUtfssosmMzuAw4szcElSP1Kmi+YR4OcRcQuNkTAAZOZXK6tKktRlZQL+N8VtL+AN1ZYjSeouuw34zLysJwqRJHWv3QZ8RDQBfw2MB16/uT0zT62wLklSF5UZB38d8CCNqQcuAx4DfllhTZKkblAm4Edk5jxgY2b+NDP/G+DZuyT1cWU+ZN1Y3D8dEWfQ+Bbrm3axviSpDygT8J8vpv79GHAFMAz4y0qrkiR1WZmAvyMzNwDPA6dUXI8kqZuUCfilEfE74D+K212Z+Xy1ZUmSumq3H7Jm5u/RmFzsARoX3r4/IlorrkuS1EVlxsEfCpwInETjuqrLgLsqrkuS1EVlumieoDHu/YuZObvieiRJ3aTMOPjjgG8D50XELyLi2xHxwYrrkiR1UZm5aO6PiM0Tjp0EvB94OzCv4tokSV1Qpg9+MbA3cDeNUTQnZ+bjVRcmSeqaMn3wp2VmW+WVSJK6VZk++L0iYl5E3AoQEePsg5ekvq9MwF8D/Bg4uHi+EvhoRfVIkrpJmYA/IDNvAF4ByMxNQEelVUmSuqxMwK+LiBFAAkTEVBrz0kiS+rAyH7L+FXALcGRE/BxoAs6ptCpJUpftMuAjYhCNMe9vB44BAngoMzfuajtJUu/bZRdNZnYAMzNzU2Yuy8ylhrsk9Q9lumh+HhFfB74PrNvcmJn3VVaVJKnLygR8c3H/2U5tiddllaQ+rcxcNF7FSZL6oTLDJCVJ/ZABL0k1ZcBLUk2V+ZCViHgbMLrz+pn57YpqkiR1gzLzwX8HOBJo5dU5aJLGVZ4kSX1UmTP4FmBcZmbVxUiSuk+ZPvilwJurLkSS1L3KnMEfACyPiEXAy5sbM/OsyqqSJHVZmYD/TFcOUExYthh4KjPP7Mq+JEnllfkm60+7eIyLgRXAsC7uR5K0B3bbBx8RUyPilxHxYkS0R0RHRLxQZucRcShwBnBVVwuVJO2ZMh+yfh2YCawChgJ/Bnyj5P4vB/6a4nJ/OxIRsyJicUQsbmtrK7lbSdLulPoma2Y+DAzKzI7MvBp49+62iYgzgWcyc8lu9j03M1sys6WpqalU0ZKk3SvzIev6iHgd0BoRfw88TblfDCcCZ0XE6cDrgWERcW1mvv+1lytJKqtMUJ9frPfnNC74cRjwx7vbKDP/JjMPzczRwPuAOw13Seo5ZUbRPB4RQ4GDMvOyHqhJktQNyoyi+SMa89DML543R8Qte3KQzFzoGHhJ6lllumg+A0wB1gJkZiswprKKJEndokzAb8zM57dpc+IxSerjyoyiWRYR5wGDIuIo4CPA3dWWJUnqqjJn8H8BjKcx0dj1wAvARyusSZLUDcqMolkPfKq4SZL6iTJXdGoBPsn2l+ybWF1ZkqSuKtMHfx3wCeABdjGnjCSpbykT8G2ZuUfj3iVJva9MwF8aEVcBC9j6ik43VlaVJKnLygT8B4CxwBBe7aJJwICXpD6sTMC/NTOPqbwSSVK3KjMO/u6IGFd5JZKkblXmDH4qjbngH6XRBx9AOkxSkvq2MgG/26s3SZL6nlLzwfdEIZKk7lXqmqySpP7HgJekmjLgJammDHhJqikDXpJqyoCXpJoy4CWppgx4SaopA16SasqAl6SaMuAlqaYMeEmqKQNekmrKgJekmjLgJammDHhJqikDXpJqyoCXpJoy4CWppgx4SaopA16SaqqygI+IwyLiJxGxPCKWRcTFVR1LkrS9wRXuexPwscy8LyLeACyJiNszc3mFx5QkFSo7g8/MpzPzvuLxfwErgEOqOp4kaWs90gcfEaOB44B7d7BsVkQsjojFbW1tPVGOJA0IlQd8ROwH/BD4aGa+sO3yzJybmS2Z2dLU1FR1OZI0YFQa8BExhEa4X5eZN1Z5LEnS1qocRRPAPGBFZn61quNIknasyjP4E4HzgVMjorW4nV7h8SRJnVQ2TDIz7wKiqv1LknbNb7JKUk0Z8JJUUwa8JNWUAS9JNWXAS1JNGfCSVFMGvCTVlAEvSTVlwEtSTRnwklRTBrwk1ZQBL0k1ZcBLUk0Z8JJUUwa8JNWUAS9JNWXAS1JNGfCSVFMGvCTVlAEvSTVlwEtSTRnwklRTBrwk1ZQBL0k1ZcBLUk0Z8JJUUwa8JNWUAS9JNWXAS1JNGfCSVFMGvCTVlAEvSTVlwEtSTRnwklRTBrwk1ZQBL0k1ZcBLUk1VGvAR8e6IeCgiHo6IS6o8liRpa5UFfEQMAr4BnAaMA2ZGxLiqjidJ2lqVZ/BTgIcz85HMbAe+B7ynwuNJkjoZXOG+DwF+2+n5k8Dx264UEbOAWcXTFyPioQpr6isOAJ7t7SLKii/3dgV9Qr95z3y/thgo79nhO1tQZcCXkplzgbm9XUdPiojFmdnS23WoPN+z/sf3rNoumqeAwzo9P7RokyT1gCoD/pfAURExJiJeB7wPuKXC40mSOqmsiyYzN0XEnwM/BgYB/5KZy6o6Xj8zoLqkasL3rP8Z8O9ZZGZv1yBJqoDfZJWkmjLgJammDPhuFBEdEdEaEcsi4v6I+FhE7NVp+bSIWBQRDxZTOHy407LPRMT6iBjZqe3Fnn4NA1VEnB0RGRFji+fNEfGL4r38dUT8Sad1h0TElyJiVUTcV6x3Wu9VP/B0+llbGhH/OyL22U37oRFxc/Ge/SYi/rEY/FFrBnz3eikzmzNzPPCHNKZpuBQgIt4MfBeYnZljgROBD0bE9E7bPwt8rIdrVsNM4K7iHmA9cEHxXr4buDwihhfLPgccBEzIzEnA2cAberRabf5ZmwC0A7N31h4RAdwI/J/MPAo4GtgP+EJvFN6T/JC1G0XEi5m5X6fnR9AYLnoA8FkgM/PTnZa/A/hcZr4tIj5TNF8ETMrM/9x2f6pGROwHPAScAvzfzDxmB+vcD5xD47scvwXGZOYLPVqotuj8sxERs4GJmfnhHbUDPwQuzcyTO20/DHgUOCwz1/f8K+gZnsFXKDMfoTFEdCQwHliyzSqLaUzEttmLwL8AF/dIgdrsPcD8zFwJPBcRkzsvjIgpwOuA3wC/BzxhuPcNETGYxl/KD+yifbufveL9e4LG+1lbBnzf80/AhRHhn/w9ZyaNyfAo7jd30xARBwHfAT6Qma/0Qm3asaER0UrjJOkJYN5u2gekXp+Lps6KLpoO4BlgOTAZuLnTKpNp/EfcIjPXRsR3gTk9VedAFhFvAk4Ffj8iksZfXBkRn6DRr/4j4FOZeU+xycPAqIgY5ll8r3opM5vLtEfEchrda53bhgGjaLyfteUZfEUiogm4Evh6Nj7o+AZwUUQ0F8tH0PiQ53M72PyrwH/HX8A94RzgO5l5eGaOzszDaPTNngTcBHw7M3+weeWiv3YesGUURkQ0RcS5vVC7ylkA7BMRF8CWa1X8A3BNnfvfwYDvbkM3D5ME7gBuAy4DyMyngfcDc4spkVcD/5SZP912J5n5LI1w2bvHKh+4ZtL4t+7sh8C/AifT+KXcWtyai+V/C7QByyNiKfBvgGfzfVRxgjUdODciVgErgQ3AJ3u1sB7gKJpeUoyB/x/AyZm5prfrkVQ/Brwk1ZRdNJJUUwa8JNWUAS9JNWXAS1JNGfCqlR3MCjm6GMrYXfu/KiLGFY8/2am9W48jdQcDXnWz7ayQ3SYiBmXmn2Xm8qKp9uOo1b8Z8KqNYlbIacAHaVzkfdvl+0TEDRGxPCJuioh7I6KlWDYzIh4o5hH/cqdtXoyIfyhmkzwhIhZGREtEfIlXv9h2XbH6oIj4VjGH/G0RMbTYx8KI+FpELI6IFRHx1oi4sZib/PNV/7to4DLgVSe7nBUS+DCwJjPHAX9HYy4gIuJg4Ms05qRpBt4aEWcX2+wL3JuZx2bmXZt3lJmX8Orc439aNB8FfKOYQ34t8Medjt2emS00pq+4mcZcQxNofFN2RHe8eGlbBrzqZKezQhambV6emUuBXxftbwUWZmZbZm4CrqMxTQE0Jov7YcnjP5qZrcXjJcDoTstuKe4fAJZl5tOZ+TLwCHBYyf1Le8TJrFQLO5sVksYkb12xITM7Sq77cqfHHcDQHSx7ZZv1XsGfQ1XEM3jVxc5mhex8dvxzYAZAMRLm94v2RcDbI+KAYqbBmcB2k8DtwMaIGNJtr0DqZga86mJns0L+Tafn/wtoKuYH/zywDHi+mOnzEuAnwP3Aksy8md2bC/y604esUp/iZGMaMIqz8yGZuSEijqQxpfMxmdney6VJlbDvTwPJPsBPim6VAD5suKvOPIOXpJqyD16SasqAl6SaMuAlqaYMeEmqKQNekmrq/wPR0VokKSRAxAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_mean(rewards[0],'mean rewards')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "46b5aa97",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEgCAYAAAC5LnRsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZ3klEQVR4nO3df5RcZZ3n8feHJPwaYBDSrgESGjaABCIxtEEUFXBRGAXU+CuHHY0HNwJm0fXHWXQHHPHHys6gguAwEcTIIMogQlAYYJg4CmdBOxASSEAjMoQQpSEQkkUDwc/+UTdYNNWdCulbnc7zeZ1TJ3Wf+9xb36JIPnXvfe5Tsk1ERJRrm+EuICIihleCICKicAmCiIjCJQgiIgqXIIiIKFyCICKicAmCiCEkaXtJv5B0t6R7JX1+gH7vlbSk6vO9pvYJkm6StLRa392x4qNYyn0EEUNHkoC/sL1W0hjgVuBjtm9v6rMfcCVwtO0nJL3c9qPVup8CX7J9s6SdgD/Zfrrz7yRKMnq4C4jYmrjxzWpttTimevT/tvXfgAttP1FtsyEEJgGjbd9cta8logNyaihiiEkaJWkh8Chws+07+nXZH9hf0m2Sbpd0bFP7k5KulnSXpL+TNKqDpUehEgQRQ8z2c7anAHsB0yQd3K/LaGA/4EhgBvAtSbtW7W8APgW8BtgXmNmRoqNoCYKImth+EpgPHNtv1cPAPNvP2v4t8CsawfAwsND2A7bXA9cAUztXcZQqQRAxhCR1Vd/ukbQDcAxwX79u19A4GkDSWBqnhB4AfgnsKqmr6nc0sKT2oqN4uVgcMbTGAXOrc/vbAFfa/rGks4Fe2/OAG4G3SFoCPAd82vbjAJI+BdxSjT5aAHxrWN5FFCXDR7dwkrYHfgZsRyO4r7L9uX59ZgJ/B6yomi6wfbGkvYEf0fgHaQzwDdsXdar2iBgZEgRbuDbHpc8EemzP7rfttjQ+43XVmPR7gNfZfqRz7yAitnQ5NbSFa3Nc+kDbPtO0uB25JhQRLeQfhhGgjXHpANMlLZJ0laTxTduOl7QIWA6ck6OBiOgvQTACtDEu/Tqg2/argJuBuU3bLq/aJwIflPSfOlR2RIwQCYIRZKBx6bYft72uWrwYOLTFto/QuEbwhprLjIgRJkGwhWtnXLqkcU2LJwBLq/a9qm2Q9DLgCOD+DpQdESNILhZv+doZl366pBOA9cAq/jwtwYHAuZIMCPh724s7/g4iYouW4aMREYXLqaGIiMIlCCIiCpcgiIgoXIIgIqJwCYKIiMIlCCIiCpcgiIgoXIIgIqJwI+7O4rFjx7q7u3u4y4iIGFEWLFjwmO2uVutGXBB0d3fT29s73GVERIwokv5joHU5NRQRUbgEQURE4RIEERGFSxBERBQuQRARUbgEQURE4RIEERGFSxBERBQuQRARUbgRd2dxlKX7jJ8MdwlbrQe/8rbhLiG2EDkiiIgoXIIgIqJwCYKIiMIlCCIiCpcgiIgoXG1BIGl7Sb+QdLekeyV9vkWfmZL6JC2sHh+uq56IiGitzuGj64Cjba+VNAa4VdINtm/v1+8HtmfXWMfzMhSxPhmKGDFy1RYEtg2srRbHVA/X9XoREfHS1HqNQNIoSQuBR4Gbbd/Rott0SYskXSVp/AD7mSWpV1JvX19fnSVHRGGWL1/OUUcdxaRJkzjooIM477zzXtRn9erVHH/88RxyyCEcdNBBXHrppQDMnz+fKVOmPP/Yfvvtueaaazr8DjZfrXcW234OmCJpV+BHkg62fU9Tl+uAK2yvk/QRYC5wdIv9zAHmAPT09OSoIiKGzOjRozn33HOZOnUqa9as4dBDD+WYY45h0qRJz/e58MILmTRpEtdddx19fX0ccMABnHTSSRx11FEsXLgQgFWrVjFx4kTe8pa3DNM7eek6MmrI9pPAfODYfu2P215XLV4MHNqJeiIiNhg3bhxTp04FYOedd+bAAw9kxYoVL+gjiTVr1mCbtWvXsttuuzF69Au/R1911VUcd9xx7Ljjjh2rfajUOWqoqzoSQNIOwDHAff36jGtaPAFYWlc9EREb8+CDD3LXXXdx2GGHvaB99uzZLF26lD322IPJkydz3nnnsc02L/zn8/vf/z4zZszoZLlDps4jgnHAfEmLgF/SuEbwY0lnSzqh6nN6NbT0buB0YGaN9UREDGjt2rVMnz6dr3/96+yyyy4vWHfjjTcyZcoUHnnkERYuXMjs2bN56qmnnl+/cuVKFi9ezFvf+tZOlz0k6hw1tAh4dYv2s5qefwb4TF01RES049lnn2X69OmcdNJJvOtd73rR+ksvvZQzzjgDSUycOJF99tmH++67j2nTpgFw5ZVX8s53vpMxY8Z0uvQhkTuLI6Jotjn55JM58MAD+cQnPtGyz4QJE7jlllsA+P3vf8/999/Pvvvu+/z6K664YsSeFoL8HkFEFO62227jsssuY/LkyUyZMgWAL3/5yzz00EMAnHLKKZx55pnMnDmTyZMnY5tzzjmHsWPHAo3rCsuXL+dNb3rTcL2FzZYgiIiiHXHEETTufx3YHnvswU033dRyXXd394tGGY00OTUUEVG4BEFEROESBBERhUsQREQULkEQEVG4BEFEROESBBERhUsQREQULkEQEVG4BEFEROESBBERhUsQREQULkEQEVG4BEFEROESBBERhUsQREQUrrYgkLS9pF9Iurv6gfrPt+iznaQfSFom6Q5J3XXVExERrdV5RLAOONr2IcAU4FhJr+3X52TgCdsTga8B59RYT0REtFBbELhhbbU4pnr0/z24E4G51fOrgDdLUl01RUTEi9V6jUDSKEkLgUeBm23f0a/LnsByANvrgdXA7i32M0tSr6Tevr6+OkuOiChOrUFg+znbU4C9gGmSDn6J+5lju8d2T1dX15DWGBFRuo6MGrL9JDAfOLbfqhXAeABJo4G/BB7vRE0REdFQ56ihLkm7Vs93AI4B7uvXbR7wwer5u4F/s93/OkJERNRodI37HgfMlTSKRuBcafvHks4Gem3PAy4BLpO0DFgFvL/GeiIiooXagsD2IuDVLdrPanr+R+A9ddUQEREblzuLIyIKV+epoYgoUPcZPxnuErZaD37lbbXsN0cEERGFSxBERBQuQRARUbgEQURE4RIEERGFSxBERBQuQRARUbgEQURE4RIEERGFSxBERBQuQRARUbgEQURE4RIEERGFSxBERBQuQRARUbgEQURE4RIEERGFqy0IJI2XNF/SEkn3SvpYiz5HSlotaWH1OKvVviIioj51/lTleuCTtu+UtDOwQNLNtpf06/dz22+vsY6IiBhEbUcEtlfavrN6vgZYCuxZ1+tFRMRL05FrBJK6gVcDd7RYfbikuyXdIOmgAbafJalXUm9fX1+dpUZEFKf2IJC0E/BD4OO2n+q3+k5gb9uHAN8Armm1D9tzbPfY7unq6qq13oiI0rR1jUDSnsDezf1t/6yN7cbQCIHLbV/df31zMNi+XtI3JY21/Vg7dUVExObbaBBIOgd4H7AEeK5qNjBoEEgScAmw1PZXB+jzCuD3ti1pGo0jlMfbLz8iIjZXO0cE7wAOsL1uE/f9euCvgcWSFlZtnwUmANi+CHg3cKqk9cAfgPfb9ia+TkREbIZ2guABYAywSUFg+1ZAG+lzAXDBpuw3IiKGVjtB8DSwUNItNIWB7dNrqyoiIjqmnSCYVz0iImIrtNEgsD1X0rbA/lXT/bafrbesiIjolHZGDR0JzAUepHHOf7ykD7YzfDQiIrZ87ZwaOhd4i+37ASTtD1wBHFpnYRER0Rnt3Fk8ZkMIANj+FY1RRBERsRVo54igV9LFwD9VyycBvfWVFBERndROEJwKfBTYMFz058A3a6soIiI6qp1RQ+uAr1aPiIjYygwYBJKutP1eSYtpzC30ArZfVWtlERHREYMdEWz4acn8elhExFZswFFDtldWT0+z/R/ND+C0zpQXERF1a2f46DEt2o4b6kIiImJ4DHaN4FQa3/z3lbSoadXOwG11FxYREZ0x2DWC7wE3AP8bOKOpfY3tVbVWFRERHTNgENheDawGZgBIejmwPbCTpJ1sP9SZEiMiok4bvUYg6XhJvwZ+C/w7jcnnbqi5roiI6JB2LhZ/EXgt8Cvb+wBvBm6vtaqIiOiYdoLgWduPA9tI2sb2fKCn5roiIqJD2gmCJyXtBPwMuFzSecD/29hGksZLmi9piaR7JX2sRR9JOl/SMkmLJE3d9LcQERGbo50gOBH4A/A/gH8BfgMc38Z264FP2p5E49TSRyVN6tfnOGC/6jEL+Ic2646IiCHSzqRzzd/+57a74+rO5JXV8zWSlgJ7Akuaup0IfNe2gdsl7SppXNNdzRERUbMBjwgk3Vr9uUbSU02PNZKe2pQXkdQNvBq4o9+qPYHlTcsPV20REdEhg91HcET1586b8wLV9YUfAh+3vUkB0rSPWTROHTFhwoTNKSciIvpp5z6C8yUd/lJ2LmkMjRC43PbVLbqsAMY3Le9Vtb2A7Tm2e2z3dHV1vZRSIiJiAO1cLF4AnCnpN5L+XlJbQ0clCbgEWGp7oB+1mQd8oBo99Fpgda4PRER0VjsXi+cCcyXtBkwHzpE0wfZ+G9n09cBfA4slLazaPgtMqPZ7EXA98FfAMuBp4EMv5U1ERMRL185vFm8wEXglsDewdGOdbd8KaCN9TOP3kCMiYpi0c43g/1RzDZ0NLAZ6bLdzH0FERIwA7RwR/AY43PZjdRcTERGd187F4m8Bx0o6C0DSBEnT6i0rIiI6pZ0guBA4nOp3CYA1VVtERGwF2jk1dJjtqZLuArD9hKRta64rIiI6pK1pqCWNAgwgqQv4U61VRUREx7QTBOcDPwJeLulLwK3Al2utKiIiOqadG8oul7SAxi+TCXiH7Y3eRxARESPDgEFQ3Um8waPAFc3rbK+qs7CIiOiMwY4IFtC4LiAa00I8UT3fFXgI2Kfu4iIion4DXiOwvY/tfYF/BY63Pdb27sDbgZs6VWBERNSrnYvFr7V9/YYF2zcAr6uvpIiI6KR27iN4RNLfAP9ULZ8EPFJfSRER0UntHBHMALpoDCG9uno+Y9AtIiJixGhn+Ogq4GMdqCUiIoZBO0cEERGxFUsQREQULkEQEVG4we4s/gbVRHOt2D69looiIqKjBjsi6KVxd/H2wFTg19VjCpBpqCMithIDHhHYngsg6VTgCNvrq+WLgJ9vbMeSvk3jLuRHbR/cYv2RwLXAb6umq22fvYn1R0TEZmrnhrKXAbsAGyaZ26lq25jvABcA3x2kz89tv72NfUVERE3aCYKvAHdJmk9j0rk3Ap/f2Ea2fyape/PKi4iIurVzQ9mlkm4ADqua/qft3w3R6x8u6W4aU1Z8yva9rTpJmgXMApgwYcIQvXREREAbw0cl3WL7d7avrR6/k3TLELz2ncDetg8BvgFcM1BH23Ns99ju6erqGoKXjoiIDQYMAknbVz9OM1bSyyTtVj26gT0394VtP2V7bfX8emCMpLGbu9+IiNg0g50a+gjwcWAPGsNIVbU/ReMi8GaR9Arg97YtaRqNUHp8c/cbERGbZrDho+cB50n677a/sak7lnQFcCSNI4qHgc8BY6p9XwS8GzhV0nrgD8D7bQ94A1tERNSjnVFDv5O0s+011e8STAW+aPvOwTayPehU1bYvYAiOLCIiYvO0M9fQmVUIHAH8F+AS4B/qLSsiIjqlnSB4rvrzbcAc2z8hU0xERGw12gmCFZL+EXgfcL2k7drcLiIiRoB2/kF/L3Aj8FbbTwK7AZ+us6iIiOicdu4sfprGbxVvWF4JrKyzqIiI6Jyc4omIKFyCICKicAmCiIjCJQgiIgqXIIiIKFyCICKicAmCiIjCJQgiIgqXIIiIKFyCICKicAmCiIjCJQgiIgqXIIiIKFyCICKicLUFgaRvS3pU0j0DrJek8yUtk7RI0tS6aomIiIHVeUTwHeDYQdYfB+xXPWaR30GOiBgWtQWB7Z8BqwbpciLwXTfcDuwqaVxd9URERGvDeY1gT2B50/LDVduLSJolqVdSb19fX0eKi4goxYi4WGx7ju0e2z1dXV3DXU5ExFZlOINgBTC+aXmvqi0iIjpoOINgHvCBavTQa4HVtlcOYz0REUUaXdeOJV0BHAmMlfQw8DlgDIDti4Drgb8ClgFPAx+qq5aIiBhYbUFge8ZG1hv4aF2vHxER7RkRF4sjIqI+CYKIiMIlCCIiCpcgiIgoXIIgIqJwCYKIiMIlCCIiCpcgiIgoXIIgIqJwCYKIiMIlCCIiCpcgiIgoXIIgIqJwCYKIiMIlCCIiCpcgiIgoXIIgIqJwCYKIiMIlCCIiCldrEEg6VtL9kpZJOqPF+pmS+iQtrB4frrOeiIh4sdp+vF7SKOBC4BjgYeCXkubZXtKv6w9sz66rjoiIGFydRwTTgGW2H7D9DPB94MQaXy8iIl6COoNgT2B50/LDVVt/0yUtknSVpPE11hMRES0M98Xi64Bu268CbgbmtuokaZakXkm9fX19HS0wImJrV2cQrACav+HvVbU9z/bjttdVixcDh7bake05tnts93R1ddVSbEREqeoMgl8C+0naR9K2wPuBec0dJI1rWjwBWFpjPRER0UJto4Zsr5c0G7gRGAV82/a9ks4Gem3PA06XdAKwHlgFzKyrnoiIaK22IACwfT1wfb+2s5qefwb4TJ01RETE4Ib7YnFERAyzBEFEROESBBERhUsQREQULkEQEVG4BEFEROESBBERhUsQREQULkEQEVG4BEFEROESBBERhUsQREQULkEQEVG4BEFEROESBBERhUsQREQULkEQEVG4BEFEROESBBERhUsQREQUrtYgkHSspPslLZN0Rov120n6QbX+DkndddYTEREvVlsQSBoFXAgcB0wCZkia1K/bycATticCXwPOqaueiIhorc4jgmnAMtsP2H4G+D5wYr8+JwJzq+dXAW+WpBprioiIfkbXuO89geVNyw8Dhw3Ux/Z6SauB3YHHmjtJmgXMqhbXSrq/loq3PGPp999iS6Ucy22Qz2xkGTGfF2z2Z7b3QCvqDIIhY3sOMGe46+g0Sb22e4a7jmhfPrORJZ9XQ52nhlYA45uW96raWvaRNBr4S+DxGmuKiIh+6gyCXwL7SdpH0rbA+4F5/frMAz5YPX838G+2XWNNERHRT22nhqpz/rOBG4FRwLdt3yvpbKDX9jzgEuAyScuAVTTCIv6suNNhW4F8ZiNLPi9A+QIeEVG23FkcEVG4BEFEROESBB0m6TlJCyXdK+luSZ+UtE3T+iMk/ULSfdX0HKc1rftbSU9LenlT29pOv4eSSXqHJEt6ZbU8RdL/rT7PRZLe19R3jKSvSPq1pDurfscNX/Vlafq7do+kf5a040ba95J0bfV5/UbSedVAl61egqDz/mB7iu2DgGNoTMHxOQBJrwC+B5xi+5XA64GTJb2zafvHgE92uOb4sxnArdWfAE8DH6g+z2OBr0vatVr3BWAccLDtqcA7gJ07Wm3ZNvxdOxh4BjhloPZqRoOrgWts7wfsD+wEfGk4Cu+0XCzuMElrbe/UtLwvjaG2Y4GzAds+q2n9m4Ev2H6dpL+tmmcCU22v6r+/qI+knYD7gaOA62wf0KLP3TSGQq+gcdf8Praf6mihAbzw75qkU4BX2T6tVTvwQ+Bztt/YtP0uwG+B8baf7vw76JwcEQwz2w/QGF77cuAgYEG/Lr00Ju3bYC3wbeBjHSkwmp0I/IvtXwGPSzq0eaWkacC2wG+AicBDCYHhV92sehyweJD2F/3dqz67h2h8llu1BMHIdD7wQUk5zdBZM2hMnkj154bTQ0gaB1wGfMj2n4ahtnixHSQtpPFl6iEa9y0N1l6sETHX0NasOjX0HPAosAQ4FLi2qcuhNP6HfZ7tJyV9D/hop+osnaTdgKOByZJM4yjOkj5N47z/T4D/Zfv2apNlwARJu+SoYNj8wfaUdtolLaFxSq+5bRdgAo3PcquWI4JhJKkLuAi4oJpa40JgpqQp1frdaVys+kKLzb8KfISEeae8G7jM9t62u22Pp3H++A3Aj4Dv2r5qQ+fqnPIlwPMjTyR1SXrPMNQeG3cLsKOkD8Dzv6dyLvCdrf36ACQIhsMOG4aPAv8K3AR8HsD2SuC/AnOqqbYfAc63/e/9d2L7MRr/AG3XscrLNoPGf+9mP6TxexpvpBHgC6vHlGr93wB9wBJJ9wA/BnJ0sAWqvoi9E3iPpF8DvwL+CHx2WAvrkIwa2oJV9xCcCrzR9hPDXU9EbJ0SBBERhcupoYiIwiUIIiIKlyCIiChcgiAionAJgihOixlEu6vhnUO1/4slTaqef7apfUhfJ2KoJAiiRP1nEB0ykkbZ/rDtJVVTEePQY2RLEERRqhlEjwBOpsVvZEvaUdKVkpZI+pGkOyT1VOtmSFpczWN/TtM2ayWdW808erikn0rqkfQV/nwD4eVV91GSvlX9fsFNknao9vFTSV+T1CtpqaTXSLq6mhv/i3X/d4myJQiiNIPOIAqcBjxhexJwJo25npC0B3AOjfmGpgCvkfSOapu/AO6wfYjtWzfsyPYZ/Hnu+5Oq5v2AC6vfL3gSmN702s/Y7qEx7ci1NOaSOpjGXcu7D8Wbj2glQRClGXAG0coRG9bbvgdYVLW/Bvip7T7b64HLaUwtAY1JA3/Y5uv/1vbC6vkCoLtp3bzqz8XAvbZX2l4HPACMb3P/EZssE5ZFMQaaQZTGZH+b44+2n2uz77qm588BO7RY96d+/f5E/q5GjXJEECUZaAbR5m/btwHvBahG/kyu2n8BvEnS2GpmyhnAiyYDbOFZSWOG7B1E1CBBECUZaAbRzzQtfxPoquan/yJwL7C6mhn2DGA+cDewwPa1bNwcYFHTxeKILU4mnYtoUn3bH2P7j5L+M42pwg+w/cwwlxZRm5x3jHihHYH51ekcAaclBGJrlyOCiIjC5RpBREThEgQREYVLEEREFC5BEBFRuARBRETh/j/UYuOE9WD6UgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_mean(rewards[1],'std deviation')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "719d552f",
   "metadata": {},
   "source": [
    "Hence, we see that while DQN has the highest average reward, it also has the second highest standard deviation:\n",
    "<ol><li> 6% higher reward than A2C but 0.96 the std deviation </li>\n",
    "    <li>15% higher reward than PPO but 122% the std deviation </li>\n",
    "    <li> Similarly, A2C has 9% higher reward than PPO but 1.3X its std deviation </li>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}