2b_Breakout-v0-recurrent_Evaluate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7d1fa20c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys\n",
    "import gc\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83d1313d",
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.append('../')\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c8c767df",
   "metadata": {},
   "outputs": [],
   "source": [
    "from get_algos import get_all_algos , run_experiment ,call_paths, plot_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c9fc3531",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "from gym.wrappers import Monitor\n",
    "from sb3_contrib import QRDQN ,RecurrentPPO, TRPO \n",
    "from stable_baselines3.common.vec_env import VecFrameStack\n",
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "from stable_baselines3.common.env_util import make_atari_env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5f7370b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "env_name='Breakout-v0'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54c5cdd5",
   "metadata": {},
   "source": [
    "### 2. Evaluation\n",
    "<ol>\n",
    "    <li>Create the environment</li>\n",
    "    <li>For each algorithm:</li>\n",
    "    <ol><li>Load the model</li>\n",
    "        <li>Evaluate the model for 5 sample iterations </li>\n",
    "        <li>Output the score  for each algo and each iteration</li>\n",
    "        <li>Assess stability of the model based on the variance / standard deviation </li>\n",
    "    </ol>\n",
    "    <li>Plot the corresponding scores</li>\n",
    "</ol> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "65fae03b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['QRDQN', 'RecurrentPPO', 'TRPO']\n"
     ]
    }
   ],
   "source": [
    "env_recurrent_type={'Discrete':['MaskablePPO', 'QRDQN', 'RecurrentPPO', 'TRPO'],'Box':['ARS', 'RecurrentPPO', 'TQC', 'TRPO'], \n",
    "                    'MultiDiscrete':['MaskablePPO', 'RecurrentPPO', 'TRPO'],'MultiBinary':['MaskablePPO', 'RecurrentPPO', 'TRPO']}\n",
    "def get_algo_recurrent(env):\n",
    "    act_space=str(type(env.action_space)).split('.')[3].split(\"'\")[0]\n",
    "    return env_recurrent_type[act_space]\n",
    "\n",
    "algo_list=get_algo_recurrent(gym.make(env_name))\n",
    "algo_list.remove('MaskablePPO') if 'MaskablePPO' in algo_list else algo_list\n",
    "print(algo_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "52178f03",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_experiment(algo_list, env_name,n_steps=200000,n_episodes=100):\n",
    "    rewards_mean={}\n",
    "    rewards_std={}\n",
    "    for algo_name in reversed(algo_list):\n",
    "        if algo_name =='QRDQN' or 'TRPO':\n",
    "            device='cpu'\n",
    "        else:\n",
    "            device='cuda' #GPU was unable to load the entire experience relay so moved to virtual memory on disk\n",
    "        policy='CnnLstmPolicy' if algo_name=='RecurrentPPO' else 'CnnPolicy'   \n",
    "        env=make_atari_env(env_name,n_envs=6)\n",
    "        env=VecFrameStack(env, n_stack=6)\n",
    "        print('******* Evaluating', algo_name,' *******')\n",
    "        log_path, render_path, model_path=call_paths(algo_name,env_name,n_steps)\n",
    "        model_name=env_name+\"_\"+algo_name+\"_model\"\n",
    "        model=eval(algo_name).load(os.path.join(model_path,model_name),env,device=device)\n",
    "        mean_rew, mean_std=evaluate_policy(model,env, n_eval_episodes=n_episodes)\n",
    "        rewards_mean[algo_name]=mean_rew\n",
    "        rewards_std[algo_name]=mean_std\n",
    "        print('Mean rewards for ',n_episodes, ' episodes of ',algo_name,' is ',mean_rew,'\\n')\n",
    "        del model\n",
    "        env.close()\n",
    "    return rewards_mean,rewards_std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "66a2bd41",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "******* Evaluating TRPO  *******\n",
      "Wrapping the env in a VecTransposeImage.\n",
      "Mean rewards for  100  episodes of  TRPO  is  2.06 \n",
      "\n",
      "******* Evaluating RecurrentPPO  *******\n",
      "Wrapping the env in a VecTransposeImage.\n",
      "Mean rewards for  100  episodes of  RecurrentPPO  is  4.39 \n",
      "\n",
      "******* Evaluating QRDQN  *******\n",
      "Wrapping the env in a VecTransposeImage.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\ai_gym\\lib\\site-packages\\stable_baselines3\\common\\buffers.py:229: UserWarning: This system does not have apparently enough memory to store the complete replay buffer 84.69GB > 10.70GB\n",
      "  \"This system does not have apparently enough memory to store the complete \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean rewards for  100  episodes of  QRDQN  is  2.12 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "rewards=evaluate_experiment(algo_list, env_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0a4d7aa0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEcCAYAAADN+K/qAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAAAXU0lEQVR4nO3de5RdZZnn8e9DAsidAYKkCRhugwkCCQkKiBfQ2DbQSAPdGFGmZyEMKg20iq2C0+qyabCDRoQeJw3CgmYUZ0RBAfHCpbkImJAQIFwiEIE0NGklQJBbwjN/7F3kpFKp7KRq1+XN97NWrTr7/pxz6vzqPe/Z+z2RmUiSyrPeYBcgSWqHAS9JhTLgJalQBrwkFcqAl6RCGfCSVCgDXloDETEiImZHxE97WHZSRNwbEXMi4taIGF/P3yAiLq6X3RMR7x3ourVuMuClNXMq8MAqlv2fzNwzMycAXwe+Uc8/ASAz9wSmAOdGhK89tc4/MqmhiBgDHApc2NPyzHy+Y3IToOsqwvHADfU6zwCLgcmtFSrVDHipuenA54DXV7VCRHwqIh6hasGfUs++Bzg8IkZGxE7AJGCHlmuVDHipiYg4DHgmM2f1tl5mXpCZuwB/B5xZz/4u8CQwk+qfxO3AsvaqlSrhWDTS6kXEPwIfA5YCbwI2B67MzI+uYv31gGczc4selt0OfDwz57VYsmQLXmoiM7+QmWMycyzwYeCG7uEeEbt1TB4KzK/nbxwRm9S3pwBLDXcNhJGDXYA0nEXEV4GZmXk1cHJEvB94DXgW+G/1atsC10fE68BCqncCUuvsopGkQtlFI0mFMuAlqVAGvCQVyoCXpEIZ8JJUKANekgplwEtSoQx4SSqUAS9JhTLgJalQBrwkFcqAl6RCDanRJLfZZpscO3bsYJchScPGrFmz/jMzR/W0bEgF/NixY5k5c+ZglyFJw0ZE/G5Vy+yikaRCGfCSVCgDXpIKZcBLUqEMeEkqlAEvSYUy4CWpUAa8JBXKgJekQg2pK1m17hj7+WsGu4RiLTj70MEuQUOELXhJKpQBL0mFMuAlqVAGvCQVyoCXpEIZ8JJUKANekgplwEtSoQx4SSqUAS9JhTLgJalQBrwkFcqAl6RCtR7wETEiImZHxE/bPpYkabmBaMGfCjwwAMeRJHVoNeAjYgxwKHBhm8eRJK2s7Rb8dOBzwOstH0eS1E1rAR8RhwHPZOas1ax3YkTMjIiZixYtaqscSVrntNmCfydweEQsAL4PHBwR/9p9pcyckZmTM3PyqFGjWixHktYtrQV8Zn4hM8dk5ljgw8ANmfnRto4nSVqR58FLUqFGDsRBMvMm4KaBOJYkqWILXpIKZcBLUqEMeEkqlAEvSYUy4CWpUAa8JBXKgJekQhnwklQoA16SCmXAS1KhDHhJKpQBL0mFMuAlqVAGvCQVyoCXpEIZ8JJUKANekgplwEtSoQx4SSqUAS9JhTLgJalQBrwkFcqAl6RCGfCSVCgDXpIKZcBLUqEMeEkqlAEvSYUy4CWpUAa8JBXKgJekQhnwklQoA16SCmXAS1KhDHhJKpQBL0mFMuAlqVAGvCQVqrWAj4g3RcRdEXFPRNwfEV9p61iSpJWNbHHfrwAHZ+aSiFgfuDUirsvMO1o8piSp1lrAZ2YCS+rJ9eufbOt4kqQVrbaLJiL+MiI2q2+fGRFXRsQ+TXYeESMiYg7wDPCLzLyzT9VKkhpr0gf/pcx8ISIOBN4PXAT8ryY7z8xlmTkBGAO8PSLe1n2diDgxImZGxMxFixatQemSpN40Cfhl9e9DgRmZeQ2wwZocJDMXAzcCH+xh2YzMnJyZk0eNGrUmu5Uk9aJJwC+MiP8NHANcGxEbNtkuIkZFxJb17Y2AKcCDfahVkrQGmgT8XwHXA39at8S3Ak5vsN1o4MaImAv8hqoP/qdrW6gkac2s8iyaiNiqY/KmjnmvADNXt+PMnAtM7GN9kqS11NtpkrOoTmsMYEfg2fr2lsDjwE5tFydJWnur7KLJzJ0yc2fgl8CfZ+Y2mbk1cBjw84EqUJK0dpr0we+Xmdd2TWTmdcAB7ZUkSeoPTa5k/feIOBP413r6WODf2ytJktQfmrTgpwKjgB8BV9a3p7ZZlCSp73oN+IgYAXw7M0/NzImZuU9mnpaZfxig+iRprTzxxBMcdNBBjB8/nj322INvfetbK63z4IMPsv/++7Phhhsybdq0Ndp2OOi1iyYzl0XEWyJig8x8daCKkqS+GjlyJOeeey777LMPL7zwApMmTWLKlCmMHz/+jXW22morzjvvPH784x+v8bbDQZM++EeB2yLiauDFrpmZ+Y3WqpKkPho9ejSjR48GYLPNNmPcuHEsXLhwhZDedttt2XbbbbnmmmvWeNvhoEkf/CPAT+t1N+v4UR80eQuYmZxyyinsuuuu7LXXXtx9991vLHv88cf5wAc+wLhx4xg/fjwLFiwYwOql4WXBggXMnj2bd7zjHQO67WBbbQs+M/0mphY0eQt43XXXMX/+fObPn8+dd97JJz7xCe68sxpx+bjjjuOMM85gypQpLFmyhPXW89sXpZ4sWbKEo446iunTp7P55psP2LZDwWoDPiJGAZ8D9gDe1DU/Mw9usa7iNXkLeNVVV3HccccREey3334sXryYp556imeffZalS5cyZcoUADbddNNBuQ/SUPfaa69x1FFHceyxx3LkkUcO2LZDRZNm3+VUo0DuBHwFWEA1eJj6yareAi5cuJAddtjhjekxY8awcOFCHn74YbbcckuOPPJIJk6cyOmnn86yZcu671Zap2Umxx9/POPGjePTn/70gG07lDT5kHXrzLwoIk7NzJuBmyPCgO8na/MWcOnSpdxyyy3Mnj2bHXfckWOOOYZLLrmE448/vuVqpeHjtttu47LLLmPPPfdkwoQJAJx11lk8/vjjAJx00kk8/fTTTJ48meeff5711luP6dOnM2/ePObOndvjtocccsgg3Zu10yTgX6t/PxURh1JdxbpVL+urodW9Bdx+++154okn3ph+8skn2X777Vm6dCkTJkxg5513BuCII47gjjvuMOClDgceeCDVV0Ov2nbbbceTTz65VtsOB026aL4WEVsAnwE+C1wI/G2rVa0DmrwFPPzww7n00kvJTO644w622GILRo8ezb777svixYvp+orDG264YdidviWpfU1a8L/MzJeB54CDWq5nndHk7eMhhxzCtddey6677srGG2/MxRdfDMCIESOYNm0a73vf+8hMJk2axAknnDBYd0XSENUk4O+LiP8Abql/bs3M59otq3xN3gJGBBdccEGPy6ZMmcLcuXPbKE1SIVbbRZOZu1INLnYv1Rdv3xMRc1quS5LUR03Ogx8DvBN4F7A3cD9wa8t1SZL6qEkXzeNU572flZkntVyPJKmfNDmLZiJwKfCRiPh1RFwaEZ6PJ0lDXJOxaO6JiEeoBh17F/BR4D3ARS3XJknqgyZ98DOBDYHbqc6ieXdm/q7twiRJfdOkD/7PMnNR65VIkvpVkz749SLiooi4DiAixtsHL0lDX5OAvwS4HviTevph4LSW6pEk9ZMmAb9NZv4AeB0gM5cCjk0rSUNck4B/MSK2BhIgIvajGpdGkjSENfmQ9dPA1cAuEXEbMAo4utWqJEl91mvAR8QIqnPe3wPsDgTwUGa+1tt2kqTB12sXTWYuA6Zm5tLMvD8z7zPcJWl4aNJFc1tEnA9cAbzYNTMz726tKklSnzUJ+An17692zEvg4H6vRpLUb5qMReO3OEnSMNTkNElJ0jBkwEtSoQx4SSpUkw9ZiYgDgLGd62fmpS3VJGkIGvv5awa7hGItOPvQVvbbZDz4y4BdgDksH4Mmqb7lSZI0RDVpwU8GxmdmrsmOI2IHqn8Cb6b6hzAjM7+15iVKktZGk4C/D9gOeGoN970U+Exm3h0RmwGzIuIXmTlvTYtswreP7Wnr7aOkdjUJ+G2AeRFxF/BK18zMPLy3jTLzKep/Cpn5QkQ8AGwPtBLwkqQVNQn4L/f1IBExFpgI3NnDshOBEwF23HHHvh5KklRrciXrzX05QERsCvwQOC0zn+9h/zOAGQCTJ09eo35+SdKqrfY8+IjYLyJ+ExFLIuLViFgWESsF9Sq2XZ8q3C/PzCv7WqwkqbkmFzqdD0wF5gMbAR8HLljdRhERwEXAA5n5jb4UKUlac42uZM3M3wIjMnNZZl4MfLDBZu8EPgYcHBFz6p9D+lCrJGkNNPmQ9Y8RsQEwJyK+TnVmzGr/MWTmrVTfACVJGgRNWvAfq9c7meoLP3YAjmqzKElS3zU5i+Z3EbERMDozvzIANUmS+kGTs2j+nGocmp/V0xMi4uqW65Ik9VGTLpovA28HFgNk5hxgp9YqkiT1iyYB/1pmPtdtnhckSdIQ1+Qsmvsj4iPAiIjYDTgFuL3dsiRJfdWkBf83wB5UA419D3geOK3FmiRJ/aDJWTR/BM6ofyRJw0STb3SaDHyRlb+yb6/2ypIk9VWTPvjLgdOBe4HX2y1HktRfmgT8osz0vHdJGmaaBPzfR8SFwK9Y8RudHP5XkoawJgH/34G3AuuzvIsmAQNekoawJgG/b2bu3nolkqR+1eQ8+NsjYnzrlUiS+lWTFvx+VGPBP0bVBx9AepqkJA1tTQK+ybc3SZKGmEbjwQ9EIZKk/tXoO1klScOPAS9JhTLgJalQBrwkFcqAl6RCGfCSVCgDXpIKZcBLUqEMeEkqlAEvSYUy4CWpUAa8JBXKgJekQhnwklQoA16SCmXAS1KhDHhJKpQBL0mFMuAlqVAGvCQVqrWAj4jvRsQzEXFfW8eQJK1amy34S4APtrh/SVIvWgv4zPw34A9t7V+S1Dv74CWpUIMe8BFxYkTMjIiZixYtGuxyJKkYgx7wmTkjMydn5uRRo0YNdjmSVIxBD3hJUjvaPE3ye8Cvgd0j4smIOL6tY0mSVjayrR1n5tS29i1JWj27aCSpUAa8JBXKgJekQhnwklQoA16SCmXAS1KhDHhJKpQBL0mFMuAlqVAGvCQVyoCXpEIZ8JJUKANekgplwEtSoQx4SSqUAS9JhTLgJalQBrwkFcqAl6RCGfCSVCgDXpIKZcBLUqEMeEkqlAEvSYUy4CWpUAa8JBXKgJekQhnwklQoA16SCmXAS1KhDHhJKpQBL0mFMuAlqVAGvCQVyoCXpEIZ8JJUKANekgplwEtSoVoN+Ij4YEQ8FBG/jYjPt3ksSdKKWgv4iBgBXAD8GTAemBoR49s6niRpRW224N8O/DYzH83MV4HvAx9q8XiSpA5tBvz2wBMd00/W8yRJA2DkYBcQEScCJ9aTSyLiocGsZ4BsA/znYBfRVJwz2BUMCcPmOfP5esO68py9ZVUL2gz4hcAOHdNj6nkryMwZwIwW6xhyImJmZk4e7DrUnM/Z8ONz1m4XzW+A3SJip4jYAPgwcHWLx5MkdWitBZ+ZSyPiZOB6YATw3cy8v63jSZJW1GoffGZeC1zb5jGGqXWqS6oQPmfDzzr/nEVmDnYNkqQWOFSBJBXKgO+jiNg6IubUP09HxMKO6ax/3xcRP4mILettxkbES/WyeRHxnYhYr162R0TcUA/xMD8ivhQRMah3cpBFxLKeHsehKiImRMQhHdN/HRGLOp7vE3qbXy87IiLmRsQDEXFvRBwxCHdlUETEmIi4qv77fzQizo+IDSPivRHxXP14PRgR0zq26XosZ9fbXR8RB3Qsj4g4s172cETcHBF7dSxfEBE/7Jg+OiIuGbA73RIDvo8y8/eZOSEzJwDfAb7ZMf1iffttwB+AT3Vs+ki9zl5UQzkcEREbUZ1pdHZm7g7sDRwAfHLA7tDQ9FIvj2O/ioiRvU03NAE4pNu8K+rn+73AWRHx5lXNj4i9gWnAhzJzHHA4MK0zkEpVN2auBH6cmbsBuwEbAV+vV7mlfrwmAodFxDs7Nr8iMyfW250NXBkR4+pln6J6Le2dmf8V+Afg6ojYpGP7SaUNp2LAD5xf08OVvJm5FLgd2BX4CHBbZv68XvZH4GTAgdqWe+NxjIhdIuJnETErIm6JiLfW898cET+KiHvqnwPqd033de0kIj4bEV+ub98UEdMjYiZwag/Tk+oW36y6ZTi6Y7tzIuKuulX4rvqU4K8Cx9QtzWM6i8/MZ4BH6HZxSrf5nwXOyszH6mWPAf8InN7fD+YQdDDwcmZeDJCZy4C/BY4DNu1aKTNfAuawiqvjM/NGqg9Zuy6i/Dvg5Po1Rf0auwU4tmOzc4Ez+vG+DDoDfgBENfDa++jhOoCI2Lhedi+wBzCrc3lmPgJsGhGbD0CpQ1oPj+MM4G8ycxJVKP5zPf884ObM3BvYB2hyeu4GmTk5M8/tnK739W3g6Po436Vq/XUZmZlvB04D/r4ed+l/UrfMM/OKbvdhZ2Bn4Le9zF/p7wCYWc8vXU+vgeeBBVSNIAAi4r9Qte7/rZd93Q28tX7tbJKZj3ZbPpPq3XOXHwD7RMSuFGLQhyoo3EYRMYeqlfEA8IuOZbvUyxK4KjOvi4gpA1/isLDS4xgRm1K95f6/HR9RbFj/PpiqxdfVAnyuDoTeXLGK6d2Bt9XHhOqajqc61ruy/j0LGNvL/o+JiAOBV4D/kZl/qPe3qvnq2bsi4h6qcJ+emU/3su6aPpDLgH8CvgBct5b1DSkGfLteyswJdSv9eqp+wPPqZV198J3mAe/unFG37JbUrZh1VU+P4yXA4h4ew1VZyorvWN/UbfmLq5gO4P7M3H8V+32l/r2M3l9PV2TmyQ3nzwMmAfd0zJtEs3ciw9084OjOGXULfDvgIao++MMiYifgjoj4QWbOWcW+JgIPZObzEfFiROzcrRU/Cfh5t20uowr4+yiAXTQDoO73OwX4zGo+tLscODAi3g9Qf+h6Hss/YFqndT6OwB+BxyLiL+GNsyT2rlf9FfCJev6IiNgC+A9g26jOetoQOKzhYR8CRkXE/vX+1o+I1XWVvABstgZ3rbtpwBciYmx9zLHAF6n6iEv3K2DjiDgO3uiWOxc4H3ipa6X6c4mzqfrWVxIR76Hqf/+XetY/AefVrynq19gewP/r3C4zXwO+SdXvP+wZ8AMkM2cDc4GpvazzEtWY+WdGNarmvVRj+pw/IEUOA90ex2OB4+u37Pez/PsGTgUOioh7qbpOxtcv3K8Cd1F1lT3Y8HivUrUoz6mPM4eqa6g3NwLje/qQteEx51AF108i4kHgJ8DnemmpFiOrKy//Ajg6IuYDvwdez8x/6GH17wDv7vpHyPIPth+m+od4VGY+UC/7NtVzPzciFgCXAlMy8+Ue9nsRhfRueCWrpCErqnPZvwf8RWbe3U/73BT4EfCbzPxif+xzqDLgJalQdtFIUqEMeEkqlAEvSYUy4CWpUAa8ihLVKIzZMS7NCmPQ9MP+L+wakCoivtgxv1+PI/UHA16lmQrcSi/XG6ytiBiRmR/PzHn1rKJPsdPwZ8CrGPX5zQcCx1N9yXv35RtHxA+iGnv9RxFxZ0RMrpdNjWrc9fsi4pyObZZExLn1RU771yNITo6Is6nHyImIy+vVR0TEv0TE/RHx846rJm+KiG9GxMyoxnffNyKujGps8q+1/bho3WXAqyQfAn6WmQ8Dv4+ISd2WfxJ4NjPHA1+iGouEiPgT4ByqQcomAPvG8i/Y2AS4MzP3zsxbu3aUmZ9n+Tj1XUPO7gZckJl7AIuBozqO/Wo9OuV3gKuoxtN5G/DXEbF1f9x5qTsDXiWZCny/vv19Vu6mObBreWbeRzXkAcC+wE2Zuagen/9ylg/6tgz4Ic081jGcQPfRJbuGOL6XavCypzLzFeBRYIeG+5fWSBHjLUgRsRVVC3zPiEiqYX0TuKCPu365HnK4iVc6bi+j+iai7ste77be6/g6VEtswasURwOXZeZbMnNsZu4APMaKrePbgL8CqM+E2bOefxfwnojYph69cCpwc4NjvhYR6/fbPZD6mQGvUkylGkCq0w+pxvbu8s9UQ//OA75GNQLlc5n5FNXXIt5INQb7rMy8qsExZ1CNTnj5ateUBoGDjWmdUbfO18/MlyNiF+CXwO71kMBScez707pkY+DGulslgE8a7iqZLXhJKpR98JJUKANekgplwEtSoQx4SSqUAS9JhTLgJalQ/x9lPOQUzoMGswAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_mean(rewards[0],'mean rewards')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "46b5aa97",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEsCAYAAADO7LQXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAAAaiklEQVR4nO3dfbRddX3n8fdHiFQNlhaCUh4MKgpGHpTIUymFUkYQFG0ZNeOqMi1DBSnYaWvRER1ZtYVWbYvYMlioYBnEKSC4BJQKRlBRkhgCJKARqYZiDSiP8pCE7/xx9g2Hm3tvDnD3ucnd79daZ92z9+939vmec3Pz2Y+/napCktRdz5nqAiRJU8sgkKSOMwgkqeMMAknqOINAkjrOIJCkjjMIpEmUZPsk1yZZmuTWJCeN0efAJPcnWdw8PtTMf2XfvMVJHkjy3qF/CHVOvI5AmjxJtgG2qapFSTYHFgJvrqqlfX0OBP60qo6YYDmbAHcBe1fVv7dbtbrOLQJpElXV3VW1qHn+ILAM2PYZLOpg4AeGgIbBIJBakmQ28Brg22M075vkpiRXJpkzRvvbgQvbrE8a4a4hqQVJZgLzgY9W1SWj2l4IPFFVDyV5A/D3VbVTX/tzgf8A5lTVfw6zbnWTWwTSJEsyA7gYuGB0CABU1QNV9VDz/ApgRpKt+rocBiwyBDQsBoE0iZIEOAdYVlWfGKfPi5t+JNmL3t/hvX1d5uFuIQ2Ru4akSZRkf+A64GbgiWb2B4AdAKrqrCQnAMcBq4FHgP9ZVd9sXv8C4EfAS6vq/iGXr44yCCSp49w1JEkdZxBIUscZBJLUcQaBJHWcQSBJHWcQSFLHGQSS1HEGgSR1nEEgSR1nEEhSxxkEktRxBoEkdZxBIEkdZxBIUscZBJLUcQaBJHWcQSBJHWcQSFLHGQSS1HEGgSR1nEEgSR1nEEhSxxkEktRxBoEkdZxBIEkdZxBIUscZBJLUcQaBJHXcplNdwNO11VZb1ezZs6e6DEnaqCxcuPCeqpo1VttGFwSzZ89mwYIFU12GJG1Ukvz7eG3uGpKkjjMIJKnjDAJJ6jiDQJI6ziCQpI4zCCSp4wwCSeo4g0CSOs4gkKSO2+iuLFa3zD75S1NdwrR152mHT3UJ2kC4RSBJHddaECTZPsm1SZYmuTXJSWP0OTDJ/UkWN48PtVWPJGlsbe4aWg38SVUtSrI5sDDJ1VW1dFS/66rqiBbrkCRNoLUtgqq6u6oWNc8fBJYB27b1fpKkZ2YoxwiSzAZeA3x7jOZ9k9yU5Mokc8Z5/bFJFiRZsHLlyjZLlaTOaT0IkswELgbeW1UPjGpeBLykqnYHPgl8YaxlVNXZVTW3qubOmjXmfRUkSc9Qq0GQZAa9ELigqi4Z3V5VD1TVQ83zK4AZSbZqsyZJ0lO1edZQgHOAZVX1iXH6vLjpR5K9mnrubasmSdK62jxr6NeB3wNuTrK4mfcBYAeAqjoLOAo4Lslq4BHg7VVVLdYkSRqltSCoquuBrKfPmcCZbdUgSVo/ryyWpI4zCCSp4wwCSeo4g0CSOs4gkKSOMwgkqeMMAknqOINAkjrOIJCkjjMIJKnjDAJJ6jiDQJI6ziCQpI4zCCSp4wwCSeo4g0CSOs4gkKSOMwgkqeMMAknqOINAkjrOIJCkjjMIJKnjDAJJ6jiDQJI6ziCQpI4zCCSp4wwCSeo4g0CSOs4gkKSOMwgkqeMMAknquNaCIMn2Sa5NsjTJrUlOGqNPkpyRZHmSJUle21Y9kqSxbdrislcDf1JVi5JsDixMcnVVLe3rcxiwU/PYG/jH5qckaUha2yKoqruralHz/EFgGbDtqG5HAudXzw3AFkm2aasmSdK6hnKMIMls4DXAt0c1bQv8uG96BeuGhSSpRa0HQZKZwMXAe6vqgWe4jGOTLEiyYOXKlZNboCR1XKtBkGQGvRC4oKouGaPLXcD2fdPbNfOeoqrOrqq5VTV31qxZ7RQrSR3V5llDAc4BllXVJ8bpdjnwzubsoX2A+6vq7rZq2hj9/u//PltvvTWvfvWrx2y/7LLL2G233dhjjz2YO3cu119//dq2973vfcyZM4dddtmFE088kaoaVtmSNiJtbhH8OvB7wG8lWdw83pDk3Une3fS5ArgDWA58Gji+xXo2SkcffTRXXXXVuO0HH3wwN910E4sXL+bcc8/lmGOOAeCb3/wm3/jGN1iyZAm33HILN954I/Pnzx9W2ZI2Iq2dPlpV1wNZT58C3tNWDdPBAQccwJ133jlu+8yZM9c+f/jhh+ltiEESHn30UR5//HGqilWrVvGiF72o7XIlbYS8sngauPTSS9l55505/PDDOffccwHYd999Oeigg9hmm23YZptteP3rX88uu+wyxZVK2hAZBNPAW97yFm677Ta+8IUvcMoppwCwfPlyli1bxooVK7jrrru45ppruO6666a4UkkbIoNgGjnggAO44447uOeee7j00kvZZ599mDlzJjNnzuSwww7jW9/61lSXKGkDZBBs5JYvX772bKBFixbx2GOPseWWW7LDDjswf/58Vq9ezapVq5g/f767hiSNqc2xhjQJ5s2bx9e+9jXuuecetttuOz7ykY+watUqAN797ndz8cUXc/755zNjxgye97zncdFFF5GEo446imuuuYZdd92VJBx66KG88Y1vnOJPI2lDlI3t3PK5c+fWggULproMDcnsk7801SVMW3eedvhUl6AhSrKwquaO1eauIUnqOINAkjrOIJCkjjMIJHXa+sbzuuCCC9htt93Ydddd2W+//bjpppsAePTRR9lrr73YfffdmTNnDh/+8IeHWfakMggkddr6xvPacccdmT9/PjfffDOnnHIKxx57LACbbbYZ11xzzdqxvq666ipuuOGGYZU9qTx9VFKnrW88r/3222/t83322YcVK1YAvfG8Rsb6WrVqFatWrVo71tfGxi0CSRrQOeecw2GHHbZ2es2aNeyxxx5svfXWHHLIIey998Z5y3WDQJIGcO2113LOOedw+umnr523ySabsHjxYlasWMF3vvMdbrnllims8JkzCCRpPZYsWcIxxxzDZZddxpZbbrlO+xZbbMFBBx004bGGDZlBIEkT+NGPfsTv/M7v8NnPfpZXvOIVa+evXLmS++67D4BHHnmEq6++mp133nmKqnx2BjpYnGRb4CX9/avq620VJUnDsr7xvE499VTuvfdejj++dwPFTTfdlAULFnD33Xfzrne9izVr1vDEE0/w1re+lSOOOGIqP8oztt6xhpKcDrwNWAqsaWZXVb2p5drG5FhD3eJYQ+1xrKFumWisoUG2CN4MvLKqHpvUqiRJG4RBjhHcAcxouxBJ0tQYZIvgF8DiJF8F1m4VVNWJrVUlSRqaQYLg8uYhSZqG1hsEVXVekucCI+dN3V5Vq9otS5I0LOsNgiQHAucBdwIBtk/yLk8flaTpYZBdQx8H/ktV3Q6Q5BXAhcCebRYmSRqOQc4amjESAgBV9T08i0iSpo1BtggWJPkn4F+a6XcAXtElSdPEIEFwHPAeYOR00euAf2itIknSUA1y1tBjwCeahyRpmhk3CJJ8vqremuRmYJ0Biapqt1YrkyQNxURbBCc1PzfO4fQkSQMZ96yhqrq7eXp8Vf17/wM4fjjlSZLaNsjpo4eMMe+wMeY9RZJzk/w0yZj3bktyYJL7kyxuHh8aoBZJ0iSb6BjBcfTW/F+aZElf0+bANwZY9meAM4HzJ+hzXVW560mSptBExwj+L3Al8FfAyX3zH6yqn61vwVX19SSzn115kqS2TXSM4P6qurOq5jXHBR6hd/bQzCQ7TNL775vkpiRXJpkzXqckxyZZkGTBypUrJ+mtJUkwwDGCJG9M8n3gh8B8eoPPXTkJ770IeElV7Q58EvjCeB2r6uyqmltVc2fNmjUJby1JGjHIweK/APYBvldVOwIHAzc82zeuqgeq6qHm+RXAjCRbPdvlSpKenkGCYFVV3Qs8J8lzqupaYMwbID8dSV6cJM3zvZpa7n22y5UkPT2DjDV0X5KZwNeBC5L8FHh4fS9KciFwILBVkhXAh2lGLa2qs4CjgOOSrKZ3/OHtVbXOFcySpHYNEgRHAo8Cf0xv5NFfBk5d34uqat562s+kd3qpJGkKDTLoXP/a/3kt1iJJmgITXVB2fVXtn+RBnjroXICqqhe2Xp0kqXXjBkFV7d/83Hx45UiShm2Qm9efAVxYVd8aQj2tmn3yl6a6hGnrztMOn+oSJD1Dg5w+uhA4JckPknwsybM+dVSStOFYbxBU1XlV9QbgdcDtwOnNlcaSpGlgkC2CES8HdgZeAtzWTjmSpGEbZKyhv262AE4FbgbmVtUbW69MkjQUg1xQ9gNg36q6p+1iJEnDN8iuoU8Dh47cQSzJDs3YQJKkaWCQIPgUsC8wMmTEg808SdI0MMiuob2r6rVJvgtQVT9P8tyW65IkDclAw1An2YRmmIkks4AnWq1KkjQ0gwTBGcClwNZJPgpcD/xlq1VJkoZmkNFHL0iykN6dyQK8uaqWtV6ZJGkoJhp99Ff7Jn8KXNjfVlU/a7MwSdJwTLRFsJDecYEAOwA/b55vAfwI2LHt4iRtfBzcsT1tDe447jGCqtqxql4K/Bvwxqraqqq2BI4AvtJKNZKkoRvkYPE+VXXFyERVXQns115JkqRhGuQ6gv9I8kHgX5rpdwD/0V5JkqRhGmSLYB4wi94ppJc0zye8Mb0kaeMxyOmjPwNOGkItkqQp8HTuRyBJmoYMAknqOINAkjpuoiuLP0kz0NxYqurEViqSJA3VRFsEC+hdXfxLwGuB7zePPQCHoZakaWLcLYKqOg8gyXHA/lW1upk+C7huOOVJkto2yDGCXwFe2Dc9s5knSZoGBrmy+DTgu0mupTfo3AHAR1qtSpI0NINcUPbPSa4E9m5m/XlV/aTdsiRJw7LeXUNJvlpVP6mqy5rHT5J8dYDXnZvkp0luGac9Sc5IsjzJkiSvfSYfQJL07IwbBEl+qbk5zVZJfiXJrzaP2cC2Ayz7M8ChE7QfBuzUPI4F/nHgqiVJk2aiXUN/CLwX+DV6p5Gmmf8AcOb6FlxVX29CYzxHAudXVQE3JNkiyTZVdfcghUuSJsdEp4/+PfD3Sf6oqj7ZwntvC/y4b3pFM2+dIEhyLL2tBnbYYYcWSpGk7hrk9NGfJNkcIMkHk1wy7P35VXV2Vc2tqrmzZs0a5ltL0rQ3SBCcUlUPJtkf+G3gHCZnf/5dwPZ909s18yRJQzRIEKxpfh4OnF1VX2Jyhpi4HHhnc/bQPsD9Hh+QpOEb5IKyu5L8H+AQ4PQkmzHYaacXAgfSO+toBfBhYAZAVZ0FXAG8AVgO/AL478/kA0iSnp1BguCt9E4D/VhV3ZdkG+DP1veiqprwdpbN2ULvGahKSVJrBrmy+Bf07lU8Mn03Y5zZI0naOHljGknqOINAkjrOIJCkjjMIJKnjDAJJ6jiDQJI6ziCQpI4zCCSp4wwCSeo4g0CSOs4gkKSOMwgkqeMMAknqOINAkjrOIJCkjjMIJKnjDAJJ6jiDQJI6ziCQpI4zCCSp4wwCSeo4g0CSOs4gkKSOMwgkqeMMAknqOINAkjrOIJCkjjMIJKnjDAJJ6jiDQJI6rtUgSHJoktuTLE9y8hjtRydZmWRx8zimzXokSevatK0FJ9kE+BRwCLACuDHJ5VW1dFTXi6rqhLbqkCRNrM0tgr2A5VV1R1U9DnwOOLLF95MkPQNtBsG2wI/7plc080b73SRLkvxrku3HWlCSY5MsSLJg5cqVbdQqSZ011QeLvwjMrqrdgKuB88bqVFVnV9Xcqpo7a9asoRYoSdNdm0FwF9C/hr9dM2+tqrq3qh5rJv8J2LPFeiRJY2gzCG4EdkqyY5LnAm8HLu/vkGSbvsk3ActarEeSNIbWzhqqqtVJTgC+DGwCnFtVtyY5FVhQVZcDJyZ5E7Aa+BlwdFv1SJLG1loQAFTVFcAVo+Z9qO/5+4H3t1mDJGliU32wWJI0xQwCSeo4g0CSOs4gkKSOMwgkqeMMAknqOINAkjrOIJCkjjMIJKnjDAJJ6jiDQJI6ziCQpI4zCCSp4wwCSeo4g0CSOs4gkKSOMwgkqeMMAknqOINAkjrOIJCkjjMIJKnjDAJJ6jiDQJI6ziCQpI4zCCSp4wwCSeo4g0CSOs4gkKSOMwgkqeMMAknqOINAkjrOIJCkjms1CJIcmuT2JMuTnDxG+2ZJLmrav51kdpv1SJLW1VoQJNkE+BRwGPAqYF6SV43q9gfAz6vq5cDfAqe3VY8kaWxtbhHsBSyvqjuq6nHgc8CRo/ocCZzXPP9X4OAkabEmSdIom7a47G2BH/dNrwD2Hq9PVa1Ocj+wJXBPf6ckxwLHNpMPJbm9lYo3PFsx6rvYUMVtuRH+zjYuG83vC5717+wl4zW0GQSTpqrOBs6e6jqGLcmCqpo71XVocP7ONi7+vnra3DV0F7B93/R2zbwx+yTZFPhl4N4Wa5IkjdJmENwI7JRkxyTPBd4OXD6qz+XAu5rnRwHXVFW1WJMkaZTWdg01+/xPAL4MbAKcW1W3JjkVWFBVlwPnAJ9Nshz4Gb2w0JM6tztsGvB3tnHx9wXEFXBJ6javLJakjjMIJKnjDIIhSLJlksXN4ydJ7uqbrubnLUm+mGSL5jWzkzzStC1NclaS5zRtc5Jc0wzf8f0kp3T9Qrwka8b6HjdUSfZI8oa+6aOTrOz7ff+PieY3bW9OsiTJsiQ3J3nzFHyUKZNkuySXNX8DdyQ5sxm25sAk9zff2W1JPtb3mpHv87vN676cZL++9iT5YNP2vSTzk+zW135nkov7po9K8pmhfeiWGARDUFX3VtUeVbUHcBbwt33TDzfPX03vgPl7+l76g6bPbvSG6XhzkufRO9vqtKp6JbA7sB9w/NA+0IbpkQm+x0nVnOo87vSA9gDeMGreRc3v+0DgL5O8aLz5SXYHPgYcWVW7AG8CPtb/n9Z01qz4XAJ8oap2AnYCngf8ddPluuY7ew1wRJJf73v5RVX1muZ1pwGXJNmlaXsPvb+n3avqFcBHgcuTvKDv9XuOMVzORs0g2LB8i97V1k9RVauBbwIvB/4b8I2q+krT9gvgBGCdQf06bO33mORlSa5KsjDJdUl2bua/KMmlSW5qHvs1W2G3jCwkyZ8m+d/N868l+bskC4CTxpjes1l7XNisZW7T97rTk3ynWcP8jfROpz4VeFuz1vq2/uKr6qfADxh1Jeio+X8K/GVV/bBp+yHwV8CfTfaXuYH6LeDRqvpngKpaA/wx8E5g5kinqnoEWMwYf1dN+7X0zhwaGbngz4ETmr8rmr+z64B39L3s48D/msTPMuUMgg1EeoP0Hcy611qQ5PlN283AHGBhf3tV/QCYmeSFQyh1gzbG93g28EdVtSe9/zz/oZl/BjC/qnYHXgvcOsDin1tVc6vq4/3TzbI+CRzVvM+59NYkR2xaVXsB7wU+3Iy99SGaNf2qumjUZ3gp8FJg+QTz1/l3ACxo5nfBWH8HDwB30lthAiDJr9DbWvj6BMtaBOzc/P28oKruGNW+gN4W+YjPA69N8nKmiY1iiIlp7nlJFtNbY1kGXN3X9rKmrYDLqurKJIcMv8SNwjrfY5KZ9Dbz/1/fIZTNmp+/RW/tcWRt8v7mP42JXDTO9CuBVzfvCb3rZu7u63dJ83MhMHuC5b8tyf7AY8AfVtXPmuWNN1/j+40kN9ELgb+rqp9M0PfpfplrgL8B3g9c+Qzr26AYBFPvkarao1nr/zK9fZRnNG0jxwj6LQUO6J/RrCk+1KwRddVY3+NngPvG+A7Hs5qnbiX/0qj2h8eZDnBrVe07znIfa36uYeK/uYuq6oQB5y8F9gRu6pu3J4Nt2UwHS+mNRrBWs0b/YuB2escIjkiyI3BDks9X1eJxlvUaYFlVPZDk4SQvHbVVsCfwlVGv+Sy9ILiFacBdQxuIZp/kicCfrOfg4wXA/kl+G6A5eHwGTx4k67T+7xH4BfDDJP8V1p4RsnvT9avAcc38TZL8MvCfwNbpneW1GXDEgG97OzAryb7N8mYkWd8umgeBzZ/GRxvtY8D709zMqfn5AXr7r7vgq8Dzk7wT1u4S/DhwJvDISKfm2Mlp9Pb9ryPJb9I7PvDpZtbfAGc0f1c0f2dz6A2Tv1ZVraJ3D5U/nryPNHUMgg1IVX0XWALMm6DPI/Tu4/DB9IbjvpneuE5nDqXIjcCo7/EdwB80uwlu5cl7YpwEHJTkZnq7bF7V/HGfCnyH3i662wZ8v8fprZ2e3rzPYnq7pCZyLfCqsQ4WD/iei+n95/bFJLcBXwTeN8Fa77TSjEn2FuCoJN+nN1jlE1X10TG6nwUckCfvgDhykP579MLzd6tqWdP2SXq//yVJ7gTOBw6pqkfHWO45TJO9Kg4xIWmjl961ABcCb6mqRZO0zJnApcCNVfWByVjmhsogkKSOc9eQJHWcQSBJHWcQSFLHGQSS1HEGgTonvVE7q2/coaeMMTQJy/+nkUHJknygb/6kvo80WQwCddE84HomuF7jmUqySVUdU1VLm1nT+rRDTQ8GgTqlOTd8f+APGOMe2Umen+Tz6Y39f2mSbyeZ27TNS2/c/1uSnN73moeSfLy5mGzfZsTRuUlOoxkDKckFTfdNknw6ya1JvtJ3BevXkvxtkgXp3V/gdUkuSW9c/L9o+3tRtxkE6pojgauq6nvAvUn2HNV+PPDzqnoVcAq9cWZI8mvA6fQGq9sDeF2evBHMC4BvV9XuVXX9yIKq6mSevE/CyDDGOwGfqqo5wH3A7/a99+PNaKZnAZfRGy/p1cDRSbacjA8vjcUgUNfMAz7XPP8c6+4e2n+kvapuoTdUBcDrgK9V1crm/hAX8OTgf2uAixnMD/uGgRg9GunI0Nk30xvE7u6qegy4A9h+wOVLT9u0GCdDGkSSX6W3Rr9rkqI3XHQBn3qWi360Gcp6EI/1PV9D765ao9ueGNXvCfxbVYvcIlCXHAV8tqpeUlWzq2p74Ic8dW37G8BbAZozf3Zt5n8H+M0kWzUjXc4D5g/wnquSzJi0TyC1wCBQl8yjN4hYv4vpjSs/4h/oDSm9FPgLeiOW3l9Vd9O7Hei19O4BsLCqLhvgPc+mN5LlBevtKU0RB52T+jRr+zOq6tEkLwP+DXhlM9S0NC2531F6qucD1za7cwIcbwhounOLQJI6zmMEktRxBoEkdZxBIEkdZxBIUscZBJLUcf8fbjMECGwXXrAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_mean(rewards[1],'std deviation')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "719d552f",
   "metadata": {},
   "source": [
    "Hence, we see that:\n",
    "<ol><li> RecurrentPPO has the highest rewards (2.1x TRPO and 2x QRDQN)  </li>\n",
    "    <li> It also has the highest std deviation (1.8x TRPO and 2X QRDQN) </li>\n",
    "    <li> Similarly, A2C has 9% higher reward than PPO but 1.3X its std deviation </li>\n",
    "    \n",
    "<b>Given that ReccurrentPPO seems to be the highest rewards we will use it for the next set of 5,000,000 steps"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}