CartPole_multi.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6dee0562",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "41485176",
   "metadata": {},
   "outputs": [],
   "source": [
    "from get_algos import get_applicable_algos, run_env, run_experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "63e619c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "from gym.wrappers import Monitor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "361281ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "from stable_baselines3.common.evaluation import evaluate_policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b71eb193",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_paths(algo_name,env_name):\n",
    "    log_path=os.path.join(env_name, algo_name,\"logs\")\n",
    "    if not os.path.exists(log_path):\n",
    "        os.makedirs(log_path)\n",
    "    render_path=os.path.join(env_name, algo_name,\"render\")\n",
    "    if not os.path.exists(render_path):\n",
    "        os.makedirs(render_path)\n",
    "    model_path=os.path.join(env_name, algo_name)\n",
    "    if not os.path.exists(model_path):\n",
    "        os.makedirs(model_path)\n",
    "    return(log_path, render_path, model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "971570e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "env_name='CartPole-v0'\n",
    "env=gym.make(env_name)\n",
    "env=DummyVecEnv([lambda:env])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ccede2b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "action_space_type=str(type(env.action_space)).split('.')[3].split(\"'\")[0] #Parsing the action_space variable to get the exact type - Discrete, Box, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "28447a52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['PPO', 'A2C', 'DQN']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "algos=get_applicable_algos(action_space_type) #Gets all algos that apply for a specific type of action_space\n",
    "algos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "40c9527f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from stable_baselines3 import DQN, PPO ,A2C, DDPG, DQN, SAC, TD3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94bb6789",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "adcac168",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_algo(algo_name,env,logpath,timesteps=40000,nn_type='MlpPolicy'):\n",
    "    model=eval(algo_name)(nn_type, env, verbose=1,tensorboard_log=logpath)\n",
    "    model.learn(total_timesteps=timesteps)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "71a4671b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(algo_name, env_name,env,log_path,model_path): \n",
    "    model=train_algo(algo_name,env,log_path)\n",
    "    model.save(model_path)\n",
    "    env.close()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c1424789",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "******** PPO ********\n",
      "Using cpu device\n",
      "Logging to CartPole-v0\\PPO\\logs\\PPO_1\n",
      "-----------------------------\n",
      "| time/              |      |\n",
      "|    fps             | 2365 |\n",
      "|    iterations      | 1    |\n",
      "|    time_elapsed    | 0    |\n",
      "|    total_timesteps | 2048 |\n",
      "-----------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 1511        |\n",
      "|    iterations           | 2           |\n",
      "|    time_elapsed         | 2           |\n",
      "|    total_timesteps      | 4096        |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.008206999 |\n",
      "|    clip_fraction        | 0.0849      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.687      |\n",
      "|    explained_variance   | -0.00171    |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 7.73        |\n",
      "|    n_updates            | 10          |\n",
      "|    policy_gradient_loss | -0.0129     |\n",
      "|    value_loss           | 49.9        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 1360        |\n",
      "|    iterations           | 3           |\n",
      "|    time_elapsed         | 4           |\n",
      "|    total_timesteps      | 6144        |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.008656051 |\n",
      "|    clip_fraction        | 0.0601      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.672      |\n",
      "|    explained_variance   | 0.11        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 14.2        |\n",
      "|    n_updates            | 20          |\n",
      "|    policy_gradient_loss | -0.0164     |\n",
      "|    value_loss           | 37.1        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 1277        |\n",
      "|    iterations           | 4           |\n",
      "|    time_elapsed         | 6           |\n",
      "|    total_timesteps      | 8192        |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.008948765 |\n",
      "|    clip_fraction        | 0.0807      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.638      |\n",
      "|    explained_variance   | 0.23        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 20.4        |\n",
      "|    n_updates            | 30          |\n",
      "|    policy_gradient_loss | -0.0186     |\n",
      "|    value_loss           | 53.6        |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 1234       |\n",
      "|    iterations           | 5          |\n",
      "|    time_elapsed         | 8          |\n",
      "|    total_timesteps      | 10240      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.00879584 |\n",
      "|    clip_fraction        | 0.0773     |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.604     |\n",
      "|    explained_variance   | 0.231      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 22.7       |\n",
      "|    n_updates            | 40         |\n",
      "|    policy_gradient_loss | -0.0167    |\n",
      "|    value_loss           | 67.3       |\n",
      "----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 1181         |\n",
      "|    iterations           | 6            |\n",
      "|    time_elapsed         | 10           |\n",
      "|    total_timesteps      | 12288        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0084968675 |\n",
      "|    clip_fraction        | 0.0688       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.592       |\n",
      "|    explained_variance   | 0.37         |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 23.5         |\n",
      "|    n_updates            | 50           |\n",
      "|    policy_gradient_loss | -0.0162      |\n",
      "|    value_loss           | 64.2         |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 1152         |\n",
      "|    iterations           | 7            |\n",
      "|    time_elapsed         | 12           |\n",
      "|    total_timesteps      | 14336        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0064183096 |\n",
      "|    clip_fraction        | 0.0348       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.589       |\n",
      "|    explained_variance   | 0.553        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 21.7         |\n",
      "|    n_updates            | 60           |\n",
      "|    policy_gradient_loss | -0.00688     |\n",
      "|    value_loss           | 48.5         |\n",
      "------------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 1139       |\n",
      "|    iterations           | 8          |\n",
      "|    time_elapsed         | 14         |\n",
      "|    total_timesteps      | 16384      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.00666376 |\n",
      "|    clip_fraction        | 0.0687     |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.593     |\n",
      "|    explained_variance   | 0.822      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 2.57       |\n",
      "|    n_updates            | 70         |\n",
      "|    policy_gradient_loss | -0.0104    |\n",
      "|    value_loss           | 31.3       |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 1120        |\n",
      "|    iterations           | 9           |\n",
      "|    time_elapsed         | 16          |\n",
      "|    total_timesteps      | 18432       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.007747368 |\n",
      "|    clip_fraction        | 0.0651      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.57       |\n",
      "|    explained_variance   | 0.502       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 28          |\n",
      "|    n_updates            | 80          |\n",
      "|    policy_gradient_loss | -0.00865    |\n",
      "|    value_loss           | 59.2        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 1111        |\n",
      "|    iterations           | 10          |\n",
      "|    time_elapsed         | 18          |\n",
      "|    total_timesteps      | 20480       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.008548997 |\n",
      "|    clip_fraction        | 0.0339      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.569      |\n",
      "|    explained_variance   | 0.542       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 17.4        |\n",
      "|    n_updates            | 90          |\n",
      "|    policy_gradient_loss | -0.00337    |\n",
      "|    value_loss           | 21.6        |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 1103         |\n",
      "|    iterations           | 11           |\n",
      "|    time_elapsed         | 20           |\n",
      "|    total_timesteps      | 22528        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0063824113 |\n",
      "|    clip_fraction        | 0.0323       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.567       |\n",
      "|    explained_variance   | 0.0607       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.12         |\n",
      "|    n_updates            | 100          |\n",
      "|    policy_gradient_loss | -0.00236     |\n",
      "|    value_loss           | 26.1         |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 1102         |\n",
      "|    iterations           | 12           |\n",
      "|    time_elapsed         | 22           |\n",
      "|    total_timesteps      | 24576        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0030293951 |\n",
      "|    clip_fraction        | 0.0164       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.543       |\n",
      "|    explained_variance   | 0.0396       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 9.54         |\n",
      "|    n_updates            | 110          |\n",
      "|    policy_gradient_loss | -0.00592     |\n",
      "|    value_loss           | 50           |\n",
      "------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 1100         |\n",
      "|    iterations           | 13           |\n",
      "|    time_elapsed         | 24           |\n",
      "|    total_timesteps      | 26624        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0021168236 |\n",
      "|    clip_fraction        | 0.0122       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.544       |\n",
      "|    explained_variance   | 0.145        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 18.1         |\n",
      "|    n_updates            | 120          |\n",
      "|    policy_gradient_loss | -0.00136     |\n",
      "|    value_loss           | 25.1         |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 1101         |\n",
      "|    iterations           | 14           |\n",
      "|    time_elapsed         | 26           |\n",
      "|    total_timesteps      | 28672        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0047765835 |\n",
      "|    clip_fraction        | 0.033        |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.545       |\n",
      "|    explained_variance   | 0.305        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 0.41         |\n",
      "|    n_updates            | 130          |\n",
      "|    policy_gradient_loss | -0.00518     |\n",
      "|    value_loss           | 3.38         |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 1087        |\n",
      "|    iterations           | 15          |\n",
      "|    time_elapsed         | 28          |\n",
      "|    total_timesteps      | 30720       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.005236912 |\n",
      "|    clip_fraction        | 0.0258      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.55       |\n",
      "|    explained_variance   | 0.346       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0972      |\n",
      "|    n_updates            | 140         |\n",
      "|    policy_gradient_loss | -0.000778   |\n",
      "|    value_loss           | 1.84        |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 1082         |\n",
      "|    iterations           | 16           |\n",
      "|    time_elapsed         | 30           |\n",
      "|    total_timesteps      | 32768        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0009936842 |\n",
      "|    clip_fraction        | 0.0154       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.551       |\n",
      "|    explained_variance   | 0.487        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 0.0736       |\n",
      "|    n_updates            | 150          |\n",
      "|    policy_gradient_loss | -0.000323    |\n",
      "|    value_loss           | 1.12         |\n",
      "------------------------------------------\n",
      "----------------------------------------\n",
      "| time/                   |            |\n",
      "|    fps                  | 1072       |\n",
      "|    iterations           | 17         |\n",
      "|    time_elapsed         | 32         |\n",
      "|    total_timesteps      | 34816      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.00208039 |\n",
      "|    clip_fraction        | 0.0248     |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.56      |\n",
      "|    explained_variance   | 0.344      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.0901     |\n",
      "|    n_updates            | 160        |\n",
      "|    policy_gradient_loss | -0.000597  |\n",
      "|    value_loss           | 0.726      |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 1066        |\n",
      "|    iterations           | 18          |\n",
      "|    time_elapsed         | 34          |\n",
      "|    total_timesteps      | 36864       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.006054862 |\n",
      "|    clip_fraction        | 0.0406      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.549      |\n",
      "|    explained_variance   | 0.357       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.062       |\n",
      "|    n_updates            | 170         |\n",
      "|    policy_gradient_loss | -0.00259    |\n",
      "|    value_loss           | 0.482       |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 1070         |\n",
      "|    iterations           | 19           |\n",
      "|    time_elapsed         | 36           |\n",
      "|    total_timesteps      | 38912        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0035211272 |\n",
      "|    clip_fraction        | 0.0254       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.545       |\n",
      "|    explained_variance   | 0.259        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 0.0506       |\n",
      "|    n_updates            | 180          |\n",
      "|    policy_gradient_loss | -0.00121     |\n",
      "|    value_loss           | 0.315        |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 1071         |\n",
      "|    iterations           | 20           |\n",
      "|    time_elapsed         | 38           |\n",
      "|    total_timesteps      | 40960        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0029980482 |\n",
      "|    clip_fraction        | 0.0279       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.529       |\n",
      "|    explained_variance   | 0.55         |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 0.0319       |\n",
      "|    n_updates            | 190          |\n",
      "|    policy_gradient_loss | -0.00478     |\n",
      "|    value_loss           | 0.189        |\n",
      "------------------------------------------\n",
      "******** A2C ********\n",
      "Using cpu device\n",
      "Logging to CartPole-v0\\A2C\\logs\\A2C_1\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1074     |\n",
      "|    iterations         | 100      |\n",
      "|    time_elapsed       | 0        |\n",
      "|    total_timesteps    | 500      |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.541   |\n",
      "|    explained_variance | 0.096    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 99       |\n",
      "|    policy_loss        | 1.95     |\n",
      "|    value_loss         | 9.38     |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1115     |\n",
      "|    iterations         | 200      |\n",
      "|    time_elapsed       | 0        |\n",
      "|    total_timesteps    | 1000     |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.606   |\n",
      "|    explained_variance | -0.0344  |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 199      |\n",
      "|    policy_loss        | 1.93     |\n",
      "|    value_loss         | 7.93     |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1123     |\n",
      "|    iterations         | 300      |\n",
      "|    time_elapsed       | 1        |\n",
      "|    total_timesteps    | 1500     |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.62    |\n",
      "|    explained_variance | 0.0174   |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 299      |\n",
      "|    policy_loss        | 1.12     |\n",
      "|    value_loss         | 6.15     |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1121     |\n",
      "|    iterations         | 400      |\n",
      "|    time_elapsed       | 1        |\n",
      "|    total_timesteps    | 2000     |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.658   |\n",
      "|    explained_variance | -0.0065  |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 399      |\n",
      "|    policy_loss        | 1.16     |\n",
      "|    value_loss         | 5.58     |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1124     |\n",
      "|    iterations         | 500      |\n",
      "|    time_elapsed       | 2        |\n",
      "|    total_timesteps    | 2500     |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.649   |\n",
      "|    explained_variance | -0.00509 |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 499      |\n",
      "|    policy_loss        | 0.991    |\n",
      "|    value_loss         | 4.97     |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1117     |\n",
      "|    iterations         | 600      |\n",
      "|    time_elapsed       | 2        |\n",
      "|    total_timesteps    | 3000     |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.494   |\n",
      "|    explained_variance | 0.00808  |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 599      |\n",
      "|    policy_loss        | 1.26     |\n",
      "|    value_loss         | 4.32     |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1113     |\n",
      "|    iterations         | 700      |\n",
      "|    time_elapsed       | 3        |\n",
      "|    total_timesteps    | 3500     |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.609   |\n",
      "|    explained_variance | 0.00166  |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 699      |\n",
      "|    policy_loss        | 0.795    |\n",
      "|    value_loss         | 3.81     |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1117     |\n",
      "|    iterations         | 800      |\n",
      "|    time_elapsed       | 3        |\n",
      "|    total_timesteps    | 4000     |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.555   |\n",
      "|    explained_variance | 0.00451  |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 799      |\n",
      "|    policy_loss        | 0.759    |\n",
      "|    value_loss         | 3.3      |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| time/                 |           |\n",
      "|    fps                | 1120      |\n",
      "|    iterations         | 900       |\n",
      "|    time_elapsed       | 4         |\n",
      "|    total_timesteps    | 4500      |\n",
      "| train/                |           |\n",
      "|    entropy_loss       | -0.554    |\n",
      "|    explained_variance | -0.000359 |\n",
      "|    learning_rate      | 0.0007    |\n",
      "|    n_updates          | 899       |\n",
      "|    policy_loss        | 0.839     |\n",
      "|    value_loss         | 2.83      |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| time/                 |           |\n",
      "|    fps                | 1110      |\n",
      "|    iterations         | 1000      |\n",
      "|    time_elapsed       | 4         |\n",
      "|    total_timesteps    | 5000      |\n",
      "| train/                |           |\n",
      "|    entropy_loss       | -0.553    |\n",
      "|    explained_variance | -0.000256 |\n",
      "|    learning_rate      | 0.0007    |\n",
      "|    n_updates          | 999       |\n",
      "|    policy_loss        | 0.65      |\n",
      "|    value_loss         | 2.38      |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1106     |\n",
      "|    iterations         | 1100     |\n",
      "|    time_elapsed       | 4        |\n",
      "|    total_timesteps    | 5500     |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.472   |\n",
      "|    explained_variance | -0.00038 |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 1099     |\n",
      "|    policy_loss        | 0.853    |\n",
      "|    value_loss         | 1.96     |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1103     |\n",
      "|    iterations         | 1200     |\n",
      "|    time_elapsed       | 5        |\n",
      "|    total_timesteps    | 6000     |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.551   |\n",
      "|    explained_variance | 0.00103  |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 1199     |\n",
      "|    policy_loss        | -28.2    |\n",
      "|    value_loss         | 3.41e+03 |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| time/                 |           |\n",
      "|    fps                | 1105      |\n",
      "|    iterations         | 1300      |\n",
      "|    time_elapsed       | 5         |\n",
      "|    total_timesteps    | 6500      |\n",
      "| train/                |           |\n",
      "|    entropy_loss       | -0.623    |\n",
      "|    explained_variance | -5.44e-05 |\n",
      "|    learning_rate      | 0.0007    |\n",
      "|    n_updates          | 1299      |\n",
      "|    policy_loss        | 0.489     |\n",
      "|    value_loss         | 1.28      |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1071     |\n",
      "|    iterations         | 1400     |\n",
      "|    time_elapsed       | 6        |\n",
      "|    total_timesteps    | 7000     |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.631   |\n",
      "|    explained_variance | 8.13e-05 |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 1399     |\n",
      "|    policy_loss        | 0.374    |\n",
      "|    value_loss         | 0.997    |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| time/                 |           |\n",
      "|    fps                | 1060      |\n",
      "|    iterations         | 1500      |\n",
      "|    time_elapsed       | 7         |\n",
      "|    total_timesteps    | 7500      |\n",
      "| train/                |           |\n",
      "|    entropy_loss       | -0.619    |\n",
      "|    explained_variance | -8.26e-05 |\n",
      "|    learning_rate      | 0.0007    |\n",
      "|    n_updates          | 1499      |\n",
      "|    policy_loss        | 0.311     |\n",
      "|    value_loss         | 0.747     |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1052     |\n",
      "|    iterations         | 1600     |\n",
      "|    time_elapsed       | 7        |\n",
      "|    total_timesteps    | 8000     |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.655   |\n",
      "|    explained_variance | 5.67e-05 |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 1599     |\n",
      "|    policy_loss        | 0.305    |\n",
      "|    value_loss         | 0.522    |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| time/                 |           |\n",
      "|    fps                | 1052      |\n",
      "|    iterations         | 1700      |\n",
      "|    time_elapsed       | 8         |\n",
      "|    total_timesteps    | 8500      |\n",
      "| train/                |           |\n",
      "|    entropy_loss       | -0.568    |\n",
      "|    explained_variance | -0.000184 |\n",
      "|    learning_rate      | 0.0007    |\n",
      "|    n_updates          | 1699      |\n",
      "|    policy_loss        | 0.307     |\n",
      "|    value_loss         | 0.342     |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1057     |\n",
      "|    iterations         | 1800     |\n",
      "|    time_elapsed       | 8        |\n",
      "|    total_timesteps    | 9000     |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.611   |\n",
      "|    explained_variance | 8.02e-05 |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 1799     |\n",
      "|    policy_loss        | 0.208    |\n",
      "|    value_loss         | 0.198    |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1060     |\n",
      "|    iterations         | 1900     |\n",
      "|    time_elapsed       | 8        |\n",
      "|    total_timesteps    | 9500     |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.575   |\n",
      "|    explained_variance | 9.83e-06 |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 1899     |\n",
      "|    policy_loss        | -10.8    |\n",
      "|    value_loss         | 3.18e+03 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1065     |\n",
      "|    iterations         | 2000     |\n",
      "|    time_elapsed       | 9        |\n",
      "|    total_timesteps    | 10000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.564   |\n",
      "|    explained_variance | -5.9e-05 |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 1999     |\n",
      "|    policy_loss        | 0.07     |\n",
      "|    value_loss         | 0.0302   |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1068     |\n",
      "|    iterations         | 2100     |\n",
      "|    time_elapsed       | 9        |\n",
      "|    total_timesteps    | 10500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.639   |\n",
      "|    explained_variance | 0.00012  |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 2099     |\n",
      "|    policy_loss        | 0.0174   |\n",
      "|    value_loss         | 0.00183  |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1066     |\n",
      "|    iterations         | 2200     |\n",
      "|    time_elapsed       | 10       |\n",
      "|    total_timesteps    | 11000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.612   |\n",
      "|    explained_variance | 0.00178  |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 2199     |\n",
      "|    policy_loss        | 0.0021   |\n",
      "|    value_loss         | 3.29e-05 |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| time/                 |           |\n",
      "|    fps                | 1063      |\n",
      "|    iterations         | 2300      |\n",
      "|    time_elapsed       | 10        |\n",
      "|    total_timesteps    | 11500     |\n",
      "| train/                |           |\n",
      "|    entropy_loss       | -0.585    |\n",
      "|    explained_variance | -1.55e-06 |\n",
      "|    learning_rate      | 0.0007    |\n",
      "|    n_updates          | 2299      |\n",
      "|    policy_loss        | -2.5      |\n",
      "|    value_loss         | 1.96e+03  |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1062     |\n",
      "|    iterations         | 2400     |\n",
      "|    time_elapsed       | 11       |\n",
      "|    total_timesteps    | 12000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.551   |\n",
      "|    explained_variance | 3.61e-05 |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 2399     |\n",
      "|    policy_loss        | 0.000728 |\n",
      "|    value_loss         | 2.13e-06 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1065     |\n",
      "|    iterations         | 2500     |\n",
      "|    time_elapsed       | 11       |\n",
      "|    total_timesteps    | 12500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.58    |\n",
      "|    explained_variance | -0.00112 |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 2499     |\n",
      "|    policy_loss        | 0.00391  |\n",
      "|    value_loss         | 8.15e-05 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1069     |\n",
      "|    iterations         | 2600     |\n",
      "|    time_elapsed       | 12       |\n",
      "|    total_timesteps    | 13000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.506   |\n",
      "|    explained_variance | 0.0272   |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 2599     |\n",
      "|    policy_loss        | 0.00128  |\n",
      "|    value_loss         | 3.41e-06 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1073     |\n",
      "|    iterations         | 2700     |\n",
      "|    time_elapsed       | 12       |\n",
      "|    total_timesteps    | 13500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.614   |\n",
      "|    explained_variance | nan      |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 2699     |\n",
      "|    policy_loss        | 7.58e-06 |\n",
      "|    value_loss         | 3.38e-10 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1077     |\n",
      "|    iterations         | 2800     |\n",
      "|    time_elapsed       | 12       |\n",
      "|    total_timesteps    | 14000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.582   |\n",
      "|    explained_variance | 0.00449  |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 2799     |\n",
      "|    policy_loss        | 0.00232  |\n",
      "|    value_loss         | 2.05e-05 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1079     |\n",
      "|    iterations         | 2900     |\n",
      "|    time_elapsed       | 13       |\n",
      "|    total_timesteps    | 14500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.52    |\n",
      "|    explained_variance | 0.0505   |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 2899     |\n",
      "|    policy_loss        | 0.00115  |\n",
      "|    value_loss         | 2.91e-06 |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| time/                 |           |\n",
      "|    fps                | 1081      |\n",
      "|    iterations         | 3000      |\n",
      "|    time_elapsed       | 13        |\n",
      "|    total_timesteps    | 15000     |\n",
      "| train/                |           |\n",
      "|    entropy_loss       | -0.586    |\n",
      "|    explained_variance | -0.000298 |\n",
      "|    learning_rate      | 0.0007    |\n",
      "|    n_updates          | 2999      |\n",
      "|    policy_loss        | 0.000678  |\n",
      "|    value_loss         | 2.19e-06  |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1083     |\n",
      "|    iterations         | 3100     |\n",
      "|    time_elapsed       | 14       |\n",
      "|    total_timesteps    | 15500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.546   |\n",
      "|    explained_variance | -0.00821 |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 3099     |\n",
      "|    policy_loss        | 0.00228  |\n",
      "|    value_loss         | 2.51e-05 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1086     |\n",
      "|    iterations         | 3200     |\n",
      "|    time_elapsed       | 14       |\n",
      "|    total_timesteps    | 16000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.515   |\n",
      "|    explained_variance | 0.547    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 3199     |\n",
      "|    policy_loss        | 4.77e-08 |\n",
      "|    value_loss         | 8.15e-11 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1089     |\n",
      "|    iterations         | 3300     |\n",
      "|    time_elapsed       | 15       |\n",
      "|    total_timesteps    | 16500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.414   |\n",
      "|    explained_variance | -0.033   |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 3299     |\n",
      "|    policy_loss        | 0.00186  |\n",
      "|    value_loss         | 1.41e-05 |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| time/                 |           |\n",
      "|    fps                | 1092      |\n",
      "|    iterations         | 3400      |\n",
      "|    time_elapsed       | 15        |\n",
      "|    total_timesteps    | 17000     |\n",
      "| train/                |           |\n",
      "|    entropy_loss       | -0.462    |\n",
      "|    explained_variance | -0.000367 |\n",
      "|    learning_rate      | 0.0007    |\n",
      "|    n_updates          | 3399      |\n",
      "|    policy_loss        | 0.000937  |\n",
      "|    value_loss         | 4.76e-06  |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1094     |\n",
      "|    iterations         | 3500     |\n",
      "|    time_elapsed       | 15       |\n",
      "|    total_timesteps    | 17500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.611   |\n",
      "|    explained_variance | -0.0028  |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 3499     |\n",
      "|    policy_loss        | 0.00113  |\n",
      "|    value_loss         | 5.86e-06 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1095     |\n",
      "|    iterations         | 3600     |\n",
      "|    time_elapsed       | 16       |\n",
      "|    total_timesteps    | 18000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.525   |\n",
      "|    explained_variance | 0.0118   |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 3599     |\n",
      "|    policy_loss        | 0.000261 |\n",
      "|    value_loss         | 5.82e-07 |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1095     |\n",
      "|    iterations         | 3700     |\n",
      "|    time_elapsed       | 16       |\n",
      "|    total_timesteps    | 18500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.582   |\n",
      "|    explained_variance | -1.62    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 3699     |\n",
      "|    policy_loss        | 0.00107  |\n",
      "|    value_loss         | 9.78e-06 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1097     |\n",
      "|    iterations         | 3800     |\n",
      "|    time_elapsed       | 17       |\n",
      "|    total_timesteps    | 19000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.547   |\n",
      "|    explained_variance | 0.311    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 3799     |\n",
      "|    policy_loss        | -0.0222  |\n",
      "|    value_loss         | 0.00305  |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1099     |\n",
      "|    iterations         | 3900     |\n",
      "|    time_elapsed       | 17       |\n",
      "|    total_timesteps    | 19500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.565   |\n",
      "|    explained_variance | -28.8    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 3899     |\n",
      "|    policy_loss        | -0.417   |\n",
      "|    value_loss         | 1.28     |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1101     |\n",
      "|    iterations         | 4000     |\n",
      "|    time_elapsed       | 18       |\n",
      "|    total_timesteps    | 20000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.536   |\n",
      "|    explained_variance | -2.68    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 3999     |\n",
      "|    policy_loss        | -0.29    |\n",
      "|    value_loss         | 0.591    |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1103     |\n",
      "|    iterations         | 4100     |\n",
      "|    time_elapsed       | 18       |\n",
      "|    total_timesteps    | 20500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.454   |\n",
      "|    explained_variance | -18.2    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 4099     |\n",
      "|    policy_loss        | -6.33    |\n",
      "|    value_loss         | 303      |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1105     |\n",
      "|    iterations         | 4200     |\n",
      "|    time_elapsed       | 18       |\n",
      "|    total_timesteps    | 21000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.416   |\n",
      "|    explained_variance | -1.51    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 4199     |\n",
      "|    policy_loss        | -0.707   |\n",
      "|    value_loss         | 30       |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1107     |\n",
      "|    iterations         | 4300     |\n",
      "|    time_elapsed       | 19       |\n",
      "|    total_timesteps    | 21500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.401   |\n",
      "|    explained_variance | 0.907    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 4299     |\n",
      "|    policy_loss        | -0.245   |\n",
      "|    value_loss         | 0.541    |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1109     |\n",
      "|    iterations         | 4400     |\n",
      "|    time_elapsed       | 19       |\n",
      "|    total_timesteps    | 22000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.301   |\n",
      "|    explained_variance | -18.9    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 4399     |\n",
      "|    policy_loss        | 0.96     |\n",
      "|    value_loss         | 3.63     |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1110     |\n",
      "|    iterations         | 4500     |\n",
      "|    time_elapsed       | 20       |\n",
      "|    total_timesteps    | 22500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.489   |\n",
      "|    explained_variance | -260     |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 4499     |\n",
      "|    policy_loss        | 0.086    |\n",
      "|    value_loss         | 0.0791   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1112     |\n",
      "|    iterations         | 4600     |\n",
      "|    time_elapsed       | 20       |\n",
      "|    total_timesteps    | 23000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.671   |\n",
      "|    explained_variance | -0.0418  |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 4599     |\n",
      "|    policy_loss        | -57.8    |\n",
      "|    value_loss         | 9.84e+03 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1113     |\n",
      "|    iterations         | 4700     |\n",
      "|    time_elapsed       | 21       |\n",
      "|    total_timesteps    | 23500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.133   |\n",
      "|    explained_variance | -84.4    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 4699     |\n",
      "|    policy_loss        | 0.00478  |\n",
      "|    value_loss         | 0.12     |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1115     |\n",
      "|    iterations         | 4800     |\n",
      "|    time_elapsed       | 21       |\n",
      "|    total_timesteps    | 24000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.149   |\n",
      "|    explained_variance | -0.00534 |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 4799     |\n",
      "|    policy_loss        | -3.17    |\n",
      "|    value_loss         | 2e+03    |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1115     |\n",
      "|    iterations         | 4900     |\n",
      "|    time_elapsed       | 21       |\n",
      "|    total_timesteps    | 24500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.209   |\n",
      "|    explained_variance | -118     |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 4899     |\n",
      "|    policy_loss        | 0.00038  |\n",
      "|    value_loss         | 0.000255 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1115     |\n",
      "|    iterations         | 5000     |\n",
      "|    time_elapsed       | 22       |\n",
      "|    total_timesteps    | 25000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.054   |\n",
      "|    explained_variance | -16.2    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 4999     |\n",
      "|    policy_loss        | 0.00083  |\n",
      "|    value_loss         | 0.0101   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1117     |\n",
      "|    iterations         | 5100     |\n",
      "|    time_elapsed       | 22       |\n",
      "|    total_timesteps    | 25500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.0493  |\n",
      "|    explained_variance | -2.68    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 5099     |\n",
      "|    policy_loss        | 0.000589 |\n",
      "|    value_loss         | 0.00743  |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1117     |\n",
      "|    iterations         | 5200     |\n",
      "|    time_elapsed       | 23       |\n",
      "|    total_timesteps    | 26000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.0807  |\n",
      "|    explained_variance | -0.906   |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 5199     |\n",
      "|    policy_loss        | 0.0157   |\n",
      "|    value_loss         | 0.00547  |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| time/                 |           |\n",
      "|    fps                | 1116      |\n",
      "|    iterations         | 5300      |\n",
      "|    time_elapsed       | 23        |\n",
      "|    total_timesteps    | 26500     |\n",
      "| train/                |           |\n",
      "|    entropy_loss       | -0.105    |\n",
      "|    explained_variance | -0.000566 |\n",
      "|    learning_rate      | 0.0007    |\n",
      "|    n_updates          | 5299      |\n",
      "|    policy_loss        | -0.693    |\n",
      "|    value_loss         | 1.88e+03  |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1116     |\n",
      "|    iterations         | 5400     |\n",
      "|    time_elapsed       | 24       |\n",
      "|    total_timesteps    | 27000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.109   |\n",
      "|    explained_variance | -0.0365  |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 5399     |\n",
      "|    policy_loss        | 0.0863   |\n",
      "|    value_loss         | 0.00537  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| time/                 |           |\n",
      "|    fps                | 1117      |\n",
      "|    iterations         | 5500      |\n",
      "|    time_elapsed       | 24        |\n",
      "|    total_timesteps    | 27500     |\n",
      "| train/                |           |\n",
      "|    entropy_loss       | -0.429    |\n",
      "|    explained_variance | -0.000417 |\n",
      "|    learning_rate      | 0.0007    |\n",
      "|    n_updates          | 5499      |\n",
      "|    policy_loss        | -44.8     |\n",
      "|    value_loss         | 7.37e+03  |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1118     |\n",
      "|    iterations         | 5600     |\n",
      "|    time_elapsed       | 25       |\n",
      "|    total_timesteps    | 28000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.255   |\n",
      "|    explained_variance | -2.75    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 5599     |\n",
      "|    policy_loss        | 0.0226   |\n",
      "|    value_loss         | 0.00193  |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1119     |\n",
      "|    iterations         | 5700     |\n",
      "|    time_elapsed       | 25       |\n",
      "|    total_timesteps    | 28500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.496   |\n",
      "|    explained_variance | -0.592   |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 5699     |\n",
      "|    policy_loss        | 0.00569  |\n",
      "|    value_loss         | 0.000257 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1120     |\n",
      "|    iterations         | 5800     |\n",
      "|    time_elapsed       | 25       |\n",
      "|    total_timesteps    | 29000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.47    |\n",
      "|    explained_variance | -0.0325  |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 5799     |\n",
      "|    policy_loss        | 0.00668  |\n",
      "|    value_loss         | 0.000306 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1121     |\n",
      "|    iterations         | 5900     |\n",
      "|    time_elapsed       | 26       |\n",
      "|    total_timesteps    | 29500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.402   |\n",
      "|    explained_variance | -0.0128  |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 5899     |\n",
      "|    policy_loss        | 0.00786  |\n",
      "|    value_loss         | 0.000196 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1122     |\n",
      "|    iterations         | 6000     |\n",
      "|    time_elapsed       | 26       |\n",
      "|    total_timesteps    | 30000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.5     |\n",
      "|    explained_variance | -0.847   |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 5999     |\n",
      "|    policy_loss        | 0.00108  |\n",
      "|    value_loss         | 1.09e-05 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1123     |\n",
      "|    iterations         | 6100     |\n",
      "|    time_elapsed       | 27       |\n",
      "|    total_timesteps    | 30500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.488   |\n",
      "|    explained_variance | -2.48    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 6099     |\n",
      "|    policy_loss        | 0.000388 |\n",
      "|    value_loss         | 7.36e-06 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1124     |\n",
      "|    iterations         | 6200     |\n",
      "|    time_elapsed       | 27       |\n",
      "|    total_timesteps    | 31000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.586   |\n",
      "|    explained_variance | 0.0905   |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 6199     |\n",
      "|    policy_loss        | 0.000783 |\n",
      "|    value_loss         | 5.01e-06 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1124     |\n",
      "|    iterations         | 6300     |\n",
      "|    time_elapsed       | 28       |\n",
      "|    total_timesteps    | 31500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.497   |\n",
      "|    explained_variance | 0.606    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 6299     |\n",
      "|    policy_loss        | 0.00395  |\n",
      "|    value_loss         | 5.29e-05 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1125     |\n",
      "|    iterations         | 6400     |\n",
      "|    time_elapsed       | 28       |\n",
      "|    total_timesteps    | 32000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.45    |\n",
      "|    explained_variance | 0.108    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 6399     |\n",
      "|    policy_loss        | 0.000634 |\n",
      "|    value_loss         | 4.91e-06 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1126     |\n",
      "|    iterations         | 6500     |\n",
      "|    time_elapsed       | 28       |\n",
      "|    total_timesteps    | 32500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.552   |\n",
      "|    explained_variance | -8.27    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 6499     |\n",
      "|    policy_loss        | 0.0001   |\n",
      "|    value_loss         | 1.36e-07 |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| time/                 |           |\n",
      "|    fps                | 1127      |\n",
      "|    iterations         | 6600      |\n",
      "|    time_elapsed       | 29        |\n",
      "|    total_timesteps    | 33000     |\n",
      "| train/                |           |\n",
      "|    entropy_loss       | -0.338    |\n",
      "|    explained_variance | -4.49e-05 |\n",
      "|    learning_rate      | 0.0007    |\n",
      "|    n_updates          | 6599      |\n",
      "|    policy_loss        | -38.8     |\n",
      "|    value_loss         | 7.61e+03  |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1128     |\n",
      "|    iterations         | 6700     |\n",
      "|    time_elapsed       | 29       |\n",
      "|    total_timesteps    | 33500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.44    |\n",
      "|    explained_variance | 0.304    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 6699     |\n",
      "|    policy_loss        | 0.000125 |\n",
      "|    value_loss         | 5.91e-07 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1129     |\n",
      "|    iterations         | 6800     |\n",
      "|    time_elapsed       | 30       |\n",
      "|    total_timesteps    | 34000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.511   |\n",
      "|    explained_variance | -0.627   |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 6799     |\n",
      "|    policy_loss        | 0.00139  |\n",
      "|    value_loss         | 1.42e-05 |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1130     |\n",
      "|    iterations         | 6900     |\n",
      "|    time_elapsed       | 30       |\n",
      "|    total_timesteps    | 34500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.535   |\n",
      "|    explained_variance | -11.6    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 6899     |\n",
      "|    policy_loss        | 0.000206 |\n",
      "|    value_loss         | 3.41e-07 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1130     |\n",
      "|    iterations         | 7000     |\n",
      "|    time_elapsed       | 30       |\n",
      "|    total_timesteps    | 35000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.402   |\n",
      "|    explained_variance | -34.2    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 6999     |\n",
      "|    policy_loss        | -0.00105 |\n",
      "|    value_loss         | 1.09e-06 |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| time/                 |           |\n",
      "|    fps                | 1131      |\n",
      "|    iterations         | 7100      |\n",
      "|    time_elapsed       | 31        |\n",
      "|    total_timesteps    | 35500     |\n",
      "| train/                |           |\n",
      "|    entropy_loss       | -0.384    |\n",
      "|    explained_variance | -11.8     |\n",
      "|    learning_rate      | 0.0007    |\n",
      "|    n_updates          | 7099      |\n",
      "|    policy_loss        | -0.000609 |\n",
      "|    value_loss         | 1.24e-06  |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| time/                 |           |\n",
      "|    fps                | 1132      |\n",
      "|    iterations         | 7200      |\n",
      "|    time_elapsed       | 31        |\n",
      "|    total_timesteps    | 36000     |\n",
      "| train/                |           |\n",
      "|    entropy_loss       | -0.52     |\n",
      "|    explained_variance | -341      |\n",
      "|    learning_rate      | 0.0007    |\n",
      "|    n_updates          | 7199      |\n",
      "|    policy_loss        | -0.000163 |\n",
      "|    value_loss         | 1.66e-07  |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1132     |\n",
      "|    iterations         | 7300     |\n",
      "|    time_elapsed       | 32       |\n",
      "|    total_timesteps    | 36500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.447   |\n",
      "|    explained_variance | -7.03    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 7299     |\n",
      "|    policy_loss        | 0.000134 |\n",
      "|    value_loss         | 3.74e-07 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1133     |\n",
      "|    iterations         | 7400     |\n",
      "|    time_elapsed       | 32       |\n",
      "|    total_timesteps    | 37000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.578   |\n",
      "|    explained_variance | -15.2    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 7399     |\n",
      "|    policy_loss        | 0.000705 |\n",
      "|    value_loss         | 3.91e-06 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1134     |\n",
      "|    iterations         | 7500     |\n",
      "|    time_elapsed       | 33       |\n",
      "|    total_timesteps    | 37500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.444   |\n",
      "|    explained_variance | -17      |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 7499     |\n",
      "|    policy_loss        | -0.00194 |\n",
      "|    value_loss         | 3.07e-05 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1134     |\n",
      "|    iterations         | 7600     |\n",
      "|    time_elapsed       | 33       |\n",
      "|    total_timesteps    | 38000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.448   |\n",
      "|    explained_variance | -2.14    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 7599     |\n",
      "|    policy_loss        | 0.00143  |\n",
      "|    value_loss         | 8.82e-06 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1135     |\n",
      "|    iterations         | 7700     |\n",
      "|    time_elapsed       | 33       |\n",
      "|    total_timesteps    | 38500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.444   |\n",
      "|    explained_variance | -0.235   |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 7699     |\n",
      "|    policy_loss        | 0.00154  |\n",
      "|    value_loss         | 1.17e-05 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1136     |\n",
      "|    iterations         | 7800     |\n",
      "|    time_elapsed       | 34       |\n",
      "|    total_timesteps    | 39000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.398   |\n",
      "|    explained_variance | 0.00375  |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 7799     |\n",
      "|    policy_loss        | -2.45    |\n",
      "|    value_loss         | 1.94e+03 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1136     |\n",
      "|    iterations         | 7900     |\n",
      "|    time_elapsed       | 34       |\n",
      "|    total_timesteps    | 39500    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.458   |\n",
      "|    explained_variance | -0.104   |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 7899     |\n",
      "|    policy_loss        | 0.00272  |\n",
      "|    value_loss         | 3.32e-05 |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| time/                 |          |\n",
      "|    fps                | 1137     |\n",
      "|    iterations         | 8000     |\n",
      "|    time_elapsed       | 35       |\n",
      "|    total_timesteps    | 40000    |\n",
      "| train/                |          |\n",
      "|    entropy_loss       | -0.418   |\n",
      "|    explained_variance | -8.56    |\n",
      "|    learning_rate      | 0.0007   |\n",
      "|    n_updates          | 7999     |\n",
      "|    policy_loss        | -0.0295  |\n",
      "|    value_loss         | 0.0119   |\n",
      "------------------------------------\n",
      "******** DQN ********\n",
      "Using cpu device\n",
      "Logging to CartPole-v0\\DQN\\logs\\DQN_1\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.981    |\n",
      "| time/               |          |\n",
      "|    episodes         | 4        |\n",
      "|    fps              | 9776     |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 78       |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.958    |\n",
      "| time/               |          |\n",
      "|    episodes         | 8        |\n",
      "|    fps              | 11028    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 176      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.933    |\n",
      "| time/               |          |\n",
      "|    episodes         | 12       |\n",
      "|    fps              | 11338    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 283      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.908    |\n",
      "| time/               |          |\n",
      "|    episodes         | 16       |\n",
      "|    fps              | 11403    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 387      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.891    |\n",
      "| time/               |          |\n",
      "|    episodes         | 20       |\n",
      "|    fps              | 11447    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 457      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.877    |\n",
      "| time/               |          |\n",
      "|    episodes         | 24       |\n",
      "|    fps              | 11305    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 519      |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.86     |\n",
      "| time/               |          |\n",
      "|    episodes         | 28       |\n",
      "|    fps              | 11258    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 590      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.827    |\n",
      "| time/               |          |\n",
      "|    episodes         | 32       |\n",
      "|    fps              | 11518    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 729      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.807    |\n",
      "| time/               |          |\n",
      "|    episodes         | 36       |\n",
      "|    fps              | 11559    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 812      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.776    |\n",
      "| time/               |          |\n",
      "|    episodes         | 40       |\n",
      "|    fps              | 11763    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 944      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.751    |\n",
      "| time/               |          |\n",
      "|    episodes         | 44       |\n",
      "|    fps              | 11768    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 1050     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.716    |\n",
      "| time/               |          |\n",
      "|    episodes         | 48       |\n",
      "|    fps              | 11948    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 1197     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.686    |\n",
      "| time/               |          |\n",
      "|    episodes         | 52       |\n",
      "|    fps              | 11991    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 1321     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.67     |\n",
      "| time/               |          |\n",
      "|    episodes         | 56       |\n",
      "|    fps              | 11951    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 1388     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.64     |\n",
      "| time/               |          |\n",
      "|    episodes         | 60       |\n",
      "|    fps              | 12007    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 1514     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.627    |\n",
      "| time/               |          |\n",
      "|    episodes         | 64       |\n",
      "|    fps              | 11885    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 1570     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.604    |\n",
      "| time/               |          |\n",
      "|    episodes         | 68       |\n",
      "|    fps              | 11907    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 1668     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.581    |\n",
      "| time/               |          |\n",
      "|    episodes         | 72       |\n",
      "|    fps              | 11907    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 1763     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.562    |\n",
      "| time/               |          |\n",
      "|    episodes         | 76       |\n",
      "|    fps              | 11900    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 1845     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.54     |\n",
      "| time/               |          |\n",
      "|    episodes         | 80       |\n",
      "|    fps              | 11851    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 1936     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.523    |\n",
      "| time/               |          |\n",
      "|    episodes         | 84       |\n",
      "|    fps              | 11869    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 2010     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.509    |\n",
      "| time/               |          |\n",
      "|    episodes         | 88       |\n",
      "|    fps              | 11863    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 2068     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.489    |\n",
      "| time/               |          |\n",
      "|    episodes         | 92       |\n",
      "|    fps              | 11799    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 2151     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.465    |\n",
      "| time/               |          |\n",
      "|    episodes         | 96       |\n",
      "|    fps              | 11768    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 2251     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.435    |\n",
      "| time/               |          |\n",
      "|    episodes         | 100      |\n",
      "|    fps              | 11832    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 2380     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.419    |\n",
      "| time/               |          |\n",
      "|    episodes         | 104      |\n",
      "|    fps              | 11797    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 2446     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.401    |\n",
      "| time/               |          |\n",
      "|    episodes         | 108      |\n",
      "|    fps              | 11817    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 2521     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.382    |\n",
      "| time/               |          |\n",
      "|    episodes         | 112      |\n",
      "|    fps              | 11759    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 2602     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.363    |\n",
      "| time/               |          |\n",
      "|    episodes         | 116      |\n",
      "|    fps              | 11744    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 2681     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.349    |\n",
      "| time/               |          |\n",
      "|    episodes         | 120      |\n",
      "|    fps              | 11700    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 2741     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.328    |\n",
      "| time/               |          |\n",
      "|    episodes         | 124      |\n",
      "|    fps              | 11680    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 2828     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.307    |\n",
      "| time/               |          |\n",
      "|    episodes         | 128      |\n",
      "|    fps              | 11713    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 2918     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.291    |\n",
      "| time/               |          |\n",
      "|    episodes         | 132      |\n",
      "|    fps              | 11709    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 2987     |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.275    |\n",
      "| time/               |          |\n",
      "|    episodes         | 136      |\n",
      "|    fps              | 11644    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 3053     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.261    |\n",
      "| time/               |          |\n",
      "|    episodes         | 140      |\n",
      "|    fps              | 11626    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 3110     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.239    |\n",
      "| time/               |          |\n",
      "|    episodes         | 144      |\n",
      "|    fps              | 11652    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 3204     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.222    |\n",
      "| time/               |          |\n",
      "|    episodes         | 148      |\n",
      "|    fps              | 11619    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 3276     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.203    |\n",
      "| time/               |          |\n",
      "|    episodes         | 152      |\n",
      "|    fps              | 11615    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 3356     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.187    |\n",
      "| time/               |          |\n",
      "|    episodes         | 156      |\n",
      "|    fps              | 11603    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 3422     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.174    |\n",
      "| time/               |          |\n",
      "|    episodes         | 160      |\n",
      "|    fps              | 11593    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 3477     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.15     |\n",
      "| time/               |          |\n",
      "|    episodes         | 164      |\n",
      "|    fps              | 11569    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 3579     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.13     |\n",
      "| time/               |          |\n",
      "|    episodes         | 168      |\n",
      "|    fps              | 11618    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 3664     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.115    |\n",
      "| time/               |          |\n",
      "|    episodes         | 172      |\n",
      "|    fps              | 11591    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 3725     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.0999   |\n",
      "| time/               |          |\n",
      "|    episodes         | 176      |\n",
      "|    fps              | 11578    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 3790     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.0752   |\n",
      "| time/               |          |\n",
      "|    episodes         | 180      |\n",
      "|    fps              | 11578    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 3894     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.0517   |\n",
      "| time/               |          |\n",
      "|    episodes         | 184      |\n",
      "|    fps              | 11584    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 3993     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 188      |\n",
      "|    fps              | 11589    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 4064     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 192      |\n",
      "|    fps              | 11581    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 4142     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 196      |\n",
      "|    fps              | 11597    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 4252     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 200      |\n",
      "|    fps              | 11612    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 4327     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 204      |\n",
      "|    fps              | 11622    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 4428     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 208      |\n",
      "|    fps              | 11608    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 4492     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 212      |\n",
      "|    fps              | 11630    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 4570     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 216      |\n",
      "|    fps              | 11606    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 4630     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 220      |\n",
      "|    fps              | 11622    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 4741     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 224      |\n",
      "|    fps              | 11631    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 4842     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 228      |\n",
      "|    fps              | 11643    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 4963     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 232      |\n",
      "|    fps              | 11643    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 5021     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 236      |\n",
      "|    fps              | 11609    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 5076     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 240      |\n",
      "|    fps              | 11623    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 5181     |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 244      |\n",
      "|    fps              | 11632    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 5243     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 248      |\n",
      "|    fps              | 11599    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 5309     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 252      |\n",
      "|    fps              | 11613    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 5385     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 256      |\n",
      "|    fps              | 11615    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 5502     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 260      |\n",
      "|    fps              | 11603    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 5581     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 264      |\n",
      "|    fps              | 11599    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 5649     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 268      |\n",
      "|    fps              | 11593    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 5704     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 272      |\n",
      "|    fps              | 11604    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 5802     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 276      |\n",
      "|    fps              | 11600    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 5869     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 280      |\n",
      "|    fps              | 11595    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 5941     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 284      |\n",
      "|    fps              | 11597    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 6046     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 288      |\n",
      "|    fps              | 11615    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 6192     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 292      |\n",
      "|    fps              | 11618    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 6286     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 296      |\n",
      "|    fps              | 11616    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 6408     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 300      |\n",
      "|    fps              | 11629    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 6528     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 304      |\n",
      "|    fps              | 11621    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 6660     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 308      |\n",
      "|    fps              | 11613    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 6734     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 312      |\n",
      "|    fps              | 11618    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 6822     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 316      |\n",
      "|    fps              | 11610    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 6905     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 320      |\n",
      "|    fps              | 11611    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 6998     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 324      |\n",
      "|    fps              | 11607    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 7088     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 328      |\n",
      "|    fps              | 11616    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 7210     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 332      |\n",
      "|    fps              | 11602    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 7305     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 336      |\n",
      "|    fps              | 11603    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 7363     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 340      |\n",
      "|    fps              | 11589    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 7435     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 344      |\n",
      "|    fps              | 11554    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 7502     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 348      |\n",
      "|    fps              | 11566    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 7591     |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 352      |\n",
      "|    fps              | 11551    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 7673     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 356      |\n",
      "|    fps              | 11546    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 7735     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 360      |\n",
      "|    fps              | 11531    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 7808     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 364      |\n",
      "|    fps              | 11526    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 7862     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 368      |\n",
      "|    fps              | 11516    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 7959     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 372      |\n",
      "|    fps              | 11522    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 8040     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 376      |\n",
      "|    fps              | 11505    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 8123     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 380      |\n",
      "|    fps              | 11519    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 8218     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 384      |\n",
      "|    fps              | 11510    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 8315     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 388      |\n",
      "|    fps              | 11498    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 8387     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 392      |\n",
      "|    fps              | 11494    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 8452     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 396      |\n",
      "|    fps              | 11483    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 8513     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 400      |\n",
      "|    fps              | 11474    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 8579     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 404      |\n",
      "|    fps              | 11466    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 8653     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 408      |\n",
      "|    fps              | 11469    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 8724     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 412      |\n",
      "|    fps              | 11464    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 8789     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 416      |\n",
      "|    fps              | 11471    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 8938     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 420      |\n",
      "|    fps              | 11473    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 9020     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 424      |\n",
      "|    fps              | 11469    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 9128     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 428      |\n",
      "|    fps              | 11470    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 9209     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 432      |\n",
      "|    fps              | 11462    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 9289     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 436      |\n",
      "|    fps              | 11470    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 9364     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 440      |\n",
      "|    fps              | 11456    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 9433     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 444      |\n",
      "|    fps              | 11451    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 9529     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 448      |\n",
      "|    fps              | 11450    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 9608     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 452      |\n",
      "|    fps              | 11437    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 9677     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 456      |\n",
      "|    fps              | 11441    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 9746     |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 460      |\n",
      "|    fps              | 11445    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 9882     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 464      |\n",
      "|    fps              | 11447    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 9975     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 468      |\n",
      "|    fps              | 11436    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 10102    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 472      |\n",
      "|    fps              | 11425    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 10212    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 476      |\n",
      "|    fps              | 11385    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 10289    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 480      |\n",
      "|    fps              | 11374    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 10370    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 484      |\n",
      "|    fps              | 11366    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 10431    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 488      |\n",
      "|    fps              | 11372    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 10532    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 492      |\n",
      "|    fps              | 11369    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 10632    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 496      |\n",
      "|    fps              | 11366    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 10719    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 500      |\n",
      "|    fps              | 11355    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 10839    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 504      |\n",
      "|    fps              | 11352    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 10914    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 508      |\n",
      "|    fps              | 11360    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 11046    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 512      |\n",
      "|    fps              | 11357    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 11112    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 516      |\n",
      "|    fps              | 11376    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 11221    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 520      |\n",
      "|    fps              | 11380    |\n",
      "|    time_elapsed     | 0        |\n",
      "|    total_timesteps  | 11332    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 524      |\n",
      "|    fps              | 11392    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 11454    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 528      |\n",
      "|    fps              | 11389    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 11542    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 532      |\n",
      "|    fps              | 11381    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 11613    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 536      |\n",
      "|    fps              | 11390    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 11702    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 540      |\n",
      "|    fps              | 11372    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 11751    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 544      |\n",
      "|    fps              | 11371    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 11819    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 548      |\n",
      "|    fps              | 11386    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 11930    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 552      |\n",
      "|    fps              | 11385    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 12008    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 556      |\n",
      "|    fps              | 11383    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 12085    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 560      |\n",
      "|    fps              | 11380    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 12162    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 564      |\n",
      "|    fps              | 11378    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 12228    |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 568      |\n",
      "|    fps              | 11372    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 12294    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 572      |\n",
      "|    fps              | 11372    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 12363    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 576      |\n",
      "|    fps              | 11383    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 12488    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 580      |\n",
      "|    fps              | 11384    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 12591    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 584      |\n",
      "|    fps              | 11377    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 12668    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 588      |\n",
      "|    fps              | 11376    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 12758    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 592      |\n",
      "|    fps              | 11370    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 12819    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 596      |\n",
      "|    fps              | 11381    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 12979    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 600      |\n",
      "|    fps              | 11365    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 13046    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 604      |\n",
      "|    fps              | 11361    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 13122    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 608      |\n",
      "|    fps              | 11353    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 13174    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 612      |\n",
      "|    fps              | 11350    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 13262    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 616      |\n",
      "|    fps              | 11343    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 13355    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 620      |\n",
      "|    fps              | 11344    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 13435    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 624      |\n",
      "|    fps              | 11348    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 13565    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 628      |\n",
      "|    fps              | 11337    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 13619    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 632      |\n",
      "|    fps              | 11329    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 13678    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 636      |\n",
      "|    fps              | 11324    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 13742    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 640      |\n",
      "|    fps              | 11331    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 13909    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 644      |\n",
      "|    fps              | 11334    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 14003    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 648      |\n",
      "|    fps              | 11330    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 14054    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 652      |\n",
      "|    fps              | 11325    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 14137    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 656      |\n",
      "|    fps              | 11324    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 14238    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 660      |\n",
      "|    fps              | 11332    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 14352    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 664      |\n",
      "|    fps              | 11334    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 14433    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 668      |\n",
      "|    fps              | 11324    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 14488    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 672      |\n",
      "|    fps              | 11335    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 14627    |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 676      |\n",
      "|    fps              | 11325    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 14686    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 680      |\n",
      "|    fps              | 11332    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 14808    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 684      |\n",
      "|    fps              | 11326    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 14869    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 688      |\n",
      "|    fps              | 11327    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 14960    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 692      |\n",
      "|    fps              | 11328    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 15041    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 696      |\n",
      "|    fps              | 11326    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 15117    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 700      |\n",
      "|    fps              | 11326    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 15232    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 704      |\n",
      "|    fps              | 11334    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 15310    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 708      |\n",
      "|    fps              | 11331    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 15408    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 712      |\n",
      "|    fps              | 11325    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 15468    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 716      |\n",
      "|    fps              | 11335    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 15572    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 720      |\n",
      "|    fps              | 11335    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 15656    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 724      |\n",
      "|    fps              | 11336    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 15713    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 728      |\n",
      "|    fps              | 11333    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 15788    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 732      |\n",
      "|    fps              | 11329    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 15851    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 736      |\n",
      "|    fps              | 11332    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 15912    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 740      |\n",
      "|    fps              | 11333    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 16025    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 744      |\n",
      "|    fps              | 11344    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 16179    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 748      |\n",
      "|    fps              | 11356    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 16320    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 752      |\n",
      "|    fps              | 11352    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 16411    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 756      |\n",
      "|    fps              | 11345    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 16469    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 760      |\n",
      "|    fps              | 11345    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 16514    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 764      |\n",
      "|    fps              | 11353    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 16640    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 768      |\n",
      "|    fps              | 11347    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 16698    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 772      |\n",
      "|    fps              | 11349    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 16811    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 776      |\n",
      "|    fps              | 11355    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 16933    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 780      |\n",
      "|    fps              | 11356    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 17003    |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 784      |\n",
      "|    fps              | 11355    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 17081    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 788      |\n",
      "|    fps              | 11362    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 17219    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 792      |\n",
      "|    fps              | 11363    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 17300    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 796      |\n",
      "|    fps              | 11357    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 17394    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 800      |\n",
      "|    fps              | 11360    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 17478    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 804      |\n",
      "|    fps              | 11360    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 17573    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 808      |\n",
      "|    fps              | 11364    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 17680    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 812      |\n",
      "|    fps              | 11361    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 17744    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 816      |\n",
      "|    fps              | 11356    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 17826    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 820      |\n",
      "|    fps              | 11354    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 17891    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 824      |\n",
      "|    fps              | 11360    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 17992    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 828      |\n",
      "|    fps              | 11360    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 18094    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 832      |\n",
      "|    fps              | 11362    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 18153    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 836      |\n",
      "|    fps              | 11364    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 18292    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 840      |\n",
      "|    fps              | 11364    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 18406    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 844      |\n",
      "|    fps              | 11364    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 18462    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 848      |\n",
      "|    fps              | 11354    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 18526    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 852      |\n",
      "|    fps              | 11353    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 18603    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 856      |\n",
      "|    fps              | 11354    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 18686    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 860      |\n",
      "|    fps              | 11351    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 18784    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 864      |\n",
      "|    fps              | 11356    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 18906    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 868      |\n",
      "|    fps              | 11353    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 19003    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 872      |\n",
      "|    fps              | 11357    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 19092    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 876      |\n",
      "|    fps              | 11361    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 19179    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 880      |\n",
      "|    fps              | 11361    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 19258    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 884      |\n",
      "|    fps              | 11363    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 19352    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 888      |\n",
      "|    fps              | 11370    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 19471    |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 892      |\n",
      "|    fps              | 11371    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 19586    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 896      |\n",
      "|    fps              | 11377    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 19744    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 900      |\n",
      "|    fps              | 11384    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 19863    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 904      |\n",
      "|    fps              | 11389    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 19964    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 908      |\n",
      "|    fps              | 11382    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 20098    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 912      |\n",
      "|    fps              | 11386    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 20207    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 916      |\n",
      "|    fps              | 11389    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 20274    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 920      |\n",
      "|    fps              | 11384    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 20346    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 924      |\n",
      "|    fps              | 11389    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 20456    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 928      |\n",
      "|    fps              | 11395    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 20558    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 932      |\n",
      "|    fps              | 11393    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 20669    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 936      |\n",
      "|    fps              | 11401    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 20792    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 940      |\n",
      "|    fps              | 11406    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 20888    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 944      |\n",
      "|    fps              | 11405    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 20976    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 948      |\n",
      "|    fps              | 11401    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 21037    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 952      |\n",
      "|    fps              | 11399    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 21101    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 956      |\n",
      "|    fps              | 11401    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 21200    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 960      |\n",
      "|    fps              | 11398    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 21262    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 964      |\n",
      "|    fps              | 11396    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 21338    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 968      |\n",
      "|    fps              | 11399    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 21432    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 972      |\n",
      "|    fps              | 11397    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 21532    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 976      |\n",
      "|    fps              | 11396    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 21598    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 980      |\n",
      "|    fps              | 11398    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 21703    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 984      |\n",
      "|    fps              | 11389    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 21792    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 988      |\n",
      "|    fps              | 11391    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 21861    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 992      |\n",
      "|    fps              | 11404    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 22023    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 996      |\n",
      "|    fps              | 11402    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 22073    |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1000     |\n",
      "|    fps              | 11395    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 22140    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1004     |\n",
      "|    fps              | 11397    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 22211    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1008     |\n",
      "|    fps              | 11400    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 22365    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1012     |\n",
      "|    fps              | 11405    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 22467    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1016     |\n",
      "|    fps              | 11404    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 22559    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1020     |\n",
      "|    fps              | 11402    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 22622    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1024     |\n",
      "|    fps              | 11410    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 22729    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1028     |\n",
      "|    fps              | 11406    |\n",
      "|    time_elapsed     | 1        |\n",
      "|    total_timesteps  | 22813    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1032     |\n",
      "|    fps              | 11407    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 22916    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1036     |\n",
      "|    fps              | 11404    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 22990    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1040     |\n",
      "|    fps              | 11403    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 23068    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1044     |\n",
      "|    fps              | 11406    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 23195    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1048     |\n",
      "|    fps              | 11403    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 23269    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1052     |\n",
      "|    fps              | 11407    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 23338    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1056     |\n",
      "|    fps              | 11402    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 23407    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1060     |\n",
      "|    fps              | 11404    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 23513    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1064     |\n",
      "|    fps              | 11408    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 23591    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1068     |\n",
      "|    fps              | 11406    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 23666    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1072     |\n",
      "|    fps              | 11403    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 23745    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1076     |\n",
      "|    fps              | 11402    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 23821    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1080     |\n",
      "|    fps              | 11400    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 23897    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1084     |\n",
      "|    fps              | 11399    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 23974    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1088     |\n",
      "|    fps              | 11404    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 24080    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1092     |\n",
      "|    fps              | 11402    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 24145    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1096     |\n",
      "|    fps              | 11401    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 24221    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1100     |\n",
      "|    fps              | 11401    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 24359    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1104     |\n",
      "|    fps              | 11399    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 24439    |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1108     |\n",
      "|    fps              | 11399    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 24541    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1112     |\n",
      "|    fps              | 11404    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 24620    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1116     |\n",
      "|    fps              | 11402    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 24707    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1120     |\n",
      "|    fps              | 11406    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 24807    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1124     |\n",
      "|    fps              | 11410    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 24944    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1128     |\n",
      "|    fps              | 11405    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 25014    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1132     |\n",
      "|    fps              | 11404    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 25090    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1136     |\n",
      "|    fps              | 11405    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 25219    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1140     |\n",
      "|    fps              | 11410    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 25319    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1144     |\n",
      "|    fps              | 11409    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 25411    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1148     |\n",
      "|    fps              | 11414    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 25525    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1152     |\n",
      "|    fps              | 11412    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 25599    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1156     |\n",
      "|    fps              | 11409    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 25672    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1160     |\n",
      "|    fps              | 11412    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 25798    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1164     |\n",
      "|    fps              | 11417    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 25889    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1168     |\n",
      "|    fps              | 11420    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 26018    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1172     |\n",
      "|    fps              | 11422    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 26159    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1176     |\n",
      "|    fps              | 11422    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 26227    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1180     |\n",
      "|    fps              | 11424    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 26347    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1184     |\n",
      "|    fps              | 11425    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 26411    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1188     |\n",
      "|    fps              | 11424    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 26499    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1192     |\n",
      "|    fps              | 11424    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 26579    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1196     |\n",
      "|    fps              | 11423    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 26669    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1200     |\n",
      "|    fps              | 11421    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 26744    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1204     |\n",
      "|    fps              | 11421    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 26816    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1208     |\n",
      "|    fps              | 11418    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 26890    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1212     |\n",
      "|    fps              | 11421    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 27004    |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1216     |\n",
      "|    fps              | 11422    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 27109    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1220     |\n",
      "|    fps              | 11425    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 27196    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1224     |\n",
      "|    fps              | 11424    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 27315    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1228     |\n",
      "|    fps              | 11427    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 27436    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1232     |\n",
      "|    fps              | 11430    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 27534    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1236     |\n",
      "|    fps              | 11428    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 27621    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1240     |\n",
      "|    fps              | 11430    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 27694    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1244     |\n",
      "|    fps              | 11429    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 27753    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1248     |\n",
      "|    fps              | 11430    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 27857    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1252     |\n",
      "|    fps              | 11427    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 27965    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1256     |\n",
      "|    fps              | 11435    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 28122    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1260     |\n",
      "|    fps              | 11435    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 28222    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1264     |\n",
      "|    fps              | 11437    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 28334    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1268     |\n",
      "|    fps              | 11441    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 28434    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1272     |\n",
      "|    fps              | 11439    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 28510    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1276     |\n",
      "|    fps              | 11439    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 28612    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1280     |\n",
      "|    fps              | 11435    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 28659    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1284     |\n",
      "|    fps              | 11439    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 28764    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1288     |\n",
      "|    fps              | 11435    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 28824    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1292     |\n",
      "|    fps              | 11431    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 28882    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1296     |\n",
      "|    fps              | 11433    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 28954    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1300     |\n",
      "|    fps              | 11431    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 29030    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1304     |\n",
      "|    fps              | 11433    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 29108    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1308     |\n",
      "|    fps              | 11430    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 29179    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1312     |\n",
      "|    fps              | 11429    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 29245    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1316     |\n",
      "|    fps              | 11428    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 29324    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1320     |\n",
      "|    fps              | 11429    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 29416    |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1324     |\n",
      "|    fps              | 11429    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 29503    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1328     |\n",
      "|    fps              | 11428    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 29583    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1332     |\n",
      "|    fps              | 11431    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 29709    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1336     |\n",
      "|    fps              | 11436    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 29813    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1340     |\n",
      "|    fps              | 11436    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 29892    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1344     |\n",
      "|    fps              | 11433    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 29988    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1348     |\n",
      "|    fps              | 11431    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 30069    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1352     |\n",
      "|    fps              | 11433    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 30176    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1356     |\n",
      "|    fps              | 11437    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 30302    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1360     |\n",
      "|    fps              | 11435    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 30364    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1364     |\n",
      "|    fps              | 11434    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 30456    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1368     |\n",
      "|    fps              | 11432    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 30531    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1372     |\n",
      "|    fps              | 11427    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 30593    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1376     |\n",
      "|    fps              | 11429    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 30690    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1380     |\n",
      "|    fps              | 11428    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 30779    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1384     |\n",
      "|    fps              | 11438    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 30976    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1388     |\n",
      "|    fps              | 11432    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 31024    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1392     |\n",
      "|    fps              | 11431    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 31175    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1396     |\n",
      "|    fps              | 11436    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 31315    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1400     |\n",
      "|    fps              | 11434    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 31365    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1404     |\n",
      "|    fps              | 11435    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 31436    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1408     |\n",
      "|    fps              | 11436    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 31558    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1412     |\n",
      "|    fps              | 11435    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 31660    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1416     |\n",
      "|    fps              | 11440    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 31808    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1420     |\n",
      "|    fps              | 11435    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 31864    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1424     |\n",
      "|    fps              | 11444    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 31999    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1428     |\n",
      "|    fps              | 11443    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 32086    |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1432     |\n",
      "|    fps              | 11443    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 32189    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1436     |\n",
      "|    fps              | 11445    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 32274    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1440     |\n",
      "|    fps              | 11445    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 32371    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1444     |\n",
      "|    fps              | 11448    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 32516    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1448     |\n",
      "|    fps              | 11451    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 32591    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1452     |\n",
      "|    fps              | 11448    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 32674    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1456     |\n",
      "|    fps              | 11444    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 32750    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1460     |\n",
      "|    fps              | 11447    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 32838    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1464     |\n",
      "|    fps              | 11449    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 32923    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1468     |\n",
      "|    fps              | 11447    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 32997    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1472     |\n",
      "|    fps              | 11446    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 33063    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1476     |\n",
      "|    fps              | 11447    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 33150    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1480     |\n",
      "|    fps              | 11443    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 33256    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1484     |\n",
      "|    fps              | 11442    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 33338    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1488     |\n",
      "|    fps              | 11440    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 33402    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1492     |\n",
      "|    fps              | 11442    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 33484    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1496     |\n",
      "|    fps              | 11447    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 33623    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1500     |\n",
      "|    fps              | 11443    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 33702    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1504     |\n",
      "|    fps              | 11448    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 33819    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1508     |\n",
      "|    fps              | 11447    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 33913    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1512     |\n",
      "|    fps              | 11447    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 33994    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1516     |\n",
      "|    fps              | 11451    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 34144    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1520     |\n",
      "|    fps              | 11449    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 34205    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1524     |\n",
      "|    fps              | 11448    |\n",
      "|    time_elapsed     | 2        |\n",
      "|    total_timesteps  | 34297    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1528     |\n",
      "|    fps              | 11448    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 34366    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1532     |\n",
      "|    fps              | 11442    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 34430    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1536     |\n",
      "|    fps              | 11442    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 34530    |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1540     |\n",
      "|    fps              | 11442    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 34600    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1544     |\n",
      "|    fps              | 11443    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 34659    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1548     |\n",
      "|    fps              | 11442    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 34759    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1552     |\n",
      "|    fps              | 11442    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 34840    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1556     |\n",
      "|    fps              | 11443    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 34925    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1560     |\n",
      "|    fps              | 11442    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 35034    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1564     |\n",
      "|    fps              | 11439    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 35106    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1568     |\n",
      "|    fps              | 11437    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 35185    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1572     |\n",
      "|    fps              | 11439    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 35269    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1576     |\n",
      "|    fps              | 11438    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 35347    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1580     |\n",
      "|    fps              | 11439    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 35441    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1584     |\n",
      "|    fps              | 11440    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 35535    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1588     |\n",
      "|    fps              | 11441    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 35647    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1592     |\n",
      "|    fps              | 11441    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 35715    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1596     |\n",
      "|    fps              | 11442    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 35820    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1600     |\n",
      "|    fps              | 11442    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 35888    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1604     |\n",
      "|    fps              | 11443    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 35979    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1608     |\n",
      "|    fps              | 11442    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 36065    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1612     |\n",
      "|    fps              | 11440    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 36138    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1616     |\n",
      "|    fps              | 11442    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 36224    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1620     |\n",
      "|    fps              | 11442    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 36318    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1624     |\n",
      "|    fps              | 11444    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 36397    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1628     |\n",
      "|    fps              | 11443    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 36473    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1632     |\n",
      "|    fps              | 11446    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 36561    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1636     |\n",
      "|    fps              | 11443    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 36620    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1640     |\n",
      "|    fps              | 11441    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 36707    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1644     |\n",
      "|    fps              | 11444    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 36788    |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1648     |\n",
      "|    fps              | 11446    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 36897    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1652     |\n",
      "|    fps              | 11444    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 36984    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1656     |\n",
      "|    fps              | 11443    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 37047    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1660     |\n",
      "|    fps              | 11442    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 37122    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1664     |\n",
      "|    fps              | 11441    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 37211    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1668     |\n",
      "|    fps              | 11442    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 37316    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1672     |\n",
      "|    fps              | 11439    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 37422    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1676     |\n",
      "|    fps              | 11439    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 37487    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1680     |\n",
      "|    fps              | 11435    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 37567    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1684     |\n",
      "|    fps              | 11437    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 37663    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1688     |\n",
      "|    fps              | 11433    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 37732    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1692     |\n",
      "|    fps              | 11433    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 37808    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1696     |\n",
      "|    fps              | 11431    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 37870    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1700     |\n",
      "|    fps              | 11431    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 37973    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1704     |\n",
      "|    fps              | 11434    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 38068    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1708     |\n",
      "|    fps              | 11434    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 38158    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1712     |\n",
      "|    fps              | 11432    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 38232    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1716     |\n",
      "|    fps              | 11431    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 38307    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1720     |\n",
      "|    fps              | 11434    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 38399    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1724     |\n",
      "|    fps              | 11431    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 38472    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1728     |\n",
      "|    fps              | 11432    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 38579    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1732     |\n",
      "|    fps              | 11430    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 38641    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1736     |\n",
      "|    fps              | 11432    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 38727    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1740     |\n",
      "|    fps              | 11431    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 38809    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1744     |\n",
      "|    fps              | 11430    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 38884    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1748     |\n",
      "|    fps              | 11434    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 38978    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1752     |\n",
      "|    fps              | 11431    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 39038    |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1756     |\n",
      "|    fps              | 11432    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 39132    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1760     |\n",
      "|    fps              | 11433    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 39221    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1764     |\n",
      "|    fps              | 11431    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 39294    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1768     |\n",
      "|    fps              | 11434    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 39395    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1772     |\n",
      "|    fps              | 11433    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 39448    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1776     |\n",
      "|    fps              | 11431    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 39510    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1780     |\n",
      "|    fps              | 11427    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 39572    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1784     |\n",
      "|    fps              | 11427    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 39640    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1788     |\n",
      "|    fps              | 11425    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 39722    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1792     |\n",
      "|    fps              | 11425    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 39781    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1796     |\n",
      "|    fps              | 11425    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 39903    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1800     |\n",
      "|    fps              | 11424    |\n",
      "|    time_elapsed     | 3        |\n",
      "|    total_timesteps  | 39974    |\n",
      "----------------------------------\n"
     ]
    }
   ],
   "source": [
    "for algo_name in algos:\n",
    "    log_path,render_path,model_path=create_paths(algo_name, env_name)\n",
    "    env_name='CartPole-v0'\n",
    "    env=gym.make(env_name)\n",
    "    env=DummyVecEnv([lambda:env])\n",
    "    print('********',algo_name,'********')\n",
    "    model=train(algo_name, env_name,env,log_path,os.path.join(model_path,\"model\"))\n",
    "    env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "19a94f08",
   "metadata": {},
   "outputs": [],
   "source": [
    "del model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10565c7b",
   "metadata": {},
   "source": [
    "### Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "aa5bca8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "env_name='CartPole-v0'\n",
    "env=gym.make(env_name)\n",
    "env=DummyVecEnv([lambda:env])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1ceb8b10",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "******* Evaluating PPO  *******\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\ai_gym\\lib\\site-packages\\stable_baselines3\\common\\evaluation.py:69: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  UserWarning,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean reward for  PPO is:  200.0\n",
      "('******* Evaluation done for:', 'PPO', ' *******')\n",
      "******* Evaluating A2C  *******\n",
      "Mean reward for  A2C is:  99.4\n",
      "('******* Evaluation done for:', 'A2C', ' *******')\n",
      "******* Evaluating DQN  *******\n",
      "Mean reward for  DQN is:  17.7\n",
      "('******* Evaluation done for:', 'DQN', ' *******')\n"
     ]
    }
   ],
   "source": [
    "rewards={}\n",
    "for algo in algos:\n",
    "    print('******* Evaluating',algo,' *******')\n",
    "    model_load_path=os.path.join(env_name,algo,\"model.zip\")\n",
    "    model=eval(algo).load(model_load_path, env)\n",
    "    mean_rew,_=evaluate_policy(model,env)\n",
    "    print('Mean reward for ',algo, 'is: ',mean_rew)\n",
    "    rewards[algo]=mean_rew\n",
    "    env.close()\n",
    "    print(('******* Evaluation done for:',algo,' *******'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "fd7d25b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def addlabels(x,y):\n",
    "    for i in range(len(x)):\n",
    "        plt.text(i,y[i],y[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f8b5ff17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAAAeHElEQVR4nO3de5hVdd338fdHUJMCTxxEYBxFVA7aCFNiGWqmt6ZJSqmkgYUPYXqrt153mtWdlj2SaR7K6sYkoYy0ROFRJBXzWGSMcTYFDQWaAPEAhorg9/ljr1ltYQ7LYfZeM8zndV372mv91um72cx8Z63fSRGBmZkZwA55B2BmZq2Hk4KZmaWcFMzMLOWkYGZmKScFMzNLdcw7gG3RtWvXqKyszDsMM7M2paam5uWI6FbftjadFCorK5kzZ07eYZiZtSmSXmxomx8fmZlZyknBzMxSTgptyPLlyzn66KMZMGAAAwcO5MYbbwTglVde4dhjj6Vfv34ce+yxvPrqqwBEBBdccAH7778/hxxyCE8//XS9562pqeHggw9m//3354ILLsC93M3aLyeFNqRjx45cd911LF68mNmzZ3PzzTezePFixo8fzzHHHMOSJUs45phjGD9+PAD3338/S5YsYcmSJUyYMIFzzz233vOee+653HLLLem+M2fOLOfHMrNWxEmhDenZsyeDBw8GoHPnzvTv35+VK1cybdo0Ro8eDcDo0aO55557AJg2bRqjRo1CEkOHDuW1116jtrb2Peesra1l3bp1DB06FEmMGjUqPd7M2h8nhTZq2bJl/PWvf+Wwww5j1apV9OzZE4C99tqLVatWAbBy5Ur69OmTHtO7d29Wrlz5nvOsXLmS3r17N7qPmbUfJUsKkvpI+oOkxZIWSbowKd9D0oOSliTvuyflknSTpKWS5ksaXKrY2ro33niDESNGcMMNN9ClS5f3bJOEpJwiM7O2rpR3CpuASyJiADAUOE/SAOAyYFZE9ANmJesAJwD9ktdY4KcljK3NeueddxgxYgRnnnkmp556KgA9evRIHwvV1tbSvXt3AHr16sXy5cvTY1esWEGvXr3ec75evXqxYsWKRvcxs/ajZEkhImoj4ulkeT3wDNALGA5MSnabBHw2WR4OTI6C2cBuknqWKr62KCIYM2YM/fv35+KLL07LTz75ZCZNKvyTTpo0ieHDh6flkydPJiKYPXs2u+66a/qYqU7Pnj3p0qULs2fPJiKYPHlyeryZtT8qR/NDSZXAY8Ag4KWI2C0pF/BqROwm6V5gfEQ8kWybBVwaEXO2ONdYCncSVFRUDHnxxQY75jWp8rL7mn1sHt5asYhVt1/Kjt0qIXlEtPuwUey094G8PG08m9atoWOX7nQdfhkddulMRPDKgz/jrb/XoI47s+enL2Lnnv0A+Mcv/pO9v/QjAN6uXcLaGdcTmzayy35D2P1T47b5EdSy8Sdu0/FmVjqSaiKiut5tpU4Kkj4EPAp8LyKmSnqtLikk21+NiN2zJoVi1dXVsS3DXLS1pNCWOCmYtV6NJYWStj6StCNwF3B7RExNilfVPRZK3lcn5SuBPkWH907KzMysTErZ+kjArcAzEfHDok3TgdHJ8mhgWlH5qKQV0lDg9Yh4b6N6MzMrqVKOkvpx4IvAAklzk7LLgfHAnZLGAC8CpyXbZgCfBpYCG4AvlTA2MzOrR8mSQlI30FBt5TH17B/AeaWKx8zMmuYezWZmlnJSMDOzlJOCmZmlnBTMzCzlpGBmZiknBTMzSzkpmJlZyknBzMxSTgpmZpZyUjAzs5STgpmZpZwUzMws5aRgZmYpJwUzM0s5KZiZWcpJwczMUqWcjnOipNWSFhaV3SFpbvJaVjcjm6RKSW8WbftZqeIyM7OGlXI6ztuAHwOT6woi4vS6ZUnXAa8X7f98RFSVMB4zM2tCKafjfExSZX3bJInC3MyfLNX1zczs/curTuETwKqIWFJUtq+kv0p6VNIncorLzKxdK+Xjo8aMBKYUrdcCFRGxVtIQ4B5JAyNi3ZYHShoLjAWoqKgoS7BmZu1F2e8UJHUETgXuqCuLiLcjYm2yXAM8DxxQ3/ERMSEiqiOiulu3buUI2cys3cjj8dGngL9FxIq6AkndJHVIlvcD+gEv5BCbmVm7VsomqVOAPwEHSlohaUyy6Qze++gIYBgwP2mi+jtgXES8UqrYzMysfqVsfTSygfKz6ym7C7irVLGYmVk27tFsZmYpJwUzM0s5KZiZWcpJwczMUk4KZmaWclIwM7OUk4KZmaWcFMzMLOWkYGZmKScFMzNLNTjMhaTBjR0YEU+3fDhmZpanxsY+uq6RbYFnTTMz2+40mBQi4uhyBmJmZvlrsk5BUidJ35Q0IVnvJ+mk0odmZmbllqWi+RfARuBjyfpK4KqSRWRmZrnJkhT6RsQ1wDsAEbEBUEmjMjOzXGRJChsl7UKhchlJfYG3SxqVmZnlIktS+DYwE+gj6XZgFvC1pg6SNFHSakkLi8qukLRS0tzk9emibV+XtFTSs5L+oxmfxczMtlGT03FGxIOSngaGUnhsdGFEvJzh3LcBPwYmb1F+fURcW1wgaQCFuZsHAnsDD0k6ICI2Z7iOmZm1kPfTea02ea+QVNFU57WIeExSZcY4hgO/iYi3gb9LWgp8FPhTxuPNzKwFZOm89gGgGphH4U7hEGAOcHgzr3m+pFHJOS6JiFeBXsDson1WJGVbkTQWGAtQUVHRzBDMzKw+DdYpRMTRSQe2WmBwRFRHxBDgUArNUpvjp0BfoCo5b2O9phuKa0ISS3W3bt2aGYaZmdUnS0XzgRGxoG4lIhYC/ZtzsYhYFRGbI+Jd4BYKj4igkGT6FO3am+YnHjMza6YsSWG+pJ9LOip53QLMb87FJPUsWj0FqGuZNB04Q9LOkvYF+gFPNecaZmbWfE22PgK+BJwLXJisP0bhMVCjJE0BjgK6SlpBoWnrUZKqKPR5WAZ8BSAiFkm6E1gMbALOc8sjM7Pyy9Ik9S1JNwMPUfhl/mxEvJPhuJH1FN/ayP7fA77X1HnNzKx0mkwKko4CJlH4y14UOrGNjojHShqZmZmVXZbHR9cBx0XEswCSDgCmAENKGZiZmZVflormHesSAkBEPAfsWLqQzMwsL1nuFOZI+jnwq2T9TAodz8zMbDuTJSmcC5wHXJCsPw78pGQRmZlZbrK0Pnob+GHyMjOz7ViW6ThPkvRXSa9IWidpvaR15QjOzMzKK8vjoxuAU4EFERGlDcfMzPKUpfXRcmChE4KZ2fYvy53C14AZkh6laBrOiHAdg5nZdiZLUvge8AaFeRV2Km04ZmaWpyxJYe+IGFTySMzMLHdZ6hRmSDqu5JGYmVnusiSFc4GZkt50k1Qzs+1bls5rncsRiJmZ5S/LnYKZmbUTTgpmZpZqMCkkcyU3m6SJklZLWlhU9gNJf5M0X9LdknZLyiuTOou5yetn23JtMzNrnsbuFH4HIGlWM899G3D8FmUPAoMi4hDgOeDrRduej4iq5DWumdc0M7Nt0FhF8w6SLgcOkHTxlhub6tEcEY9Jqtyi7IGi1dnA595HrGZmVmKN3SmcAWymkDg61/PaVl8G7i9a3zcZjfVRSZ9o6CBJYyXNkTRnzZo1LRCGmZnVafBOIZmC8/uS5kfE/Q3t1xySvgFsAm5PimqBiohYK2kIcI+kgRGxVX+IiJgATACorq72IH1mZi0oS+ujP0r6Yd1f55Kuk7Rrcy8o6WzgJODMupFXI+LtiFibLNcAzwMHNPcaZmbWPFmSwkRgPXBa8loH/KI5F5N0PIVRV0+OiA1F5d0kdUiW9wP6AS805xpmZtZ8WQbE6xsRI4rWr5Q0t6mDJE0BjgK6SloBfJtCa6OdgQclAcxOWhoNA74j6R3gXWBcRLzyfj6ImZltuyxJ4U1JR0TEEwCSPg682dRBETGynuJbG9j3LuCuDLGYmVkJZUkK44DJRfUIrwKjSxeSmZnlJcuAePOAD0vqkqx7hFQzs+1UljsFwMnAzKw98IB4ZmaWclIwM7NUk0lB0ucldU6WvylpqqTBpQ/NzMzKLcudwrciYr2kI4BPUWhW+tPShmVmZnnIkhQ2J+8nAhMi4j5gp9KFZGZmecmSFFZK+l/gdGCGpJ0zHmdmZm1Mll/upwG/B/4jIl4D9gD+u5RBmZlZPhrspyBpj6LVR4rK3gbmlDYsMzPLQ2Od12qAAARUUBjeQsBuwEvANs3hbGZmrU+Dj48iYt+I2A94CPhMRHSNiD0pzIXwQEPHmZlZ25WlTmFoRMyoW0lmYftY6UIyM7O8ZEkK/0g6rVUmr28A/yh1YGbbgxtvvJFBgwYxcOBAbrjhBgDmzZvH4YcfzsEHH8xnPvMZ1q1reFixzZs3c+ihh3LSSSeVKWJr77IkhZFAN+BuYGqyXN9cCWZWZOHChdxyyy089dRTzJs3j3vvvZelS5dyzjnnMH78eBYsWMApp5zCD37wgwbPceONN9K/f/8yRm3tXaNJIZki80cRcWFEHBoRgyPiIs+KZta0Z555hsMOO4xOnTrRsWNHjjzySKZOncpzzz3HsGHDADj22GO5667655dasWIF9913H+ecc045w7Z2rtGkEBGbgX0kNasHs6SJklZLWlhUtoekByUtSd53T8ol6SZJSyXN9/hK1tYNGjSIxx9/nLVr17JhwwZmzJjB8uXLGThwINOmTQPgt7/9LcuXL6/3+IsuuohrrrmGHXZwX1Ernyz/214AnpT0LUkX170ynv824Pgtyi4DZkVEP2BWsg5wAtAveY3F4ytZG9e/f38uvfRSjjvuOI4//niqqqro0KEDEydO5Cc/+QlDhgxh/fr17LTT1n9z3XvvvXTv3p0hQ4bkELm1Z1mSwvPAvcm+nYteTYqIx4AtHzUNByYly5OAzxaVT46C2cBuknpmuY5ZazVmzBhqamp47LHH2H333TnggAM46KCDeOCBB6ipqWHkyJH07dt3q+OefPJJpk+fTmVlJWeccQYPP/wwZ511Vg6fwNqbLNNxXtnC1+wREbXJ8j+BHslyL6D4PnpFUlZbVIaksRTuJKioqGjh0Mxa1urVq+nevTsvvfQSU6dOZfbs2WnZu+++y1VXXcW4ceO2Ou7qq6/m6quvBuCRRx7h2muv5Ve/+lW5w7d2qMmkIKkb8DVgIPCBuvKI+OS2XjwiQlK8z2MmABMAqqur39ex1rZVXnZf3iG8b/+8/Wu8++Z62KEDu3/yHKrGP8m6OdNY/3Ths3Q64GM82rM737nsPjatX8vamTfR4/Pv/TvsrZfms+5vq0v6+ZeNP7Fk57a2JcsczbcDd1DoyTwOGA2s2YZrrpLUMyJqk8dDq5PylUCfov16J2VmbdZeZ16zVVmX6uF0qR6+VXnHzntulRAAPlBxCB+oOKQk8ZltKUudwp4RcSvwTkQ8GhFfBrblLmE6hcRC8j6tqHxU0gppKPB60WMmMzMrgyx3Cu8k77WSTqTQm3mPRvZPSZoCHAV0lbQC+DYwHrhT0hjgRQpDcwPMAD4NLAU2AF/K+BnMzKyFZEkKV0naFbgE+BHQBfivLCePiIZ6Ph9Tz74BnJflvGZmVhpZksJDEfEW8DpwdInjMTOzHGVJCgslrQIeT15PRMTrpQ3LzMzy0GRFc0TsT2EAvAXAicA8SXNLHJeZmeUgSz+F3sDHgU8AHwYWAU+UOC4zM8tBlsdHLwF/Af5vRGzd9dLMzLYbWfopHApMBr4g6U+SJifNSc3MbDuTZeyjeZKepzAw3ieAs4AjgVtLHJuZmZVZljqFOcDOwB8ptD4aFhEvljowMzMrvyx1CidExLaMdWRmZm1EljqFHSTdKul+AEkDXKdgZrZ9ypIUbgN+D+ydrD8HXFSieMzMLEdZkkLXiLgTeBcgIjYBm0salZmZ5SJLUviXpD2BAKgb1rqkUZmZWS6yVDRfTGGug76SngS6AZ8raVRmZpaLRpOCpA4U+iQcCRwICHg2It5p7DgzM2ubGn18FBGbgZERsSkiFkXEQicEM7PtV5bHR09K+jGFeZr/VVcYEU+XLCozM8tFlqRQlbx/p6gsaOY8zZIOpJBg6uwH/A+wG/B/gLqOcpdHxIzmXMPMzJony9hHLTrbWkQ8S5JokjqLlcDdFOZkvj4irm3J65mZWXZZmqSW0jHA8x5Lycysdcg7KZwBTClaP1/SfEkTJe1e3wGSxkqaI2nOmjUeksnMrCXllhQk7QScDPw2Kfop0JfCo6Va4Lr6jouICRFRHRHV3bp1K0eoZmbtRpaKZiR9DKgs3j8iJm/jtU8Ano6IVcn5VhVd7xbg3m08v5mZvU9Z5lP4JYW/4Ofy7zGPgsJsbNtiJEWPjiT1jIjaZPUUYOE2nt/MzN6nLHcK1cCAiIiWuqikDwLHAl8pKr5GUhWFhLNsi21mZlYGWZLCQmAvCs/5W0RE/AvYc4uyL7bU+c3MrHmyJIWuwGJJTwFv1xVGxMkli8rMzHKRJSlcUeogzMysdcjSo/nRcgRiZmb5a7KfgqShkv4i6Q1JGyVtlrSuHMGZmVl5Zem89mMKzUeXALsA5wA3lzIoMzPLR6YezRGxFOgQEZsj4hfA8aUNy8zM8pClonlDMiTFXEnXUGiamveYSWZmVgJZfrl/MdnvfAqT7PQBRpQyKDMzy0eW1kcvStoF6BkRV5YhJjMzy0mW1kefoTDu0cxkvUrS9BLHZWZmOcjy+OgK4KPAawARMRfYt2QRmZlZbrIkhXci4vUtylpscDwzM2s9srQ+WiTpC0AHSf2AC4A/ljYsMzPLQ5Y7hf8EBlIYDG8KsA64qIQxmZlZTrK0PtoAfCN5mZnZdqzBpNBUCyMPnW1mtv1p7E7hcGA5hUdGfwZUlojMzCw3jSWFvShMmTkS+AJwHzAlIha1xIUlLQPWU5j3eVNEVEvaA7gDqKQwJedpEfFqS1zPzMya1mBFczL43cyIGA0MBZYCj0g6vwWvf3REVEVEdbJ+GTArIvoBs5J1MzMrk0YrmiXtDJxI4W6hErgJuLuE8QwHjkqWJwGPAJeW8HpmZlaksYrmycAgYAZwZUQsbOFrB/CApAD+NyImAD0iojbZ/k+gRz1xjQXGAlRUVLRwSGZm7VtjdwpnURgV9ULgAimtZxYQEdFlG699RESslNQdeFDS34o3RkQkCYMtyicAEwCqq6vds9rMrAU1mBQioqRzJkTEyuR9taS7KYyvtEpSz4ioldQTWF3KGMzM7L1ymSxH0gclda5bBo4DFgLTgdHJbqOBaXnEZ2bWXmUZ+6gUegB3J4+kOgK/joiZkv4C3ClpDPAicFpO8ZmZtUu5JIWIeAH4cD3la4Fjyh+RmZmB51o2M7MiTgpmZpZyUjAzs5STgpmZpZwUzMws5aRgZmYpJwUzM0s5KZiZWcpJwczMUk4KZmaWclIwM7OUk4KZmaWcFMzMLOWkYGZmKScFMzNLOSmYmVnKScHMzFJlTwqS+kj6g6TFkhZJujApv0LSSklzk9enyx2bmVl7l8d0nJuASyLiaUmdgRpJDybbro+Ia3OIyczMyCEpREQtUJssr5f0DNCr3HGYmdnWcq1TkFQJHAr8OSk6X9J8SRMl7d7AMWMlzZE0Z82aNeUK1czaiS9/+ct0796dQYMGpWWnn346VVVVVFVVUVlZSVVV1VbHPfvss+k+VVVVdOnShRtuuKF8gbeQ3JKCpA8BdwEXRcQ64KdAX6CKwp3EdfUdFxETIqI6Iqq7detWrnDNrJ04++yzmTlz5nvK7rjjDubOncvcuXMZMWIEp5566lbHHXjggek+NTU1dOrUiVNOOaVcYbeYPOoUkLQjhYRwe0RMBYiIVUXbbwHuzSM2M2vfhg0bxrJly+rdFhHceeedPPzww42eY9asWfTt25d99tmnBBGWVh6tjwTcCjwTET8sKu9ZtNspwMJyx2Zm1pjHH3+cHj160K9fv0b3+81vfsPIkSPLFFXLyuNO4ePAF4EFkuYmZZcDIyVVAQEsA76SQ2xmZg2aMmVKk7/sN27cyPTp07n66qvLFFXLyqP10ROA6tk0o9yxmJlltWnTJqZOnUpNTU2j+91///0MHjyYHj16lCmyluUezWZmGTz00EMcdNBB9O7du9H9stxNtGa5VDSbWftQedl9eYfwvq2Zfg1vv7SAzW+uo2Pnrux6xJl0/vBxvHzf9ey896D3fKZN69eyduZN9Pj8lQC8u/EtVk6bwR97nsqVJf7sy8afWJLzOimYmRXpdvLX6i3veuJ/bVXWsfOeaUIA2GGnD9Dnwikli60c/PjIzMxSTgpmZpZyUjAzs5STgpmZpZwUzMws5aRgZmYpJwUzM0s5KZiZWcpJwczMUk4KZmaWclIwM7OUk4KZmaWcFMzMLOWkYGZmqVaXFCQdL+lZSUslXZZ3PGZm7UmrSgqSOgA3AycAAyjM2zwg36jMzNqPVpUUgI8CSyPihYjYCPwGGJ5zTGZm7UZrm3mtF7C8aH0FcFjxDpLGAmOT1TckPVum2PLWFXg57yCy0vfzjqBVaDPfmb8voA19X7DN39k+DW1obUmhSRExAZiQdxzlJmlORFTnHYdl5++sbfH3VdDaHh+tBPoUrfdOyszMrAxaW1L4C9BP0r6SdgLOAKbnHJOZWbvRqh4fRcQmSecDvwc6ABMjYlHOYbUW7e6R2XbA31nb4u8LUETkHYOZmbUSre3xkZmZ5chJwczMUk4KrYCkzZLmSloo6beSOjVR3lvSNElLJD0v6cakYt7KSNJnJYWkg5L1Kkl/krRI0nxJpxftu6Ok8cl39nSy3wn5Rd9+FP0cLZI0T9IlknYo2n6EpKck/S0ZYuerRduukLRBUveisjfK/RnKyUmhdXgzIqoiYhCwERjXULkkAVOBeyKiH3AA8CHge3kE3s6NBJ5I3gE2AKMiYiBwPHCDpN2Sbd8FegKDImIw8Fmgc1mjbb/qfo4GAsdSGEbn2wCS9gJ+DYyLiIOAjwNjJJ1SdPzLwCVljjk3rmhuBSS9EREfSpbHAYdExFfrKwfuAr4dEcOKju8C/B3oExEbyv8J2h9JHwKeBY4G/l9EHFjPPvOAz1Hoa7Mc2Dci1pU1UHvPz1eyvh+F5u9dge8AERH/U7T9GOC7EfExSVckxWcDgyPilS3Pt73xnUIrIqkjhb9iFjRSPhCoKd6e/KJ5Cdi/PJEahTG5ZkbEc8BaSUOKN0r6KLAT8DyF7+UlJ4TWISJeoNDkvTv1/DwBcygMyFnnDWAicGFZAsyZk0LrsIukuRT+M74E3NpEueVvJIUBG0ne6x4hIakn8EvgSxHxbg6xWcu7CRgtabt/5NeqOq+1Y29GRFWWckmLKTySKC7rAlQAS0sVoP2bpD2ATwIHSwoKf3WGpP+mUE9wH/CNiJidHLIUqJDUxXcL+UseH20GVgOLgSHAtKJdhlD4QywVEa9J+jVwXrnizIvvFNqeWUAnSaMgnYPiOuA21yeUzeeAX0bEPhFRGRF9KNTpfAK4G5gcEb+r2zn5Xm4F0lZikrpJ+nwOsbdrkroBPwN+HIUK1ZuBsyVVJdv3pNBo47v1HP5D4Cts539MOym0Mcl/5FOAz0taAjwHvAVcnmtg7ctICr/8i90FTAKGUfglMzd5VSXbvwmsARZLWgjcC/iuoTx2qWuSCjwEPABcCRARtcBZwIRkGP5/ADdFxKNbniQiXqbwve9ctshz4NZHZmaJpI/CucCwiHg173jy4KRgZmYpPz4yM7OUk4KZmaWcFMzMLOWkYGZmKScFa9fqGem0Mmky2lLn/7mkAcny5UXlLXods5bipGDt3ZYjnbYYSR0i4pyIWJwUuS+JtXpOCtZuJSOdHgGMAc6oZ3snSXdKWizpbkl/llSdbBspaUEy18X3i455Q9J1yQiph0t6RFK1pPH8uxPV7cnuHSTdkozz/4CkXZJzPCLpeklzJD0j6SOSpiZzMVxV6n8Xa9+cFKw9a3SkU+CrwKsRMQD4FoUxcZC0N/B9CuMfVQEfkfTZ5JgPAn+OiA9HxBN1J4qIy/j3uP5nJsX9gJuTcf5fA0YUXXtjRFRTGJJhGoUxdwZR6C29Z0t8eLP6OClYe9bgSKeJI+q2R8RCYH5S/hHgkYhYExGbgNspDG8BhYHW7sp4/b9HxNxkuQaoLNo2PXlfACyKiNqIeBt4AeiT8fxm79t2PbCTWUMaGumUwgBp2+KtiNiccd+3i5Y3A7vUs+3dLfZ7F//cWgn5TsHaq4ZGOi3+K/xJ4DSApAXRwUn5U8CRkromo9SOBLYaQK0e70jascU+gVkJOClYe9XQSKdfL1r/CdAtmcPiKmAR8HoysuZlwB+AeUBNREyjaROA+UUVzWatjgfEM2tAchewY0S8JakvhWGXD4yIjTmHZlYyfjZp1rBOwB+SRz4CvuqEYNs73ymYmVnKdQpmZpZyUjAzs5STgpmZpZwUzMws5aRgZmap/w/f1ktlkGdXXwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x=list(rewards.keys())\n",
    "y=list(rewards.values())\n",
    "plt.bar(range(len(x)),y,tick_label=x)\n",
    "plt.xlabel(\"Algorithm\")\n",
    "plt.ylabel(\"Mean rewards of model\")\n",
    "addlabels(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "21fbadae",
   "metadata": {},
   "outputs": [],
   "source": [
    "del model\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1db7ecd6",
   "metadata": {},
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9c87fffe",
   "metadata": {},
   "outputs": [],
   "source": [
    "env_name='CartPole-v0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e240d916",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "******* Testing PPO  *******\n",
      "******* Testing A2C  *******\n",
      "******* Testing DQN  *******\n"
     ]
    }
   ],
   "source": [
    "rewards={}\n",
    "for algo_name in algos:\n",
    "    _,render_path,model_path=create_paths(algo_name, env_name)   \n",
    "    print('******* Testing',algo_name,' *******')\n",
    "    model_load_path=os.path.join(env_name,algo_name,\"model.zip\")\n",
    "    model=eval(algo_name).load(model_load_path, env)\n",
    "    _, total_rewards = run_experiment(env_name,env,render_path=render_path,random=False, episodes=5, model=model)\n",
    "    env.close()\n",
    "    rewards[algo_name]=total_rewards\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4f11c85d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'PPO': [200.0, 200.0, 200.0, 200.0, 200.0],\n",
       " 'A2C': [90.0, 131.0, 95.0, 89.0, 116.0],\n",
       " 'DQN': [15.0, 18.0, 19.0, 23.0, 13.0]}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b770679e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}