day_train_ta.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f158dfe0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import gym\n",
    "from gym import spaces\n",
    "import copy\n",
    "import os\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e2d42d8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pull in training data\n",
    "data = pickle.load(open('train_ta', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d23bca19",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Technical indicators list. TODO: put these in a config file\n",
    "TECHNICAL_INDICATORS_LIST = [\"macd\", \"macds\",\n",
    "                             \"boll_ub\",\"boll_lb\",\n",
    "                             \"rsi_5\", \"rsi_14\", \"rsi_30\", \n",
    "                             \"cci_30\", \"dx_30\",\n",
    "                             \"open_5_sma\", \"open_14_sma\", \"open_30_sma\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "86f849a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#stock tickers being looked at\n",
    "tickers = list(data.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "90168601",
   "metadata": {},
   "outputs": [],
   "source": [
    "#features is a list of all (non-price) features that are used in the observation\n",
    "\n",
    "features = list(data['DIS'].keys())\n",
    "features.remove('date')\n",
    "features.remove('tic')\n",
    "features.remove('price')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "id": "a9206794",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Helper function to perform softmax\n",
    "def softmax(x):\n",
    "    return np.exp(x)/sum(np.exp(x))\n",
    "\n",
    "\n",
    "INITIAL_BALANCE = 1000.0 #Start with balance of 1000 dollars\n",
    "NUM_PAST_STATES = 14 #use past 14 days of price data as part of observation\n",
    "EPISODE_LENGTH = 30 #each episode goes for 30 trading days\n",
    "\n",
    "\n",
    "#OpenAI Gym style environment for RL\n",
    "class TradeEnv(gym.Env):\n",
    "    def __init__(self, tickers):\n",
    "        super(TradeEnv, self).__init__()\n",
    "\n",
    "        self.tickers = tickers\n",
    "\n",
    "        self.data = {}\n",
    "        self.features = {}\n",
    "        self.means = {}\n",
    "        self.stds = {}\n",
    "        self.prices = {}\n",
    "        for key, value in data.items():\n",
    "            self.data[key] = value[features]\n",
    "            self.means[key] = np.mean(value[features], axis = 0)\n",
    "            self.stds[key] = np.std(value[features], axis = 0)\n",
    "            \n",
    "            #Normalize features to have zero mean and unit standard deviation\n",
    "            self.features[key] = np.divide(value[features] - self.means[key],\n",
    "                                          self.stds[key])\n",
    "            \n",
    "            self.prices[key] = value['price'].values\n",
    "        \n",
    "        self.prices = pd.DataFrame.from_dict(self.prices)\n",
    "        \n",
    "        #self.prices is a dataframe with each ticker being a key \n",
    "        #and the corresponding series representing the stock prices\n",
    "        \n",
    "        #Will be used later for normalization\n",
    "        self.price_means = np.mean(self.prices, axis = 0).values\n",
    "        self.price_stds = np.std(self.prices, axis = 0).values\n",
    "        \n",
    "\n",
    "        self.episode_length = EPISODE_LENGTH #number of trading minutes in episode\n",
    "\n",
    "        self.num_past_states = NUM_PAST_STATES #number of past days that are used in state\n",
    "\n",
    "        self.action_space = spaces.Box(low=-10, high=10, shape=(len(self.tickers) + 1,))\n",
    "                                            \n",
    "\n",
    "        obs_length = len(self.tickers)*(self.num_past_states) #observation due to past stacked states\n",
    "        obs_length += 1 #balance\n",
    "        obs_length += len(self.tickers) #holdings\n",
    "        obs_length += len(self.tickers)*len(features) #number of technical analysis features\n",
    "        self.observation_space = spaces.Box(low=-np.inf, high=np.inf,\n",
    "                                            shape=(obs_length,))\n",
    "\n",
    "\n",
    "    def step(self, action_):\n",
    "        \n",
    "        #Apply softmax to RL output so that actions sum to 1\n",
    "        action = softmax(action_)\n",
    "\n",
    "        #Liquidate past holdings\n",
    "        self.balance += np.sum(self.holdings)\n",
    "        \n",
    "        \n",
    "        #New Portfolio at end of day\n",
    "        self.holdings = self.balance*action[:-1]\n",
    "        self.balance = self.balance*action[-1]\n",
    "        \n",
    "        #Net worth at end of day\n",
    "        self.last_net_worth = self.balance + np.sum(self.holdings)\n",
    "        \n",
    "        #Step into next day\n",
    "        self.index += 1\n",
    "        #Get stock prices at next day\n",
    "        stock_obs = self.get_stock_obs(self.index)\n",
    "        self.next_prices = stock_obs[-1]\n",
    "        \n",
    "        #Update value of current holdings\n",
    "        perc_change = np.divide(self.next_prices, self.curr_prices)\n",
    "        self.holdings = np.multiply(self.holdings, perc_change)\n",
    "\n",
    "        self.curr_prices = self.next_prices\n",
    "        \n",
    "        self.net_worth = self.balance + np.sum(self.holdings)\n",
    "\n",
    "        rew = self.net_worth - self.last_net_worth # reward is the delta between last net worth and current net worth\n",
    "\n",
    "        self.steps += 1\n",
    "        done = (self.net_worth <= 0) or (self.steps >= self.episode_length)\n",
    "\n",
    "        obs = self.get_obs(stock_obs, self.balance, self.holdings, self.index)\n",
    "        self.cum_rew += rew\n",
    "\n",
    "        return obs, rew, done, {}\n",
    "    \n",
    "    \n",
    "    def get_stock_obs(self, index):\n",
    "\n",
    "        \n",
    "        ret= self.prices[index - self.num_past_states:index][self.tickers].values #stack data\n",
    "        return ret\n",
    "\n",
    "    def get_obs(self, stock_obs, balance, holdings, index):\n",
    "        #Normalize stock prices for inclusion in observations\n",
    "        prices_norm = np.divide(stock_obs - self.price_means,\n",
    "                               self.price_stds).reshape(-1,)\n",
    "        \n",
    "        feature_vals = np.array([])\n",
    "        ix = index - 1\n",
    "        #Add in features at current timestep, for each ticker\n",
    "        for tic in self.tickers:\n",
    "            feature_vals = np.append(feature_vals, (self.features[tic].iloc[ix][features].values))\n",
    "        \n",
    "        #Form observation and normalize balance and holdings\n",
    "        return np.concatenate([prices_norm, [balance/1000.0], holdings/1000.0, feature_vals])\n",
    "\n",
    "        \n",
    "    def reset(self):\n",
    "\n",
    "        self.cum_rew = 0.0\n",
    "        self.steps = 0\n",
    "        self.index = np.random.randint(2*NUM_PAST_STATES, len(self.prices) - EPISODE_LENGTH - 10)\n",
    "\n",
    "        self.init_prices = self.prices[self.index-1:self.index + EPISODE_LENGTH]\n",
    "        stock_obs = self.get_stock_obs(self.index)\n",
    "        self.holdings = np.zeros(len(self.tickers)) #holdings of each stock in number of shares\n",
    "        self.balance = INITIAL_BALANCE\n",
    "        self.last_net_worth = INITIAL_BALANCE\n",
    "        self.net_worth = INITIAL_BALANCE\n",
    "        \n",
    "        self.curr_prices = stock_obs[-1]\n",
    "        \n",
    "\n",
    "        obs = self.get_obs(stock_obs, self.balance, self.holdings, self.index)\n",
    "        return obs  # reward, done, info can't be included\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "id": "9be303b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_env = TradeEnv(tickers = tickers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "id": "35577354",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DIS</th>\n",
       "      <th>IFF</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36.500500</td>\n",
       "      <td>32.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>36.377190</td>\n",
       "      <td>33.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>36.377190</td>\n",
       "      <td>33.375000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>35.883938</td>\n",
       "      <td>32.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>34.959095</td>\n",
       "      <td>31.312500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4251</th>\n",
       "      <td>108.580002</td>\n",
       "      <td>115.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4252</th>\n",
       "      <td>107.900002</td>\n",
       "      <td>116.029999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4253</th>\n",
       "      <td>108.540001</td>\n",
       "      <td>115.279999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4254</th>\n",
       "      <td>108.120003</td>\n",
       "      <td>116.080002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4255</th>\n",
       "      <td>107.160004</td>\n",
       "      <td>115.580002</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4256 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             DIS         IFF\n",
       "0      36.500500   32.250000\n",
       "1      36.377190   33.500000\n",
       "2      36.377190   33.375000\n",
       "3      35.883938   32.000000\n",
       "4      34.959095   31.312500\n",
       "...          ...         ...\n",
       "4251  108.580002  115.250000\n",
       "4252  107.900002  116.029999\n",
       "4253  108.540001  115.279999\n",
       "4254  108.120003  116.080002\n",
       "4255  107.160004  115.580002\n",
       "\n",
       "[4256 rows x 2 columns]"
      ]
     },
     "execution_count": 258,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#This is what self.prices looks like:\n",
    "test_env.prices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "id": "f70bf6f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>volume</th>\n",
       "      <th>macd</th>\n",
       "      <th>macds</th>\n",
       "      <th>boll_ub</th>\n",
       "      <th>boll_lb</th>\n",
       "      <th>rsi_5</th>\n",
       "      <th>rsi_14</th>\n",
       "      <th>rsi_30</th>\n",
       "      <th>cci_30</th>\n",
       "      <th>dx_30</th>\n",
       "      <th>open_5_sma</th>\n",
       "      <th>open_14_sma</th>\n",
       "      <th>open_30_sma</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.401020</td>\n",
       "      <td>0.489463</td>\n",
       "      <td>0.617954</td>\n",
       "      <td>-0.353622</td>\n",
       "      <td>-0.428058</td>\n",
       "      <td>-0.481647</td>\n",
       "      <td>0.307562</td>\n",
       "      <td>0.849076</td>\n",
       "      <td>0.113132</td>\n",
       "      <td>-0.236348</td>\n",
       "      <td>-0.253884</td>\n",
       "      <td>-0.256690</td>\n",
       "      <td>-0.315423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.870514</td>\n",
       "      <td>0.426991</td>\n",
       "      <td>0.585422</td>\n",
       "      <td>-0.353426</td>\n",
       "      <td>-0.421663</td>\n",
       "      <td>-0.402383</td>\n",
       "      <td>0.333753</td>\n",
       "      <td>0.868129</td>\n",
       "      <td>0.234302</td>\n",
       "      <td>-0.201806</td>\n",
       "      <td>-0.258847</td>\n",
       "      <td>-0.256044</td>\n",
       "      <td>-0.307076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.819901</td>\n",
       "      <td>0.336381</td>\n",
       "      <td>0.540152</td>\n",
       "      <td>-0.358686</td>\n",
       "      <td>-0.407370</td>\n",
       "      <td>-0.845417</td>\n",
       "      <td>0.091875</td>\n",
       "      <td>0.672550</td>\n",
       "      <td>0.001486</td>\n",
       "      <td>-0.887515</td>\n",
       "      <td>-0.264713</td>\n",
       "      <td>-0.256690</td>\n",
       "      <td>-0.300703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.507045</td>\n",
       "      <td>0.179425</td>\n",
       "      <td>0.470598</td>\n",
       "      <td>-0.372529</td>\n",
       "      <td>-0.384293</td>\n",
       "      <td>-1.506754</td>\n",
       "      <td>-0.412795</td>\n",
       "      <td>0.248515</td>\n",
       "      <td>-0.278378</td>\n",
       "      <td>-1.161282</td>\n",
       "      <td>-0.268774</td>\n",
       "      <td>-0.256852</td>\n",
       "      <td>-0.296529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.618350</td>\n",
       "      <td>-0.118873</td>\n",
       "      <td>0.351596</td>\n",
       "      <td>-0.373050</td>\n",
       "      <td>-0.383547</td>\n",
       "      <td>-2.092038</td>\n",
       "      <td>-1.237887</td>\n",
       "      <td>-0.529302</td>\n",
       "      <td>-1.183956</td>\n",
       "      <td>-0.093750</td>\n",
       "      <td>-0.282762</td>\n",
       "      <td>-0.259116</td>\n",
       "      <td>-0.291901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4251</th>\n",
       "      <td>-0.560822</td>\n",
       "      <td>2.733369</td>\n",
       "      <td>2.795886</td>\n",
       "      <td>2.319042</td>\n",
       "      <td>2.394432</td>\n",
       "      <td>1.108077</td>\n",
       "      <td>1.835282</td>\n",
       "      <td>2.232961</td>\n",
       "      <td>0.958134</td>\n",
       "      <td>2.694594</td>\n",
       "      <td>2.319728</td>\n",
       "      <td>2.279055</td>\n",
       "      <td>2.207089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4252</th>\n",
       "      <td>-0.253521</td>\n",
       "      <td>2.740388</td>\n",
       "      <td>2.819063</td>\n",
       "      <td>2.334341</td>\n",
       "      <td>2.391258</td>\n",
       "      <td>1.116279</td>\n",
       "      <td>1.840214</td>\n",
       "      <td>2.236842</td>\n",
       "      <td>0.882031</td>\n",
       "      <td>1.867211</td>\n",
       "      <td>2.338756</td>\n",
       "      <td>2.284824</td>\n",
       "      <td>2.218398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4253</th>\n",
       "      <td>-0.304855</td>\n",
       "      <td>2.825701</td>\n",
       "      <td>2.855721</td>\n",
       "      <td>2.355450</td>\n",
       "      <td>2.388831</td>\n",
       "      <td>1.500313</td>\n",
       "      <td>2.090202</td>\n",
       "      <td>2.437998</td>\n",
       "      <td>1.092081</td>\n",
       "      <td>2.077695</td>\n",
       "      <td>2.352661</td>\n",
       "      <td>2.292429</td>\n",
       "      <td>2.230729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4254</th>\n",
       "      <td>-0.196444</td>\n",
       "      <td>2.655015</td>\n",
       "      <td>2.848802</td>\n",
       "      <td>2.359830</td>\n",
       "      <td>2.397907</td>\n",
       "      <td>-0.161575</td>\n",
       "      <td>0.917907</td>\n",
       "      <td>1.614418</td>\n",
       "      <td>0.775609</td>\n",
       "      <td>1.143172</td>\n",
       "      <td>2.359613</td>\n",
       "      <td>2.301633</td>\n",
       "      <td>2.241658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4255</th>\n",
       "      <td>-0.507921</td>\n",
       "      <td>2.545616</td>\n",
       "      <td>2.820034</td>\n",
       "      <td>2.366021</td>\n",
       "      <td>2.406284</td>\n",
       "      <td>0.152025</td>\n",
       "      <td>1.077484</td>\n",
       "      <td>1.725621</td>\n",
       "      <td>0.670383</td>\n",
       "      <td>0.666556</td>\n",
       "      <td>2.353320</td>\n",
       "      <td>2.307402</td>\n",
       "      <td>2.251207</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4256 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        volume      macd     macds   boll_ub   boll_lb     rsi_5    rsi_14  \\\n",
       "0    -0.401020  0.489463  0.617954 -0.353622 -0.428058 -0.481647  0.307562   \n",
       "1    -0.870514  0.426991  0.585422 -0.353426 -0.421663 -0.402383  0.333753   \n",
       "2    -0.819901  0.336381  0.540152 -0.358686 -0.407370 -0.845417  0.091875   \n",
       "3    -0.507045  0.179425  0.470598 -0.372529 -0.384293 -1.506754 -0.412795   \n",
       "4     0.618350 -0.118873  0.351596 -0.373050 -0.383547 -2.092038 -1.237887   \n",
       "...        ...       ...       ...       ...       ...       ...       ...   \n",
       "4251 -0.560822  2.733369  2.795886  2.319042  2.394432  1.108077  1.835282   \n",
       "4252 -0.253521  2.740388  2.819063  2.334341  2.391258  1.116279  1.840214   \n",
       "4253 -0.304855  2.825701  2.855721  2.355450  2.388831  1.500313  2.090202   \n",
       "4254 -0.196444  2.655015  2.848802  2.359830  2.397907 -0.161575  0.917907   \n",
       "4255 -0.507921  2.545616  2.820034  2.366021  2.406284  0.152025  1.077484   \n",
       "\n",
       "        rsi_30    cci_30     dx_30  open_5_sma  open_14_sma  open_30_sma  \n",
       "0     0.849076  0.113132 -0.236348   -0.253884    -0.256690    -0.315423  \n",
       "1     0.868129  0.234302 -0.201806   -0.258847    -0.256044    -0.307076  \n",
       "2     0.672550  0.001486 -0.887515   -0.264713    -0.256690    -0.300703  \n",
       "3     0.248515 -0.278378 -1.161282   -0.268774    -0.256852    -0.296529  \n",
       "4    -0.529302 -1.183956 -0.093750   -0.282762    -0.259116    -0.291901  \n",
       "...        ...       ...       ...         ...          ...          ...  \n",
       "4251  2.232961  0.958134  2.694594    2.319728     2.279055     2.207089  \n",
       "4252  2.236842  0.882031  1.867211    2.338756     2.284824     2.218398  \n",
       "4253  2.437998  1.092081  2.077695    2.352661     2.292429     2.230729  \n",
       "4254  1.614418  0.775609  1.143172    2.359613     2.301633     2.241658  \n",
       "4255  1.725621  0.670383  0.666556    2.353320     2.307402     2.251207  \n",
       "\n",
       "[4256 rows x 13 columns]"
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#This is an (example) of what self.features looks like for one ticker\n",
    "test_env.features['DIS']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a2f0c58",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "id": "6734d028",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Use stable baselines for reinforcement learning\n",
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "from stable_baselines3.common import results_plotter\n",
    "from stable_baselines3.common.monitor import Monitor\n",
    "from stable_baselines3.common.results_plotter import load_results, ts2xy, plot_results\n",
    "from stable_baselines3.common.callbacks import BaseCallback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "id": "76c2f4fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SaveRewardCallback(BaseCallback):\n",
    "    \"\"\"\n",
    "    Callback for saving a model (the check is done every ``check_freq`` steps)\n",
    "    based on the training reward (in practice, we recommend using ``EvalCallback``).\n",
    "\n",
    "    :param check_freq: (int)\n",
    "    :param log_dir: (str) Path to the folder where the model will be saved.\n",
    "      It must contains the file created by the ``Monitor`` wrapper.\n",
    "    :param verbose: (int)\n",
    "    \"\"\"\n",
    "    def __init__(self, check_freq: int, log_dir: str, verbose=1):\n",
    "        super(SaveRewardCallback, self).__init__(verbose)\n",
    "        self.check_freq = check_freq\n",
    "        self.log_dir = log_dir\n",
    "        self.save_path = os.path.join(log_dir, 'best_model')\n",
    "        self.best_mean_reward = -np.inf\n",
    "\n",
    "    def _init_callback(self) -> None:\n",
    "        # Create folder if needed\n",
    "        if self.save_path is not None:\n",
    "            os.makedirs(self.save_path, exist_ok=True)\n",
    "\n",
    "    def _on_step(self) -> bool:\n",
    "\n",
    "        if self.n_calls % self.check_freq == 0:\n",
    "\n",
    "          # Retrieve training reward\n",
    "          x, y = ts2xy(load_results(self.log_dir), 'timesteps')\n",
    "          if len(x) > 0:\n",
    "              # Mean training reward over the last 100 episodes\n",
    "              mean_reward = np.mean(y[-100:])\n",
    "              if self.verbose > 0:\n",
    "                print(\"Num timesteps: {}\".format(self.num_timesteps))\n",
    "                print(\"Best mean reward: {:.2f} - Last mean reward per episode: {:.2f}\".format(self.best_mean_reward, mean_reward))\n",
    "\n",
    "              # New best model, you could save the agent here\n",
    "              if mean_reward > self.best_mean_reward:\n",
    "                  self.best_mean_reward = mean_reward\n",
    "                  # Example for saving best model\n",
    "                  if self.verbose > 0:\n",
    "                    print(\"Saving new best model to {}\".format(self.save_path))\n",
    "                  self.model.save(self.save_path)\n",
    "\n",
    "        return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "id": "304a82e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Make diretory for saving training statistics and best model\n",
    "log_dir = \"rl_with_ta/\"\n",
    "os.makedirs(log_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "id": "95f10750",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Make wrapped environment\n",
    "trade_env = TradeEnv(tickers=tickers)\n",
    "env = Monitor(trade_env, log_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "id": "5b95f494",
   "metadata": {},
   "outputs": [],
   "source": [
    "#make callback\n",
    "callback = SaveRewardCallback(check_freq=10000, log_dir=log_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65a5b711",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "id": "996fbe7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch as th\n",
    "#neural network architecture and training parameters\n",
    "policy_kwargs = dict(activation_fn=th.nn.Tanh,\n",
    "                     net_arch=[dict(vf=[256, 128], pi=[256, 150])])\n",
    "train_kwargs = dict(batch_size=512,\n",
    "                   n_epochs = 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "id": "ff089ffd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cpu device\n",
      "Wrapping the env in a DummyVecEnv.\n"
     ]
    }
   ],
   "source": [
    "#Make model\n",
    "model = PPO('MlpPolicy', env, gamma = .9998, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "id": "66933ff9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 1120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 160.77\n",
      "Num timesteps: 11120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 141.66\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 146       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 421       |\n",
      "|    iterations           | 10        |\n",
      "|    time_elapsed         | 48        |\n",
      "|    total_timesteps      | 20480     |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 1.2885337 |\n",
      "|    clip_fraction        | 0.698     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | 0.972     |\n",
      "|    explained_variance   | 0.752     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 154       |\n",
      "|    n_updates            | 33520     |\n",
      "|    policy_gradient_loss | 0.0758    |\n",
      "|    std                  | 0.178     |\n",
      "|    value_loss           | 484       |\n",
      "---------------------------------------\n",
      "Num timesteps: 21120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 139.95\n",
      "Num timesteps: 31120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 118.03\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 144        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 428        |\n",
      "|    iterations           | 20         |\n",
      "|    time_elapsed         | 95         |\n",
      "|    total_timesteps      | 40960      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.48465285 |\n",
      "|    clip_fraction        | 0.7        |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | 0.324      |\n",
      "|    explained_variance   | 0.798      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 76.1       |\n",
      "|    n_updates            | 33620      |\n",
      "|    policy_gradient_loss | 0.0574     |\n",
      "|    std                  | 0.22       |\n",
      "|    value_loss           | 298        |\n",
      "----------------------------------------\n",
      "Num timesteps: 41120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 134.29\n",
      "Num timesteps: 51120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 142.16\n",
      "Num timesteps: 61120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 110.41\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 107        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 432        |\n",
      "|    iterations           | 30         |\n",
      "|    time_elapsed         | 142        |\n",
      "|    total_timesteps      | 61440      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.18945208 |\n",
      "|    clip_fraction        | 0.613      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.0616    |\n",
      "|    explained_variance   | 0.821      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 44.9       |\n",
      "|    n_updates            | 33720      |\n",
      "|    policy_gradient_loss | -0.0106    |\n",
      "|    std                  | 0.249      |\n",
      "|    value_loss           | 266        |\n",
      "----------------------------------------\n",
      "Num timesteps: 71120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.64\n",
      "Num timesteps: 81120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 114.66\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 112        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 434        |\n",
      "|    iterations           | 40         |\n",
      "|    time_elapsed         | 188        |\n",
      "|    total_timesteps      | 81920      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.16079144 |\n",
      "|    clip_fraction        | 0.547      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.454     |\n",
      "|    explained_variance   | 0.722      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 54.3       |\n",
      "|    n_updates            | 33820      |\n",
      "|    policy_gradient_loss | -0.0132    |\n",
      "|    std                  | 0.285      |\n",
      "|    value_loss           | 253        |\n",
      "----------------------------------------\n",
      "Num timesteps: 91120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 108.85\n",
      "Num timesteps: 101120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 127.62\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 126        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 436        |\n",
      "|    iterations           | 50         |\n",
      "|    time_elapsed         | 234        |\n",
      "|    total_timesteps      | 102400     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.14318909 |\n",
      "|    clip_fraction        | 0.586      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.681     |\n",
      "|    explained_variance   | 0.868      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 64         |\n",
      "|    n_updates            | 33920      |\n",
      "|    policy_gradient_loss | 0.000623   |\n",
      "|    std                  | 0.305      |\n",
      "|    value_loss           | 198        |\n",
      "----------------------------------------\n",
      "Num timesteps: 111120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 122.73\n",
      "Num timesteps: 121120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.06\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 134        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 437        |\n",
      "|    iterations           | 60         |\n",
      "|    time_elapsed         | 281        |\n",
      "|    total_timesteps      | 122880     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.11129449 |\n",
      "|    clip_fraction        | 0.539      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.662     |\n",
      "|    explained_variance   | 0.776      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 179        |\n",
      "|    n_updates            | 34020      |\n",
      "|    policy_gradient_loss | -0.0058    |\n",
      "|    std                  | 0.304      |\n",
      "|    value_loss           | 291        |\n",
      "----------------------------------------\n",
      "Num timesteps: 131120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 141.40\n",
      "Num timesteps: 141120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 140.70\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 155        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 438        |\n",
      "|    iterations           | 70         |\n",
      "|    time_elapsed         | 327        |\n",
      "|    total_timesteps      | 143360     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.15903899 |\n",
      "|    clip_fraction        | 0.558      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.76      |\n",
      "|    explained_variance   | 0.838      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 102        |\n",
      "|    n_updates            | 34120      |\n",
      "|    policy_gradient_loss | -0.00276   |\n",
      "|    std                  | 0.315      |\n",
      "|    value_loss           | 279        |\n",
      "----------------------------------------\n",
      "Num timesteps: 151120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.37\n",
      "Num timesteps: 161120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 127.78\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 126        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 438        |\n",
      "|    iterations           | 80         |\n",
      "|    time_elapsed         | 373        |\n",
      "|    total_timesteps      | 163840     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.20497662 |\n",
      "|    clip_fraction        | 0.582      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.894     |\n",
      "|    explained_variance   | 0.822      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 59.2       |\n",
      "|    n_updates            | 34220      |\n",
      "|    policy_gradient_loss | 0.000434   |\n",
      "|    std                  | 0.33       |\n",
      "|    value_loss           | 195        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 171120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 125.27\n",
      "Num timesteps: 181120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 120.63\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 116        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 439        |\n",
      "|    iterations           | 90         |\n",
      "|    time_elapsed         | 419        |\n",
      "|    total_timesteps      | 184320     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.36105424 |\n",
      "|    clip_fraction        | 0.606      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.89      |\n",
      "|    explained_variance   | 0.804      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 72.3       |\n",
      "|    n_updates            | 34320      |\n",
      "|    policy_gradient_loss | 0.00482    |\n",
      "|    std                  | 0.326      |\n",
      "|    value_loss           | 243        |\n",
      "----------------------------------------\n",
      "Num timesteps: 191120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 134.49\n",
      "Num timesteps: 201120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 122.41\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 30          |\n",
      "|    ep_rew_mean          | 136         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 439         |\n",
      "|    iterations           | 100         |\n",
      "|    time_elapsed         | 466         |\n",
      "|    total_timesteps      | 204800      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.117044926 |\n",
      "|    clip_fraction        | 0.507       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.973      |\n",
      "|    explained_variance   | 0.796       |\n",
      "|    learning_rate        | 0.001       |\n",
      "|    loss                 | 103         |\n",
      "|    n_updates            | 34420       |\n",
      "|    policy_gradient_loss | -0.00669    |\n",
      "|    std                  | 0.336       |\n",
      "|    value_loss           | 294         |\n",
      "-----------------------------------------\n",
      "Num timesteps: 211120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 134.60\n",
      "Num timesteps: 221120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.35\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 131        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 439        |\n",
      "|    iterations           | 110        |\n",
      "|    time_elapsed         | 513        |\n",
      "|    total_timesteps      | 225280     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.18618296 |\n",
      "|    clip_fraction        | 0.568      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.03      |\n",
      "|    explained_variance   | 0.774      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 47         |\n",
      "|    n_updates            | 34520      |\n",
      "|    policy_gradient_loss | 0.00166    |\n",
      "|    std                  | 0.346      |\n",
      "|    value_loss           | 239        |\n",
      "----------------------------------------\n",
      "Num timesteps: 231120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 123.47\n",
      "Num timesteps: 241120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.83\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 137        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 437        |\n",
      "|    iterations           | 120        |\n",
      "|    time_elapsed         | 561        |\n",
      "|    total_timesteps      | 245760     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.22042665 |\n",
      "|    clip_fraction        | 0.485      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.03      |\n",
      "|    explained_variance   | 0.802      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 171        |\n",
      "|    n_updates            | 34620      |\n",
      "|    policy_gradient_loss | -0.0129    |\n",
      "|    std                  | 0.346      |\n",
      "|    value_loss           | 311        |\n",
      "----------------------------------------\n",
      "Num timesteps: 251120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 127.56\n",
      "Num timesteps: 261120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 145.15\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 146        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 437        |\n",
      "|    iterations           | 130        |\n",
      "|    time_elapsed         | 608        |\n",
      "|    total_timesteps      | 266240     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.08286392 |\n",
      "|    clip_fraction        | 0.396      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.03      |\n",
      "|    explained_variance   | 0.743      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 263        |\n",
      "|    n_updates            | 34720      |\n",
      "|    policy_gradient_loss | -0.0231    |\n",
      "|    std                  | 0.344      |\n",
      "|    value_loss           | 677        |\n",
      "----------------------------------------\n",
      "Num timesteps: 271120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 148.32\n",
      "Num timesteps: 281120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 133.42\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 129        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 437        |\n",
      "|    iterations           | 140        |\n",
      "|    time_elapsed         | 655        |\n",
      "|    total_timesteps      | 286720     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.26757672 |\n",
      "|    clip_fraction        | 0.567      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.892     |\n",
      "|    explained_variance   | 0.789      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 72.4       |\n",
      "|    n_updates            | 34820      |\n",
      "|    policy_gradient_loss | -0.00538   |\n",
      "|    std                  | 0.327      |\n",
      "|    value_loss           | 256        |\n",
      "----------------------------------------\n",
      "Num timesteps: 291120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.32\n",
      "Num timesteps: 301120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 153.62\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 135        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 437        |\n",
      "|    iterations           | 150        |\n",
      "|    time_elapsed         | 701        |\n",
      "|    total_timesteps      | 307200     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.15354022 |\n",
      "|    clip_fraction        | 0.517      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.972     |\n",
      "|    explained_variance   | 0.866      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 68.5       |\n",
      "|    n_updates            | 34920      |\n",
      "|    policy_gradient_loss | -0.0103    |\n",
      "|    std                  | 0.336      |\n",
      "|    value_loss           | 256        |\n",
      "----------------------------------------\n",
      "Num timesteps: 311120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.50\n",
      "Num timesteps: 321120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 148.44\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 154        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 437        |\n",
      "|    iterations           | 160        |\n",
      "|    time_elapsed         | 748        |\n",
      "|    total_timesteps      | 327680     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.20333815 |\n",
      "|    clip_fraction        | 0.527      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.916     |\n",
      "|    explained_variance   | 0.739      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 44.1       |\n",
      "|    n_updates            | 35020      |\n",
      "|    policy_gradient_loss | -0.018     |\n",
      "|    std                  | 0.328      |\n",
      "|    value_loss           | 248        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 331120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 159.64\n",
      "Num timesteps: 341120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 162.46\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 153        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 437        |\n",
      "|    iterations           | 170        |\n",
      "|    time_elapsed         | 795        |\n",
      "|    total_timesteps      | 348160     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.10335225 |\n",
      "|    clip_fraction        | 0.493      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.966     |\n",
      "|    explained_variance   | 0.802      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 69.1       |\n",
      "|    n_updates            | 35120      |\n",
      "|    policy_gradient_loss | -0.0184    |\n",
      "|    std                  | 0.334      |\n",
      "|    value_loss           | 307        |\n",
      "----------------------------------------\n",
      "Num timesteps: 351120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 147.78\n",
      "Num timesteps: 361120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 151.76\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 152        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 437        |\n",
      "|    iterations           | 180        |\n",
      "|    time_elapsed         | 841        |\n",
      "|    total_timesteps      | 368640     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.81249446 |\n",
      "|    clip_fraction        | 0.623      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.05      |\n",
      "|    explained_variance   | 0.712      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 109        |\n",
      "|    n_updates            | 35220      |\n",
      "|    policy_gradient_loss | 0.0119     |\n",
      "|    std                  | 0.344      |\n",
      "|    value_loss           | 403        |\n",
      "----------------------------------------\n",
      "Num timesteps: 371120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 145.58\n",
      "Num timesteps: 381120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 161.08\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 141        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 437        |\n",
      "|    iterations           | 190        |\n",
      "|    time_elapsed         | 888        |\n",
      "|    total_timesteps      | 389120     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.15181027 |\n",
      "|    clip_fraction        | 0.496      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.01      |\n",
      "|    explained_variance   | 0.836      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 90         |\n",
      "|    n_updates            | 35320      |\n",
      "|    policy_gradient_loss | -0.0259    |\n",
      "|    std                  | 0.338      |\n",
      "|    value_loss           | 346        |\n",
      "----------------------------------------\n",
      "Num timesteps: 391120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 146.63\n",
      "Num timesteps: 401120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 153.79\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 145        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 438        |\n",
      "|    iterations           | 200        |\n",
      "|    time_elapsed         | 935        |\n",
      "|    total_timesteps      | 409600     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.19246098 |\n",
      "|    clip_fraction        | 0.58       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.974     |\n",
      "|    explained_variance   | 0.754      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 29.1       |\n",
      "|    n_updates            | 35420      |\n",
      "|    policy_gradient_loss | -0.0011    |\n",
      "|    std                  | 0.337      |\n",
      "|    value_loss           | 258        |\n",
      "----------------------------------------\n",
      "Num timesteps: 411120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 139.30\n",
      "Num timesteps: 421120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 151.89\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 151        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 438        |\n",
      "|    iterations           | 210        |\n",
      "|    time_elapsed         | 981        |\n",
      "|    total_timesteps      | 430080     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.15335429 |\n",
      "|    clip_fraction        | 0.527      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.01      |\n",
      "|    explained_variance   | 0.798      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 78.2       |\n",
      "|    n_updates            | 35520      |\n",
      "|    policy_gradient_loss | -0.0135    |\n",
      "|    std                  | 0.338      |\n",
      "|    value_loss           | 289        |\n",
      "----------------------------------------\n",
      "Num timesteps: 431120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 147.43\n",
      "Num timesteps: 441120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 159.53\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 163        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 438        |\n",
      "|    iterations           | 220        |\n",
      "|    time_elapsed         | 1027       |\n",
      "|    total_timesteps      | 450560     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.24871954 |\n",
      "|    clip_fraction        | 0.619      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.998     |\n",
      "|    explained_variance   | 0.855      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 25.5       |\n",
      "|    n_updates            | 35620      |\n",
      "|    policy_gradient_loss | 0.00136    |\n",
      "|    std                  | 0.334      |\n",
      "|    value_loss           | 145        |\n",
      "----------------------------------------\n",
      "Num timesteps: 451120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 162.78\n",
      "Num timesteps: 461120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.74\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 157        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 438        |\n",
      "|    iterations           | 230        |\n",
      "|    time_elapsed         | 1073       |\n",
      "|    total_timesteps      | 471040     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.38798112 |\n",
      "|    clip_fraction        | 0.583      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.02      |\n",
      "|    explained_variance   | 0.848      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 29.3       |\n",
      "|    n_updates            | 35720      |\n",
      "|    policy_gradient_loss | -0.00821   |\n",
      "|    std                  | 0.34       |\n",
      "|    value_loss           | 156        |\n",
      "----------------------------------------\n",
      "Num timesteps: 471120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 153.60\n",
      "Num timesteps: 481120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.86\n",
      "Num timesteps: 491120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 141.29\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 30          |\n",
      "|    ep_rew_mean          | 144         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 438         |\n",
      "|    iterations           | 240         |\n",
      "|    time_elapsed         | 1120        |\n",
      "|    total_timesteps      | 491520      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.069095515 |\n",
      "|    clip_fraction        | 0.449       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.973      |\n",
      "|    explained_variance   | 0.822       |\n",
      "|    learning_rate        | 0.001       |\n",
      "|    loss                 | 76.9        |\n",
      "|    n_updates            | 35820       |\n",
      "|    policy_gradient_loss | -0.0287     |\n",
      "|    std                  | 0.336       |\n",
      "|    value_loss           | 268         |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 501120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 147.09\n",
      "Num timesteps: 511120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 145.86\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 30          |\n",
      "|    ep_rew_mean          | 146         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 438         |\n",
      "|    iterations           | 250         |\n",
      "|    time_elapsed         | 1166        |\n",
      "|    total_timesteps      | 512000      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.105347104 |\n",
      "|    clip_fraction        | 0.496       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.941      |\n",
      "|    explained_variance   | 0.725       |\n",
      "|    learning_rate        | 0.001       |\n",
      "|    loss                 | 95.2        |\n",
      "|    n_updates            | 35920       |\n",
      "|    policy_gradient_loss | -0.0209     |\n",
      "|    std                  | 0.33        |\n",
      "|    value_loss           | 341         |\n",
      "-----------------------------------------\n",
      "Num timesteps: 521120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 144.06\n",
      "Num timesteps: 531120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.05\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 142        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 439        |\n",
      "|    iterations           | 260        |\n",
      "|    time_elapsed         | 1212       |\n",
      "|    total_timesteps      | 532480     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.18345892 |\n",
      "|    clip_fraction        | 0.549      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.991     |\n",
      "|    explained_variance   | 0.821      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 37.3       |\n",
      "|    n_updates            | 36020      |\n",
      "|    policy_gradient_loss | -0.0131    |\n",
      "|    std                  | 0.337      |\n",
      "|    value_loss           | 198        |\n",
      "----------------------------------------\n",
      "Num timesteps: 541120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 147.61\n",
      "Num timesteps: 551120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 148.37\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 141        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 439        |\n",
      "|    iterations           | 270        |\n",
      "|    time_elapsed         | 1258       |\n",
      "|    total_timesteps      | 552960     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.20126179 |\n",
      "|    clip_fraction        | 0.548      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.1       |\n",
      "|    explained_variance   | 0.795      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 161        |\n",
      "|    n_updates            | 36120      |\n",
      "|    policy_gradient_loss | -0.0105    |\n",
      "|    std                  | 0.347      |\n",
      "|    value_loss           | 282        |\n",
      "----------------------------------------\n",
      "Num timesteps: 561120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 145.97\n",
      "Num timesteps: 571120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 145.97\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 158        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 439        |\n",
      "|    iterations           | 280        |\n",
      "|    time_elapsed         | 1305       |\n",
      "|    total_timesteps      | 573440     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.15169385 |\n",
      "|    clip_fraction        | 0.529      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.1       |\n",
      "|    explained_variance   | 0.828      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 40.6       |\n",
      "|    n_updates            | 36220      |\n",
      "|    policy_gradient_loss | -0.0262    |\n",
      "|    std                  | 0.347      |\n",
      "|    value_loss           | 233        |\n",
      "----------------------------------------\n",
      "Num timesteps: 581120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 139.31\n",
      "Num timesteps: 591120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 144.16\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 151        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 439        |\n",
      "|    iterations           | 290        |\n",
      "|    time_elapsed         | 1352       |\n",
      "|    total_timesteps      | 593920     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.26532793 |\n",
      "|    clip_fraction        | 0.576      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1         |\n",
      "|    explained_variance   | 0.846      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 41.9       |\n",
      "|    n_updates            | 36320      |\n",
      "|    policy_gradient_loss | 0.00354    |\n",
      "|    std                  | 0.34       |\n",
      "|    value_loss           | 244        |\n",
      "----------------------------------------\n",
      "Num timesteps: 601120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 147.13\n",
      "Num timesteps: 611120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 146.86\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 138        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 438        |\n",
      "|    iterations           | 300        |\n",
      "|    time_elapsed         | 1401       |\n",
      "|    total_timesteps      | 614400     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.17487851 |\n",
      "|    clip_fraction        | 0.523      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.988     |\n",
      "|    explained_variance   | 0.768      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 61.3       |\n",
      "|    n_updates            | 36420      |\n",
      "|    policy_gradient_loss | -0.0181    |\n",
      "|    std                  | 0.339      |\n",
      "|    value_loss           | 296        |\n",
      "----------------------------------------\n",
      "Num timesteps: 621120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 159.72\n",
      "Num timesteps: 631120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 163.49\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 154       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 437       |\n",
      "|    iterations           | 310       |\n",
      "|    time_elapsed         | 1451      |\n",
      "|    total_timesteps      | 634880    |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.1217733 |\n",
      "|    clip_fraction        | 0.52      |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.13     |\n",
      "|    explained_variance   | 0.837     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 115       |\n",
      "|    n_updates            | 36520     |\n",
      "|    policy_gradient_loss | -0.0262   |\n",
      "|    std                  | 0.353     |\n",
      "|    value_loss           | 286       |\n",
      "---------------------------------------\n",
      "Num timesteps: 641120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.23\n",
      "Num timesteps: 651120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 157.86\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 167        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 436        |\n",
      "|    iterations           | 320        |\n",
      "|    time_elapsed         | 1499       |\n",
      "|    total_timesteps      | 655360     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.24486913 |\n",
      "|    clip_fraction        | 0.576      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.08      |\n",
      "|    explained_variance   | 0.789      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 49.5       |\n",
      "|    n_updates            | 36620      |\n",
      "|    policy_gradient_loss | -0.000284  |\n",
      "|    std                  | 0.348      |\n",
      "|    value_loss           | 230        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 661120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 146.36\n",
      "Num timesteps: 671120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 139.70\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 138       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 436       |\n",
      "|    iterations           | 330       |\n",
      "|    time_elapsed         | 1548      |\n",
      "|    total_timesteps      | 675840    |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.1384635 |\n",
      "|    clip_fraction        | 0.533     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.976    |\n",
      "|    explained_variance   | 0.784     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 56.8      |\n",
      "|    n_updates            | 36720     |\n",
      "|    policy_gradient_loss | -0.0176   |\n",
      "|    std                  | 0.336     |\n",
      "|    value_loss           | 258       |\n",
      "---------------------------------------\n",
      "Num timesteps: 681120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 151.18\n",
      "Num timesteps: 691120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 151.34\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 151        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 435        |\n",
      "|    iterations           | 340        |\n",
      "|    time_elapsed         | 1597       |\n",
      "|    total_timesteps      | 696320     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.20459923 |\n",
      "|    clip_fraction        | 0.517      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.15      |\n",
      "|    explained_variance   | 0.859      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 67         |\n",
      "|    n_updates            | 36820      |\n",
      "|    policy_gradient_loss | -0.0187    |\n",
      "|    std                  | 0.355      |\n",
      "|    value_loss           | 236        |\n",
      "----------------------------------------\n",
      "Num timesteps: 701120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 160.42\n",
      "Num timesteps: 711120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 145.75\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 147        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 434        |\n",
      "|    iterations           | 350        |\n",
      "|    time_elapsed         | 1648       |\n",
      "|    total_timesteps      | 716800     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.15361199 |\n",
      "|    clip_fraction        | 0.518      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.01      |\n",
      "|    explained_variance   | 0.747      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 79.1       |\n",
      "|    n_updates            | 36920      |\n",
      "|    policy_gradient_loss | -0.0285    |\n",
      "|    std                  | 0.339      |\n",
      "|    value_loss           | 354        |\n",
      "----------------------------------------\n",
      "Num timesteps: 721120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 142.83\n",
      "Num timesteps: 731120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 145.95\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 153        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 433        |\n",
      "|    iterations           | 360        |\n",
      "|    time_elapsed         | 1700       |\n",
      "|    total_timesteps      | 737280     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.23691368 |\n",
      "|    clip_fraction        | 0.552      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.03      |\n",
      "|    explained_variance   | 0.814      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 74.5       |\n",
      "|    n_updates            | 37020      |\n",
      "|    policy_gradient_loss | -0.00804   |\n",
      "|    std                  | 0.341      |\n",
      "|    value_loss           | 270        |\n",
      "----------------------------------------\n",
      "Num timesteps: 741120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 147.22\n",
      "Num timesteps: 751120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 155.07\n",
      "--------------------------------------\n",
      "| rollout/                |          |\n",
      "|    ep_len_mean          | 30       |\n",
      "|    ep_rew_mean          | 154      |\n",
      "| time/                   |          |\n",
      "|    fps                  | 432      |\n",
      "|    iterations           | 370      |\n",
      "|    time_elapsed         | 1752     |\n",
      "|    total_timesteps      | 757760   |\n",
      "| train/                  |          |\n",
      "|    approx_kl            | 0.178909 |\n",
      "|    clip_fraction        | 0.534    |\n",
      "|    clip_range           | 0.2      |\n",
      "|    entropy_loss         | -1.14    |\n",
      "|    explained_variance   | 0.753    |\n",
      "|    learning_rate        | 0.001    |\n",
      "|    loss                 | 45.7     |\n",
      "|    n_updates            | 37120    |\n",
      "|    policy_gradient_loss | -0.0133  |\n",
      "|    std                  | 0.358    |\n",
      "|    value_loss           | 231      |\n",
      "--------------------------------------\n",
      "Num timesteps: 761120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.60\n",
      "Num timesteps: 771120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 156.48\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 174        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 431        |\n",
      "|    iterations           | 380        |\n",
      "|    time_elapsed         | 1803       |\n",
      "|    total_timesteps      | 778240     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.10093893 |\n",
      "|    clip_fraction        | 0.44       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.19      |\n",
      "|    explained_variance   | 0.837      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 73.4       |\n",
      "|    n_updates            | 37220      |\n",
      "|    policy_gradient_loss | -0.0339    |\n",
      "|    std                  | 0.36       |\n",
      "|    value_loss           | 379        |\n",
      "----------------------------------------\n",
      "Num timesteps: 781120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 160.01\n",
      "Num timesteps: 791120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.79\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 165        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 430        |\n",
      "|    iterations           | 390        |\n",
      "|    time_elapsed         | 1857       |\n",
      "|    total_timesteps      | 798720     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.57554805 |\n",
      "|    clip_fraction        | 0.662      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.01      |\n",
      "|    explained_variance   | 0.786      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 48.2       |\n",
      "|    n_updates            | 37320      |\n",
      "|    policy_gradient_loss | 0.0258     |\n",
      "|    std                  | 0.339      |\n",
      "|    value_loss           | 205        |\n",
      "----------------------------------------\n",
      "Num timesteps: 801120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 148.98\n",
      "Num timesteps: 811120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 169.32\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 160        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 429        |\n",
      "|    iterations           | 400        |\n",
      "|    time_elapsed         | 1909       |\n",
      "|    total_timesteps      | 819200     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.13657404 |\n",
      "|    clip_fraction        | 0.541      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.92      |\n",
      "|    explained_variance   | 0.904      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 70.5       |\n",
      "|    n_updates            | 37420      |\n",
      "|    policy_gradient_loss | -0.026     |\n",
      "|    std                  | 0.329      |\n",
      "|    value_loss           | 215        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 821120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 153.01\n",
      "Num timesteps: 831120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.11\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 157        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 428        |\n",
      "|    iterations           | 410        |\n",
      "|    time_elapsed         | 1960       |\n",
      "|    total_timesteps      | 839680     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.21375106 |\n",
      "|    clip_fraction        | 0.554      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.942     |\n",
      "|    explained_variance   | 0.833      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 91.4       |\n",
      "|    n_updates            | 37520      |\n",
      "|    policy_gradient_loss | -0.0216    |\n",
      "|    std                  | 0.334      |\n",
      "|    value_loss           | 311        |\n",
      "----------------------------------------\n",
      "Num timesteps: 841120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 158.66\n",
      "Num timesteps: 851120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 140.19\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 152        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 427        |\n",
      "|    iterations           | 420        |\n",
      "|    time_elapsed         | 2011       |\n",
      "|    total_timesteps      | 860160     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.18743338 |\n",
      "|    clip_fraction        | 0.552      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.03      |\n",
      "|    explained_variance   | 0.863      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 66.6       |\n",
      "|    n_updates            | 37620      |\n",
      "|    policy_gradient_loss | -0.0132    |\n",
      "|    std                  | 0.344      |\n",
      "|    value_loss           | 210        |\n",
      "----------------------------------------\n",
      "Num timesteps: 861120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 153.73\n",
      "Num timesteps: 871120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 148.61\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 152        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 426        |\n",
      "|    iterations           | 430        |\n",
      "|    time_elapsed         | 2063       |\n",
      "|    total_timesteps      | 880640     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.21231228 |\n",
      "|    clip_fraction        | 0.573      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1         |\n",
      "|    explained_variance   | 0.827      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 77.9       |\n",
      "|    n_updates            | 37720      |\n",
      "|    policy_gradient_loss | 0.0045     |\n",
      "|    std                  | 0.341      |\n",
      "|    value_loss           | 272        |\n",
      "----------------------------------------\n",
      "Num timesteps: 881120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 147.42\n",
      "Num timesteps: 891120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 150.95\n",
      "Num timesteps: 901120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 155.45\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 155        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 425        |\n",
      "|    iterations           | 440        |\n",
      "|    time_elapsed         | 2116       |\n",
      "|    total_timesteps      | 901120     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.16730492 |\n",
      "|    clip_fraction        | 0.554      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.987     |\n",
      "|    explained_variance   | 0.836      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 58.2       |\n",
      "|    n_updates            | 37820      |\n",
      "|    policy_gradient_loss | -0.00617   |\n",
      "|    std                  | 0.336      |\n",
      "|    value_loss           | 220        |\n",
      "----------------------------------------\n",
      "Num timesteps: 911120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 162.81\n",
      "Num timesteps: 921120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 155.87\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 155        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 424        |\n",
      "|    iterations           | 450        |\n",
      "|    time_elapsed         | 2168       |\n",
      "|    total_timesteps      | 921600     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.15936683 |\n",
      "|    clip_fraction        | 0.504      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.997     |\n",
      "|    explained_variance   | 0.854      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 129        |\n",
      "|    n_updates            | 37920      |\n",
      "|    policy_gradient_loss | -0.0243    |\n",
      "|    std                  | 0.338      |\n",
      "|    value_loss           | 326        |\n",
      "----------------------------------------\n",
      "Num timesteps: 931120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 118.90\n",
      "Num timesteps: 941120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 140.68\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 123        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 423        |\n",
      "|    iterations           | 460        |\n",
      "|    time_elapsed         | 2223       |\n",
      "|    total_timesteps      | 942080     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.21751356 |\n",
      "|    clip_fraction        | 0.574      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.04      |\n",
      "|    explained_variance   | 0.815      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 45         |\n",
      "|    n_updates            | 38020      |\n",
      "|    policy_gradient_loss | -0.0125    |\n",
      "|    std                  | 0.346      |\n",
      "|    value_loss           | 175        |\n",
      "----------------------------------------\n",
      "Num timesteps: 951120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 134.12\n",
      "Num timesteps: 961120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 149.19\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 161        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 422        |\n",
      "|    iterations           | 470        |\n",
      "|    time_elapsed         | 2277       |\n",
      "|    total_timesteps      | 962560     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.17792445 |\n",
      "|    clip_fraction        | 0.532      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.819     |\n",
      "|    explained_variance   | 0.706      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 153        |\n",
      "|    n_updates            | 38120      |\n",
      "|    policy_gradient_loss | -0.0131    |\n",
      "|    std                  | 0.318      |\n",
      "|    value_loss           | 437        |\n",
      "----------------------------------------\n",
      "Num timesteps: 971120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 153.92\n",
      "Num timesteps: 981120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.61\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 123        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 422        |\n",
      "|    iterations           | 480        |\n",
      "|    time_elapsed         | 2329       |\n",
      "|    total_timesteps      | 983040     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.19927657 |\n",
      "|    clip_fraction        | 0.565      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.86      |\n",
      "|    explained_variance   | 0.764      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 83.3       |\n",
      "|    n_updates            | 38220      |\n",
      "|    policy_gradient_loss | -0.0098    |\n",
      "|    std                  | 0.322      |\n",
      "|    value_loss           | 260        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 991120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.15\n",
      "Num timesteps: 1001120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 154.16\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 140        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 421        |\n",
      "|    iterations           | 490        |\n",
      "|    time_elapsed         | 2381       |\n",
      "|    total_timesteps      | 1003520    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.19285901 |\n",
      "|    clip_fraction        | 0.587      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.799     |\n",
      "|    explained_variance   | 0.846      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 54.3       |\n",
      "|    n_updates            | 38320      |\n",
      "|    policy_gradient_loss | -0.00607   |\n",
      "|    std                  | 0.316      |\n",
      "|    value_loss           | 205        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1011120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 146.30\n",
      "Num timesteps: 1021120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 138.22\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 160        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 420        |\n",
      "|    iterations           | 500        |\n",
      "|    time_elapsed         | 2433       |\n",
      "|    total_timesteps      | 1024000    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.13146815 |\n",
      "|    clip_fraction        | 0.566      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.773     |\n",
      "|    explained_variance   | 0.793      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 68.3       |\n",
      "|    n_updates            | 38420      |\n",
      "|    policy_gradient_loss | -0.015     |\n",
      "|    std                  | 0.314      |\n",
      "|    value_loss           | 291        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1031120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 155.02\n",
      "Num timesteps: 1041120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 138.65\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 157        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 420        |\n",
      "|    iterations           | 510        |\n",
      "|    time_elapsed         | 2485       |\n",
      "|    total_timesteps      | 1044480    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.18968487 |\n",
      "|    clip_fraction        | 0.546      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.735     |\n",
      "|    explained_variance   | 0.788      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 50.3       |\n",
      "|    n_updates            | 38520      |\n",
      "|    policy_gradient_loss | -0.0124    |\n",
      "|    std                  | 0.308      |\n",
      "|    value_loss           | 269        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1051120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 158.03\n",
      "Num timesteps: 1061120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 154.36\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 161        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 419        |\n",
      "|    iterations           | 520        |\n",
      "|    time_elapsed         | 2539       |\n",
      "|    total_timesteps      | 1064960    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.41400045 |\n",
      "|    clip_fraction        | 0.649      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.818     |\n",
      "|    explained_variance   | 0.78       |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 131        |\n",
      "|    n_updates            | 38620      |\n",
      "|    policy_gradient_loss | 0.00407    |\n",
      "|    std                  | 0.32       |\n",
      "|    value_loss           | 369        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1071120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 145.09\n",
      "Num timesteps: 1081120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 140.75\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 154        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 418        |\n",
      "|    iterations           | 530        |\n",
      "|    time_elapsed         | 2592       |\n",
      "|    total_timesteps      | 1085440    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.25211832 |\n",
      "|    clip_fraction        | 0.567      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.718     |\n",
      "|    explained_variance   | 0.666      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 47         |\n",
      "|    n_updates            | 38720      |\n",
      "|    policy_gradient_loss | -0.0145    |\n",
      "|    std                  | 0.306      |\n",
      "|    value_loss           | 263        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1091120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 153.23\n",
      "Num timesteps: 1101120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 152.08\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 166       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 417       |\n",
      "|    iterations           | 540       |\n",
      "|    time_elapsed         | 2646      |\n",
      "|    total_timesteps      | 1105920   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 7.7587833 |\n",
      "|    clip_fraction        | 0.817     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.705    |\n",
      "|    explained_variance   | 0.856     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 77.6      |\n",
      "|    n_updates            | 38820     |\n",
      "|    policy_gradient_loss | 0.124     |\n",
      "|    std                  | 0.308     |\n",
      "|    value_loss           | 249       |\n",
      "---------------------------------------\n",
      "Num timesteps: 1111120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 156.28\n",
      "Num timesteps: 1121120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 149.42\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 149       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 417       |\n",
      "|    iterations           | 550       |\n",
      "|    time_elapsed         | 2700      |\n",
      "|    total_timesteps      | 1126400   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.2089422 |\n",
      "|    clip_fraction        | 0.56      |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.685    |\n",
      "|    explained_variance   | 0.854     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 74.7      |\n",
      "|    n_updates            | 38920     |\n",
      "|    policy_gradient_loss | -0.0185   |\n",
      "|    std                  | 0.302     |\n",
      "|    value_loss           | 246       |\n",
      "---------------------------------------\n",
      "Num timesteps: 1131120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 145.36\n",
      "Num timesteps: 1141120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 151.00\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 143        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 416        |\n",
      "|    iterations           | 560        |\n",
      "|    time_elapsed         | 2753       |\n",
      "|    total_timesteps      | 1146880    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.27560866 |\n",
      "|    clip_fraction        | 0.621      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.771     |\n",
      "|    explained_variance   | 0.841      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 38.3       |\n",
      "|    n_updates            | 39020      |\n",
      "|    policy_gradient_loss | 0.00505    |\n",
      "|    std                  | 0.313      |\n",
      "|    value_loss           | 167        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 1151120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 158.78\n",
      "Num timesteps: 1161120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 137.65\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 145        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 415        |\n",
      "|    iterations           | 570        |\n",
      "|    time_elapsed         | 2807       |\n",
      "|    total_timesteps      | 1167360    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.24746326 |\n",
      "|    clip_fraction        | 0.577      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.849     |\n",
      "|    explained_variance   | 0.848      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 57.8       |\n",
      "|    n_updates            | 39120      |\n",
      "|    policy_gradient_loss | -0.0212    |\n",
      "|    std                  | 0.322      |\n",
      "|    value_loss           | 219        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1171120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 154.28\n",
      "Num timesteps: 1181120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 150.18\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 140        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 415        |\n",
      "|    iterations           | 580        |\n",
      "|    time_elapsed         | 2858       |\n",
      "|    total_timesteps      | 1187840    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.87459826 |\n",
      "|    clip_fraction        | 0.776      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.891     |\n",
      "|    explained_variance   | 0.739      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 67.7       |\n",
      "|    n_updates            | 39220      |\n",
      "|    policy_gradient_loss | 0.1        |\n",
      "|    std                  | 0.329      |\n",
      "|    value_loss           | 262        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1191120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 142.85\n",
      "Num timesteps: 1201120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 145.17\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 148        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 415        |\n",
      "|    iterations           | 590        |\n",
      "|    time_elapsed         | 2909       |\n",
      "|    total_timesteps      | 1208320    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.17522171 |\n",
      "|    clip_fraction        | 0.526      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.01      |\n",
      "|    explained_variance   | 0.772      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 39.1       |\n",
      "|    n_updates            | 39320      |\n",
      "|    policy_gradient_loss | -0.024     |\n",
      "|    std                  | 0.34       |\n",
      "|    value_loss           | 236        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1211120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.92\n",
      "Num timesteps: 1221120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 169.72\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 147        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 415        |\n",
      "|    iterations           | 600        |\n",
      "|    time_elapsed         | 2956       |\n",
      "|    total_timesteps      | 1228800    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.21338344 |\n",
      "|    clip_fraction        | 0.595      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1         |\n",
      "|    explained_variance   | 0.821      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 131        |\n",
      "|    n_updates            | 39420      |\n",
      "|    policy_gradient_loss | -0.0144    |\n",
      "|    std                  | 0.337      |\n",
      "|    value_loss           | 228        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1231120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 153.97\n",
      "Num timesteps: 1241120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 144.45\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 132        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 416        |\n",
      "|    iterations           | 610        |\n",
      "|    time_elapsed         | 3000       |\n",
      "|    total_timesteps      | 1249280    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.23903985 |\n",
      "|    clip_fraction        | 0.535      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.979     |\n",
      "|    explained_variance   | 0.808      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 72.1       |\n",
      "|    n_updates            | 39520      |\n",
      "|    policy_gradient_loss | -0.0223    |\n",
      "|    std                  | 0.336      |\n",
      "|    value_loss           | 214        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1251120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 131.83\n",
      "Num timesteps: 1261120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 152.90\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 134        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 416        |\n",
      "|    iterations           | 620        |\n",
      "|    time_elapsed         | 3045       |\n",
      "|    total_timesteps      | 1269760    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.20188819 |\n",
      "|    clip_fraction        | 0.557      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.962     |\n",
      "|    explained_variance   | 0.815      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 48.2       |\n",
      "|    n_updates            | 39620      |\n",
      "|    policy_gradient_loss | -0.0197    |\n",
      "|    std                  | 0.333      |\n",
      "|    value_loss           | 229        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1271120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.03\n",
      "Num timesteps: 1281120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 159.32\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 152       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 417       |\n",
      "|    iterations           | 630       |\n",
      "|    time_elapsed         | 3089      |\n",
      "|    total_timesteps      | 1290240   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.4511068 |\n",
      "|    clip_fraction        | 0.591     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.02     |\n",
      "|    explained_variance   | 0.801     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 71        |\n",
      "|    n_updates            | 39720     |\n",
      "|    policy_gradient_loss | -0.00961  |\n",
      "|    std                  | 0.34      |\n",
      "|    value_loss           | 262       |\n",
      "---------------------------------------\n",
      "Num timesteps: 1291120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 150.89\n",
      "Num timesteps: 1301120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 149.62\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 146        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 418        |\n",
      "|    iterations           | 640        |\n",
      "|    time_elapsed         | 3134       |\n",
      "|    total_timesteps      | 1310720    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.14306301 |\n",
      "|    clip_fraction        | 0.538      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.16      |\n",
      "|    explained_variance   | 0.764      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 110        |\n",
      "|    n_updates            | 39820      |\n",
      "|    policy_gradient_loss | -0.0295    |\n",
      "|    std                  | 0.357      |\n",
      "|    value_loss           | 264        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 1311120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.47\n",
      "Num timesteps: 1321120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 154.69\n",
      "Num timesteps: 1331120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 153.48\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 151        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 418        |\n",
      "|    iterations           | 650        |\n",
      "|    time_elapsed         | 3178       |\n",
      "|    total_timesteps      | 1331200    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.19494024 |\n",
      "|    clip_fraction        | 0.548      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.12      |\n",
      "|    explained_variance   | 0.738      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 63         |\n",
      "|    n_updates            | 39920      |\n",
      "|    policy_gradient_loss | -0.026     |\n",
      "|    std                  | 0.354      |\n",
      "|    value_loss           | 287        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1341120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 131.95\n",
      "Num timesteps: 1351120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 145.88\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 141        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 419        |\n",
      "|    iterations           | 660        |\n",
      "|    time_elapsed         | 3222       |\n",
      "|    total_timesteps      | 1351680    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.15945125 |\n",
      "|    clip_fraction        | 0.514      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.14      |\n",
      "|    explained_variance   | 0.811      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 97.7       |\n",
      "|    n_updates            | 40020      |\n",
      "|    policy_gradient_loss | -0.0293    |\n",
      "|    std                  | 0.354      |\n",
      "|    value_loss           | 248        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1361120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 153.94\n",
      "Num timesteps: 1371120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 158.90\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 147        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 420        |\n",
      "|    iterations           | 670        |\n",
      "|    time_elapsed         | 3266       |\n",
      "|    total_timesteps      | 1372160    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.11721493 |\n",
      "|    clip_fraction        | 0.535      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.14      |\n",
      "|    explained_variance   | 0.811      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 80.4       |\n",
      "|    n_updates            | 40120      |\n",
      "|    policy_gradient_loss | -0.0264    |\n",
      "|    std                  | 0.353      |\n",
      "|    value_loss           | 305        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1381120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 148.82\n",
      "Num timesteps: 1391120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 157.88\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 183        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 420        |\n",
      "|    iterations           | 680        |\n",
      "|    time_elapsed         | 3310       |\n",
      "|    total_timesteps      | 1392640    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.17368522 |\n",
      "|    clip_fraction        | 0.519      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.967     |\n",
      "|    explained_variance   | 0.862      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 116        |\n",
      "|    n_updates            | 40220      |\n",
      "|    policy_gradient_loss | -0.0269    |\n",
      "|    std                  | 0.332      |\n",
      "|    value_loss           | 275        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1401120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 149.70\n",
      "Num timesteps: 1411120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 147.87\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 155       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 421       |\n",
      "|    iterations           | 690       |\n",
      "|    time_elapsed         | 3354      |\n",
      "|    total_timesteps      | 1413120   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.1838078 |\n",
      "|    clip_fraction        | 0.571     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.922    |\n",
      "|    explained_variance   | 0.824     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 55.7      |\n",
      "|    n_updates            | 40320     |\n",
      "|    policy_gradient_loss | -0.0145   |\n",
      "|    std                  | 0.329     |\n",
      "|    value_loss           | 242       |\n",
      "---------------------------------------\n",
      "Num timesteps: 1421120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 153.72\n",
      "Num timesteps: 1431120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.22\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 138        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 421        |\n",
      "|    iterations           | 700        |\n",
      "|    time_elapsed         | 3399       |\n",
      "|    total_timesteps      | 1433600    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.14105415 |\n",
      "|    clip_fraction        | 0.547      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.798     |\n",
      "|    explained_variance   | 0.834      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 68.7       |\n",
      "|    n_updates            | 40420      |\n",
      "|    policy_gradient_loss | -0.0167    |\n",
      "|    std                  | 0.313      |\n",
      "|    value_loss           | 227        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1441120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 152.85\n",
      "Num timesteps: 1451120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 174.00\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 176        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 422        |\n",
      "|    iterations           | 710        |\n",
      "|    time_elapsed         | 3443       |\n",
      "|    total_timesteps      | 1454080    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.20308715 |\n",
      "|    clip_fraction        | 0.589      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.769     |\n",
      "|    explained_variance   | 0.896      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 70.7       |\n",
      "|    n_updates            | 40520      |\n",
      "|    policy_gradient_loss | -0.0153    |\n",
      "|    std                  | 0.313      |\n",
      "|    value_loss           | 263        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1461120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 148.87\n",
      "Num timesteps: 1471120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 149.84\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 165        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 422        |\n",
      "|    iterations           | 720        |\n",
      "|    time_elapsed         | 3487       |\n",
      "|    total_timesteps      | 1474560    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.18374579 |\n",
      "|    clip_fraction        | 0.588      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.843     |\n",
      "|    explained_variance   | 0.851      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 118        |\n",
      "|    n_updates            | 40620      |\n",
      "|    policy_gradient_loss | -0.0164    |\n",
      "|    std                  | 0.321      |\n",
      "|    value_loss           | 212        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 1481120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 166.98\n",
      "Num timesteps: 1491120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 166.19\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 159       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 423       |\n",
      "|    iterations           | 730       |\n",
      "|    time_elapsed         | 3532      |\n",
      "|    total_timesteps      | 1495040   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.2291115 |\n",
      "|    clip_fraction        | 0.59      |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.928    |\n",
      "|    explained_variance   | 0.854     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 75.7      |\n",
      "|    n_updates            | 40720     |\n",
      "|    policy_gradient_loss | -0.013    |\n",
      "|    std                  | 0.33      |\n",
      "|    value_loss           | 215       |\n",
      "---------------------------------------\n",
      "Num timesteps: 1501120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 147.63\n",
      "Num timesteps: 1511120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 164.48\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 145        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 423        |\n",
      "|    iterations           | 740        |\n",
      "|    time_elapsed         | 3576       |\n",
      "|    total_timesteps      | 1515520    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.20315358 |\n",
      "|    clip_fraction        | 0.577      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.853     |\n",
      "|    explained_variance   | 0.884      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 54.2       |\n",
      "|    n_updates            | 40820      |\n",
      "|    policy_gradient_loss | -0.0211    |\n",
      "|    std                  | 0.319      |\n",
      "|    value_loss           | 220        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1521120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 167.42\n",
      "Num timesteps: 1531120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 150.70\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 141       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 424       |\n",
      "|    iterations           | 750       |\n",
      "|    time_elapsed         | 3620      |\n",
      "|    total_timesteps      | 1536000   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.1277644 |\n",
      "|    clip_fraction        | 0.521     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.79     |\n",
      "|    explained_variance   | 0.855     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 61.1      |\n",
      "|    n_updates            | 40920     |\n",
      "|    policy_gradient_loss | -0.0254   |\n",
      "|    std                  | 0.313     |\n",
      "|    value_loss           | 266       |\n",
      "---------------------------------------\n",
      "Num timesteps: 1541120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 162.87\n",
      "Num timesteps: 1551120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 138.94\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 168        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 424        |\n",
      "|    iterations           | 760        |\n",
      "|    time_elapsed         | 3664       |\n",
      "|    total_timesteps      | 1556480    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.14216018 |\n",
      "|    clip_fraction        | 0.515      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.722     |\n",
      "|    explained_variance   | 0.846      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 142        |\n",
      "|    n_updates            | 41020      |\n",
      "|    policy_gradient_loss | -0.0173    |\n",
      "|    std                  | 0.308      |\n",
      "|    value_loss           | 305        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1561120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 161.20\n",
      "Num timesteps: 1571120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 139.46\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 134       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 425       |\n",
      "|    iterations           | 770       |\n",
      "|    time_elapsed         | 3708      |\n",
      "|    total_timesteps      | 1576960   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.6447177 |\n",
      "|    clip_fraction        | 0.675     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.774    |\n",
      "|    explained_variance   | 0.754     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 45.3      |\n",
      "|    n_updates            | 41120     |\n",
      "|    policy_gradient_loss | 0.00378   |\n",
      "|    std                  | 0.312     |\n",
      "|    value_loss           | 172       |\n",
      "---------------------------------------\n",
      "Num timesteps: 1581120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 150.97\n",
      "Num timesteps: 1591120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 137.12\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 164       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 425       |\n",
      "|    iterations           | 780       |\n",
      "|    time_elapsed         | 3753      |\n",
      "|    total_timesteps      | 1597440   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.6716298 |\n",
      "|    clip_fraction        | 0.683     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.799    |\n",
      "|    explained_variance   | 0.836     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 38.8      |\n",
      "|    n_updates            | 41220     |\n",
      "|    policy_gradient_loss | 0.0264    |\n",
      "|    std                  | 0.316     |\n",
      "|    value_loss           | 198       |\n",
      "---------------------------------------\n",
      "Num timesteps: 1601120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 151.27\n",
      "Num timesteps: 1611120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 156.71\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 148        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 426        |\n",
      "|    iterations           | 790        |\n",
      "|    time_elapsed         | 3797       |\n",
      "|    total_timesteps      | 1617920    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.24776855 |\n",
      "|    clip_fraction        | 0.565      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.841     |\n",
      "|    explained_variance   | 0.781      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 105        |\n",
      "|    n_updates            | 41320      |\n",
      "|    policy_gradient_loss | -0.0166    |\n",
      "|    std                  | 0.319      |\n",
      "|    value_loss           | 249        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1621120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 153.19\n",
      "Num timesteps: 1631120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 155.16\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 166        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 426        |\n",
      "|    iterations           | 800        |\n",
      "|    time_elapsed         | 3841       |\n",
      "|    total_timesteps      | 1638400    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.23154142 |\n",
      "|    clip_fraction        | 0.602      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.805     |\n",
      "|    explained_variance   | 0.883      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 63.4       |\n",
      "|    n_updates            | 41420      |\n",
      "|    policy_gradient_loss | -0.0158    |\n",
      "|    std                  | 0.317      |\n",
      "|    value_loss           | 197        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 1641120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 159.98\n",
      "Num timesteps: 1651120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.49\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 149        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 426        |\n",
      "|    iterations           | 810        |\n",
      "|    time_elapsed         | 3885       |\n",
      "|    total_timesteps      | 1658880    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.16006964 |\n",
      "|    clip_fraction        | 0.538      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.868     |\n",
      "|    explained_variance   | 0.864      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 108        |\n",
      "|    n_updates            | 41520      |\n",
      "|    policy_gradient_loss | -0.0234    |\n",
      "|    std                  | 0.325      |\n",
      "|    value_loss           | 344        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1661120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 147.55\n",
      "Num timesteps: 1671120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 157.57\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 144        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 427        |\n",
      "|    iterations           | 820        |\n",
      "|    time_elapsed         | 3930       |\n",
      "|    total_timesteps      | 1679360    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.15807043 |\n",
      "|    clip_fraction        | 0.542      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.01      |\n",
      "|    explained_variance   | 0.823      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 129        |\n",
      "|    n_updates            | 41620      |\n",
      "|    policy_gradient_loss | -0.0138    |\n",
      "|    std                  | 0.34       |\n",
      "|    value_loss           | 310        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1681120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 170.83\n",
      "Num timesteps: 1691120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 146.14\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 143        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 427        |\n",
      "|    iterations           | 830        |\n",
      "|    time_elapsed         | 3974       |\n",
      "|    total_timesteps      | 1699840    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.23444706 |\n",
      "|    clip_fraction        | 0.592      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.94      |\n",
      "|    explained_variance   | 0.847      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 63.7       |\n",
      "|    n_updates            | 41720      |\n",
      "|    policy_gradient_loss | -0.0158    |\n",
      "|    std                  | 0.331      |\n",
      "|    value_loss           | 203        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1701120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 162.20\n",
      "Num timesteps: 1711120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 155.34\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 136        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 428        |\n",
      "|    iterations           | 840        |\n",
      "|    time_elapsed         | 4018       |\n",
      "|    total_timesteps      | 1720320    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.40780264 |\n",
      "|    clip_fraction        | 0.65       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.987     |\n",
      "|    explained_variance   | 0.803      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 55.8       |\n",
      "|    n_updates            | 41820      |\n",
      "|    policy_gradient_loss | 0.0081     |\n",
      "|    std                  | 0.336      |\n",
      "|    value_loss           | 191        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1721120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 146.24\n",
      "Num timesteps: 1731120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 154.10\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 159        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 428        |\n",
      "|    iterations           | 850        |\n",
      "|    time_elapsed         | 4062       |\n",
      "|    total_timesteps      | 1740800    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.16919139 |\n",
      "|    clip_fraction        | 0.519      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.991     |\n",
      "|    explained_variance   | 0.841      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 90.2       |\n",
      "|    n_updates            | 41920      |\n",
      "|    policy_gradient_loss | -0.0311    |\n",
      "|    std                  | 0.336      |\n",
      "|    value_loss           | 302        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1741120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 153.76\n",
      "Num timesteps: 1751120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 155.74\n",
      "Num timesteps: 1761120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 139.84\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 138        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 428        |\n",
      "|    iterations           | 860        |\n",
      "|    time_elapsed         | 4106       |\n",
      "|    total_timesteps      | 1761280    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.30072477 |\n",
      "|    clip_fraction        | 0.634      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.909     |\n",
      "|    explained_variance   | 0.794      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 39.1       |\n",
      "|    n_updates            | 42020      |\n",
      "|    policy_gradient_loss | 0.0078     |\n",
      "|    std                  | 0.328      |\n",
      "|    value_loss           | 235        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1771120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 159.88\n",
      "Num timesteps: 1781120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 155.29\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 141        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 429        |\n",
      "|    iterations           | 870        |\n",
      "|    time_elapsed         | 4151       |\n",
      "|    total_timesteps      | 1781760    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.21621165 |\n",
      "|    clip_fraction        | 0.585      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.89      |\n",
      "|    explained_variance   | 0.829      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 48.2       |\n",
      "|    n_updates            | 42120      |\n",
      "|    policy_gradient_loss | -0.0167    |\n",
      "|    std                  | 0.326      |\n",
      "|    value_loss           | 218        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1791120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 169.21\n",
      "Num timesteps: 1801120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 153.21\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 148       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 429       |\n",
      "|    iterations           | 880       |\n",
      "|    time_elapsed         | 4195      |\n",
      "|    total_timesteps      | 1802240   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.2315535 |\n",
      "|    clip_fraction        | 0.548     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.03     |\n",
      "|    explained_variance   | 0.845     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 95.7      |\n",
      "|    n_updates            | 42220     |\n",
      "|    policy_gradient_loss | -0.0241   |\n",
      "|    std                  | 0.341     |\n",
      "|    value_loss           | 263       |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 1811120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 139.41\n",
      "Num timesteps: 1821120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 158.72\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 155        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 429        |\n",
      "|    iterations           | 890        |\n",
      "|    time_elapsed         | 4239       |\n",
      "|    total_timesteps      | 1822720    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.16283824 |\n",
      "|    clip_fraction        | 0.53       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.905     |\n",
      "|    explained_variance   | 0.77       |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 126        |\n",
      "|    n_updates            | 42320      |\n",
      "|    policy_gradient_loss | -0.0204    |\n",
      "|    std                  | 0.325      |\n",
      "|    value_loss           | 313        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1831120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 157.20\n",
      "Num timesteps: 1841120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 162.08\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 154       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 430       |\n",
      "|    iterations           | 900       |\n",
      "|    time_elapsed         | 4283      |\n",
      "|    total_timesteps      | 1843200   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.3152003 |\n",
      "|    clip_fraction        | 0.588     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.813    |\n",
      "|    explained_variance   | 0.764     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 41        |\n",
      "|    n_updates            | 42420     |\n",
      "|    policy_gradient_loss | -0.00919  |\n",
      "|    std                  | 0.317     |\n",
      "|    value_loss           | 218       |\n",
      "---------------------------------------\n",
      "Num timesteps: 1851120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 138.59\n",
      "Num timesteps: 1861120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 156.72\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 30          |\n",
      "|    ep_rew_mean          | 157         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 430         |\n",
      "|    iterations           | 910         |\n",
      "|    time_elapsed         | 4327        |\n",
      "|    total_timesteps      | 1863680     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.072157815 |\n",
      "|    clip_fraction        | 0.464       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.658      |\n",
      "|    explained_variance   | 0.826       |\n",
      "|    learning_rate        | 0.001       |\n",
      "|    loss                 | 159         |\n",
      "|    n_updates            | 42520       |\n",
      "|    policy_gradient_loss | -0.0289     |\n",
      "|    std                  | 0.301       |\n",
      "|    value_loss           | 595         |\n",
      "-----------------------------------------\n",
      "Num timesteps: 1871120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.73\n",
      "Num timesteps: 1881120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 148.74\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 148        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 430        |\n",
      "|    iterations           | 920        |\n",
      "|    time_elapsed         | 4371       |\n",
      "|    total_timesteps      | 1884160    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.28866065 |\n",
      "|    clip_fraction        | 0.631      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.586     |\n",
      "|    explained_variance   | 0.827      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 45.5       |\n",
      "|    n_updates            | 42620      |\n",
      "|    policy_gradient_loss | -0.00463   |\n",
      "|    std                  | 0.296      |\n",
      "|    value_loss           | 207        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1891120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.43\n",
      "Num timesteps: 1901120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 139.04\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 145        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 431        |\n",
      "|    iterations           | 930        |\n",
      "|    time_elapsed         | 4415       |\n",
      "|    total_timesteps      | 1904640    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.13484639 |\n",
      "|    clip_fraction        | 0.553      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.793     |\n",
      "|    explained_variance   | 0.816      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 42.6       |\n",
      "|    n_updates            | 42720      |\n",
      "|    policy_gradient_loss | -0.0205    |\n",
      "|    std                  | 0.317      |\n",
      "|    value_loss           | 257        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1911120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 140.39\n",
      "Num timesteps: 1921120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.55\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 134        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 431        |\n",
      "|    iterations           | 940        |\n",
      "|    time_elapsed         | 4459       |\n",
      "|    total_timesteps      | 1925120    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.23854981 |\n",
      "|    clip_fraction        | 0.614      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.824     |\n",
      "|    explained_variance   | 0.82       |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 46.8       |\n",
      "|    n_updates            | 42820      |\n",
      "|    policy_gradient_loss | -0.0158    |\n",
      "|    std                  | 0.318      |\n",
      "|    value_loss           | 181        |\n",
      "----------------------------------------\n",
      "Num timesteps: 1931120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 149.13\n",
      "Num timesteps: 1941120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 134.40\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 135       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 431       |\n",
      "|    iterations           | 950       |\n",
      "|    time_elapsed         | 4504      |\n",
      "|    total_timesteps      | 1945600   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 13.168222 |\n",
      "|    clip_fraction        | 0.707     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.98     |\n",
      "|    explained_variance   | 0.827     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 55.2      |\n",
      "|    n_updates            | 42920     |\n",
      "|    policy_gradient_loss | 0.162     |\n",
      "|    std                  | 0.345     |\n",
      "|    value_loss           | 199       |\n",
      "---------------------------------------\n",
      "Num timesteps: 1951120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 156.81\n",
      "Num timesteps: 1961120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 142.12\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 151        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 432        |\n",
      "|    iterations           | 960        |\n",
      "|    time_elapsed         | 4548       |\n",
      "|    total_timesteps      | 1966080    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.39340848 |\n",
      "|    clip_fraction        | 0.548      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.921     |\n",
      "|    explained_variance   | 0.76       |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 85.6       |\n",
      "|    n_updates            | 43020      |\n",
      "|    policy_gradient_loss | -0.0166    |\n",
      "|    std                  | 0.33       |\n",
      "|    value_loss           | 307        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 1971120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 161.79\n",
      "Num timesteps: 1981120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 158.02\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 143       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 432       |\n",
      "|    iterations           | 970       |\n",
      "|    time_elapsed         | 4592      |\n",
      "|    total_timesteps      | 1986560   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.3548528 |\n",
      "|    clip_fraction        | 0.595     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.871    |\n",
      "|    explained_variance   | 0.784     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 59.7      |\n",
      "|    n_updates            | 43120     |\n",
      "|    policy_gradient_loss | -0.00456  |\n",
      "|    std                  | 0.323     |\n",
      "|    value_loss           | 220       |\n",
      "---------------------------------------\n",
      "Num timesteps: 1991120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.58\n",
      "Num timesteps: 2001120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 144.79\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 129        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 432        |\n",
      "|    iterations           | 980        |\n",
      "|    time_elapsed         | 4636       |\n",
      "|    total_timesteps      | 2007040    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.19377716 |\n",
      "|    clip_fraction        | 0.585      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.893     |\n",
      "|    explained_variance   | 0.857      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 50.1       |\n",
      "|    n_updates            | 43220      |\n",
      "|    policy_gradient_loss | -0.0149    |\n",
      "|    std                  | 0.327      |\n",
      "|    value_loss           | 181        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2011120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 134.48\n",
      "Num timesteps: 2021120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 150.92\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 157        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 433        |\n",
      "|    iterations           | 990        |\n",
      "|    time_elapsed         | 4680       |\n",
      "|    total_timesteps      | 2027520    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.35077614 |\n",
      "|    clip_fraction        | 0.582      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.851     |\n",
      "|    explained_variance   | 0.824      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 83         |\n",
      "|    n_updates            | 43320      |\n",
      "|    policy_gradient_loss | 0.00213    |\n",
      "|    std                  | 0.324      |\n",
      "|    value_loss           | 299        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2031120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 125.46\n",
      "Num timesteps: 2041120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 150.09\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 152        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 433        |\n",
      "|    iterations           | 1000       |\n",
      "|    time_elapsed         | 4724       |\n",
      "|    total_timesteps      | 2048000    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.16340962 |\n",
      "|    clip_fraction        | 0.574      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.9       |\n",
      "|    explained_variance   | 0.756      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 48.1       |\n",
      "|    n_updates            | 43420      |\n",
      "|    policy_gradient_loss | -0.0229    |\n",
      "|    std                  | 0.327      |\n",
      "|    value_loss           | 215        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2051120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 144.26\n",
      "Num timesteps: 2061120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 144.74\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 146        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 433        |\n",
      "|    iterations           | 1010       |\n",
      "|    time_elapsed         | 4768       |\n",
      "|    total_timesteps      | 2068480    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.20957933 |\n",
      "|    clip_fraction        | 0.545      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.89      |\n",
      "|    explained_variance   | 0.821      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 37.1       |\n",
      "|    n_updates            | 43520      |\n",
      "|    policy_gradient_loss | -0.0176    |\n",
      "|    std                  | 0.323      |\n",
      "|    value_loss           | 229        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2071120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 150.59\n",
      "Num timesteps: 2081120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 142.07\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 176        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 434        |\n",
      "|    iterations           | 1020       |\n",
      "|    time_elapsed         | 4813       |\n",
      "|    total_timesteps      | 2088960    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.19395885 |\n",
      "|    clip_fraction        | 0.58       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.847     |\n",
      "|    explained_variance   | 0.877      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 38.9       |\n",
      "|    n_updates            | 43620      |\n",
      "|    policy_gradient_loss | -0.0079    |\n",
      "|    std                  | 0.321      |\n",
      "|    value_loss           | 195        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2091120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 154.05\n",
      "Num timesteps: 2101120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 144.90\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 169        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 434        |\n",
      "|    iterations           | 1030       |\n",
      "|    time_elapsed         | 4857       |\n",
      "|    total_timesteps      | 2109440    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.30852425 |\n",
      "|    clip_fraction        | 0.648      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.828     |\n",
      "|    explained_variance   | 0.853      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 73.6       |\n",
      "|    n_updates            | 43720      |\n",
      "|    policy_gradient_loss | 0.0193     |\n",
      "|    std                  | 0.322      |\n",
      "|    value_loss           | 331        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2111120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 159.59\n",
      "Num timesteps: 2121120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 138.03\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 149       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 434       |\n",
      "|    iterations           | 1040      |\n",
      "|    time_elapsed         | 4901      |\n",
      "|    total_timesteps      | 2129920   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.2137194 |\n",
      "|    clip_fraction        | 0.575     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.877    |\n",
      "|    explained_variance   | 0.759     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 41.8      |\n",
      "|    n_updates            | 43820     |\n",
      "|    policy_gradient_loss | -0.0335   |\n",
      "|    std                  | 0.323     |\n",
      "|    value_loss           | 209       |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 2131120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 146.18\n",
      "Num timesteps: 2141120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 134.38\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 138       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 434       |\n",
      "|    iterations           | 1050      |\n",
      "|    time_elapsed         | 4945      |\n",
      "|    total_timesteps      | 2150400   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.4673414 |\n",
      "|    clip_fraction        | 0.639     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.864    |\n",
      "|    explained_variance   | 0.802     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 40.7      |\n",
      "|    n_updates            | 43920     |\n",
      "|    policy_gradient_loss | -0.00725  |\n",
      "|    std                  | 0.324     |\n",
      "|    value_loss           | 156       |\n",
      "---------------------------------------\n",
      "Num timesteps: 2151120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.11\n",
      "Num timesteps: 2161120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 146.27\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 147        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 435        |\n",
      "|    iterations           | 1060       |\n",
      "|    time_elapsed         | 4989       |\n",
      "|    total_timesteps      | 2170880    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.16877073 |\n",
      "|    clip_fraction        | 0.574      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.817     |\n",
      "|    explained_variance   | 0.8        |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 99.4       |\n",
      "|    n_updates            | 44020      |\n",
      "|    policy_gradient_loss | -0.0223    |\n",
      "|    std                  | 0.318      |\n",
      "|    value_loss           | 349        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2171120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 144.02\n",
      "Num timesteps: 2181120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 155.80\n",
      "Num timesteps: 2191120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 161.26\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 156       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 435       |\n",
      "|    iterations           | 1070      |\n",
      "|    time_elapsed         | 5033      |\n",
      "|    total_timesteps      | 2191360   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.1840097 |\n",
      "|    clip_fraction        | 0.543     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.746    |\n",
      "|    explained_variance   | 0.796     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 129       |\n",
      "|    n_updates            | 44120     |\n",
      "|    policy_gradient_loss | -0.0246   |\n",
      "|    std                  | 0.31      |\n",
      "|    value_loss           | 334       |\n",
      "---------------------------------------\n",
      "Num timesteps: 2201120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.19\n",
      "Num timesteps: 2211120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.58\n",
      "--------------------------------------\n",
      "| rollout/                |          |\n",
      "|    ep_len_mean          | 30       |\n",
      "|    ep_rew_mean          | 144      |\n",
      "| time/                   |          |\n",
      "|    fps                  | 435      |\n",
      "|    iterations           | 1080     |\n",
      "|    time_elapsed         | 5077     |\n",
      "|    total_timesteps      | 2211840  |\n",
      "| train/                  |          |\n",
      "|    approx_kl            | 5.927268 |\n",
      "|    clip_fraction        | 0.864    |\n",
      "|    clip_range           | 0.2      |\n",
      "|    entropy_loss         | -0.835   |\n",
      "|    explained_variance   | 0.746    |\n",
      "|    learning_rate        | 0.001    |\n",
      "|    loss                 | 64.2     |\n",
      "|    n_updates            | 44220    |\n",
      "|    policy_gradient_loss | 0.166    |\n",
      "|    std                  | 0.324    |\n",
      "|    value_loss           | 243      |\n",
      "--------------------------------------\n",
      "Num timesteps: 2221120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 147.68\n",
      "Num timesteps: 2231120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 144.61\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 139       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 435       |\n",
      "|    iterations           | 1090      |\n",
      "|    time_elapsed         | 5121      |\n",
      "|    total_timesteps      | 2232320   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 1.0943868 |\n",
      "|    clip_fraction        | 0.726     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.77     |\n",
      "|    explained_variance   | 0.86      |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 105       |\n",
      "|    n_updates            | 44320     |\n",
      "|    policy_gradient_loss | 0.0213    |\n",
      "|    std                  | 0.312     |\n",
      "|    value_loss           | 189       |\n",
      "---------------------------------------\n",
      "Num timesteps: 2241120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 149.91\n",
      "Num timesteps: 2251120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 140.47\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 151        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 436        |\n",
      "|    iterations           | 1100       |\n",
      "|    time_elapsed         | 5165       |\n",
      "|    total_timesteps      | 2252800    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.49078032 |\n",
      "|    clip_fraction        | 0.622      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.794     |\n",
      "|    explained_variance   | 0.795      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 63.1       |\n",
      "|    n_updates            | 44420      |\n",
      "|    policy_gradient_loss | -0.014     |\n",
      "|    std                  | 0.316      |\n",
      "|    value_loss           | 209        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2261120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 131.56\n",
      "Num timesteps: 2271120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 138.71\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 154        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 436        |\n",
      "|    iterations           | 1110       |\n",
      "|    time_elapsed         | 5209       |\n",
      "|    total_timesteps      | 2273280    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.28341973 |\n",
      "|    clip_fraction        | 0.615      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.72      |\n",
      "|    explained_variance   | 0.659      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 42.1       |\n",
      "|    n_updates            | 44520      |\n",
      "|    policy_gradient_loss | -0.00913   |\n",
      "|    std                  | 0.309      |\n",
      "|    value_loss           | 239        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2281120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 120.92\n",
      "Num timesteps: 2291120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 142.03\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 133       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 436       |\n",
      "|    iterations           | 1120      |\n",
      "|    time_elapsed         | 5254      |\n",
      "|    total_timesteps      | 2293760   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.9662658 |\n",
      "|    clip_fraction        | 0.759     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.774    |\n",
      "|    explained_variance   | 0.805     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 62.7      |\n",
      "|    n_updates            | 44620     |\n",
      "|    policy_gradient_loss | 0.0752    |\n",
      "|    std                  | 0.318     |\n",
      "|    value_loss           | 190       |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 2301120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.33\n",
      "Num timesteps: 2311120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.31\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 154        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 436        |\n",
      "|    iterations           | 1130       |\n",
      "|    time_elapsed         | 5298       |\n",
      "|    total_timesteps      | 2314240    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.21381235 |\n",
      "|    clip_fraction        | 0.533      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.842     |\n",
      "|    explained_variance   | 0.751      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 45.2       |\n",
      "|    n_updates            | 44720      |\n",
      "|    policy_gradient_loss | -0.0192    |\n",
      "|    std                  | 0.32       |\n",
      "|    value_loss           | 235        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2321120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 150.13\n",
      "Num timesteps: 2331120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 115.82\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 123        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 437        |\n",
      "|    iterations           | 1140       |\n",
      "|    time_elapsed         | 5342       |\n",
      "|    total_timesteps      | 2334720    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.13266093 |\n",
      "|    clip_fraction        | 0.511      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.825     |\n",
      "|    explained_variance   | 0.795      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 54.6       |\n",
      "|    n_updates            | 44820      |\n",
      "|    policy_gradient_loss | -0.0349    |\n",
      "|    std                  | 0.317      |\n",
      "|    value_loss           | 251        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2341120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 153.26\n",
      "Num timesteps: 2351120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 121.18\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 129        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 437        |\n",
      "|    iterations           | 1150       |\n",
      "|    time_elapsed         | 5386       |\n",
      "|    total_timesteps      | 2355200    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.16484621 |\n",
      "|    clip_fraction        | 0.566      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.804     |\n",
      "|    explained_variance   | 0.808      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 94.5       |\n",
      "|    n_updates            | 44920      |\n",
      "|    policy_gradient_loss | -0.0255    |\n",
      "|    std                  | 0.315      |\n",
      "|    value_loss           | 336        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2361120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.58\n",
      "Num timesteps: 2371120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 125.10\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 122       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 437       |\n",
      "|    iterations           | 1160      |\n",
      "|    time_elapsed         | 5430      |\n",
      "|    total_timesteps      | 2375680   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.4157651 |\n",
      "|    clip_fraction        | 0.649     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.881    |\n",
      "|    explained_variance   | 0.814     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 67.2      |\n",
      "|    n_updates            | 45020     |\n",
      "|    policy_gradient_loss | -0.00505  |\n",
      "|    std                  | 0.325     |\n",
      "|    value_loss           | 213       |\n",
      "---------------------------------------\n",
      "Num timesteps: 2381120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.89\n",
      "Num timesteps: 2391120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 125.61\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 153        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 437        |\n",
      "|    iterations           | 1170       |\n",
      "|    time_elapsed         | 5474       |\n",
      "|    total_timesteps      | 2396160    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.22030947 |\n",
      "|    clip_fraction        | 0.581      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.771     |\n",
      "|    explained_variance   | 0.799      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 176        |\n",
      "|    n_updates            | 45120      |\n",
      "|    policy_gradient_loss | -0.0132    |\n",
      "|    std                  | 0.313      |\n",
      "|    value_loss           | 482        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2401120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.83\n",
      "Num timesteps: 2411120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.61\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 143        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 437        |\n",
      "|    iterations           | 1180       |\n",
      "|    time_elapsed         | 5519       |\n",
      "|    total_timesteps      | 2416640    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.17209673 |\n",
      "|    clip_fraction        | 0.563      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.955     |\n",
      "|    explained_variance   | 0.841      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 49.6       |\n",
      "|    n_updates            | 45220      |\n",
      "|    policy_gradient_loss | -0.0288    |\n",
      "|    std                  | 0.331      |\n",
      "|    value_loss           | 317        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2421120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 115.69\n",
      "Num timesteps: 2431120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 142.58\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 117        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 438        |\n",
      "|    iterations           | 1190       |\n",
      "|    time_elapsed         | 5563       |\n",
      "|    total_timesteps      | 2437120    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.25444126 |\n",
      "|    clip_fraction        | 0.635      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.87      |\n",
      "|    explained_variance   | 0.79       |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 77.8       |\n",
      "|    n_updates            | 45320      |\n",
      "|    policy_gradient_loss | -0.0237    |\n",
      "|    std                  | 0.32       |\n",
      "|    value_loss           | 179        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2441120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.74\n",
      "Num timesteps: 2451120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 116.42\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 126        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 438        |\n",
      "|    iterations           | 1200       |\n",
      "|    time_elapsed         | 5607       |\n",
      "|    total_timesteps      | 2457600    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.22539756 |\n",
      "|    clip_fraction        | 0.626      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.84      |\n",
      "|    explained_variance   | 0.793      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 109        |\n",
      "|    n_updates            | 45420      |\n",
      "|    policy_gradient_loss | -0.0152    |\n",
      "|    std                  | 0.319      |\n",
      "|    value_loss           | 255        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 2461120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.92\n",
      "Num timesteps: 2471120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 154.21\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 124       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 438       |\n",
      "|    iterations           | 1210      |\n",
      "|    time_elapsed         | 5651      |\n",
      "|    total_timesteps      | 2478080   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.5567544 |\n",
      "|    clip_fraction        | 0.616     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.781    |\n",
      "|    explained_variance   | 0.727     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 75.1      |\n",
      "|    n_updates            | 45520     |\n",
      "|    policy_gradient_loss | 0.00259   |\n",
      "|    std                  | 0.316     |\n",
      "|    value_loss           | 328       |\n",
      "---------------------------------------\n",
      "Num timesteps: 2481120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 121.68\n",
      "Num timesteps: 2491120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 115.70\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 140        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 438        |\n",
      "|    iterations           | 1220       |\n",
      "|    time_elapsed         | 5695       |\n",
      "|    total_timesteps      | 2498560    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.17403443 |\n",
      "|    clip_fraction        | 0.555      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.938     |\n",
      "|    explained_variance   | 0.764      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 46.7       |\n",
      "|    n_updates            | 45620      |\n",
      "|    policy_gradient_loss | -0.0273    |\n",
      "|    std                  | 0.33       |\n",
      "|    value_loss           | 269        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2501120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 134.61\n",
      "Num timesteps: 2511120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 125.66\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 152       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 438       |\n",
      "|    iterations           | 1230      |\n",
      "|    time_elapsed         | 5739      |\n",
      "|    total_timesteps      | 2519040   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.8631295 |\n",
      "|    clip_fraction        | 0.732     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.926    |\n",
      "|    explained_variance   | 0.86      |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 49.8      |\n",
      "|    n_updates            | 45720     |\n",
      "|    policy_gradient_loss | 0.0187    |\n",
      "|    std                  | 0.328     |\n",
      "|    value_loss           | 149       |\n",
      "---------------------------------------\n",
      "Num timesteps: 2521120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 150.98\n",
      "Num timesteps: 2531120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 134.95\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 140        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 439        |\n",
      "|    iterations           | 1240       |\n",
      "|    time_elapsed         | 5783       |\n",
      "|    total_timesteps      | 2539520    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.28110224 |\n",
      "|    clip_fraction        | 0.613      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.01      |\n",
      "|    explained_variance   | 0.835      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 54.3       |\n",
      "|    n_updates            | 45820      |\n",
      "|    policy_gradient_loss | -0.0169    |\n",
      "|    std                  | 0.34       |\n",
      "|    value_loss           | 209        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2541120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 96.18\n",
      "Num timesteps: 2551120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 129.85\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 127       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 439       |\n",
      "|    iterations           | 1250      |\n",
      "|    time_elapsed         | 5827      |\n",
      "|    total_timesteps      | 2560000   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.3988091 |\n",
      "|    clip_fraction        | 0.562     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.04     |\n",
      "|    explained_variance   | 0.817     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 55.7      |\n",
      "|    n_updates            | 45920     |\n",
      "|    policy_gradient_loss | -0.0193   |\n",
      "|    std                  | 0.339     |\n",
      "|    value_loss           | 218       |\n",
      "---------------------------------------\n",
      "Num timesteps: 2561120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.35\n",
      "Num timesteps: 2571120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 119.78\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 130        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 439        |\n",
      "|    iterations           | 1260       |\n",
      "|    time_elapsed         | 5872       |\n",
      "|    total_timesteps      | 2580480    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.33737117 |\n",
      "|    clip_fraction        | 0.605      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.98      |\n",
      "|    explained_variance   | 0.751      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 164        |\n",
      "|    n_updates            | 46020      |\n",
      "|    policy_gradient_loss | -0.029     |\n",
      "|    std                  | 0.335      |\n",
      "|    value_loss           | 265        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2581120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 141.72\n",
      "Num timesteps: 2591120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.76\n",
      "--------------------------------------\n",
      "| rollout/                |          |\n",
      "|    ep_len_mean          | 30       |\n",
      "|    ep_rew_mean          | 124      |\n",
      "| time/                   |          |\n",
      "|    fps                  | 439      |\n",
      "|    iterations           | 1270     |\n",
      "|    time_elapsed         | 5916     |\n",
      "|    total_timesteps      | 2600960  |\n",
      "| train/                  |          |\n",
      "|    approx_kl            | 0.369281 |\n",
      "|    clip_fraction        | 0.638    |\n",
      "|    clip_range           | 0.2      |\n",
      "|    entropy_loss         | -0.844   |\n",
      "|    explained_variance   | 0.82     |\n",
      "|    learning_rate        | 0.001    |\n",
      "|    loss                 | 67.7     |\n",
      "|    n_updates            | 46120    |\n",
      "|    policy_gradient_loss | -0.0154  |\n",
      "|    std                  | 0.318    |\n",
      "|    value_loss           | 246      |\n",
      "--------------------------------------\n",
      "Num timesteps: 2601120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 124.14\n",
      "Num timesteps: 2611120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 107.79\n",
      "Num timesteps: 2621120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 133.45\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 129        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 439        |\n",
      "|    iterations           | 1280       |\n",
      "|    time_elapsed         | 5960       |\n",
      "|    total_timesteps      | 2621440    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.37204075 |\n",
      "|    clip_fraction        | 0.62       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.952     |\n",
      "|    explained_variance   | 0.776      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 69.7       |\n",
      "|    n_updates            | 46220      |\n",
      "|    policy_gradient_loss | -0.00756   |\n",
      "|    std                  | 0.333      |\n",
      "|    value_loss           | 331        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 2631120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 112.62\n",
      "Num timesteps: 2641120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 127.87\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 119        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 439        |\n",
      "|    iterations           | 1290       |\n",
      "|    time_elapsed         | 6004       |\n",
      "|    total_timesteps      | 2641920    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.34078717 |\n",
      "|    clip_fraction        | 0.617      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.927     |\n",
      "|    explained_variance   | 0.758      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 61.7       |\n",
      "|    n_updates            | 46320      |\n",
      "|    policy_gradient_loss | -0.0167    |\n",
      "|    std                  | 0.328      |\n",
      "|    value_loss           | 297        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2651120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.64\n",
      "Num timesteps: 2661120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 117.71\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 117        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 440        |\n",
      "|    iterations           | 1300       |\n",
      "|    time_elapsed         | 6048       |\n",
      "|    total_timesteps      | 2662400    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.33521813 |\n",
      "|    clip_fraction        | 0.608      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.805     |\n",
      "|    explained_variance   | 0.808      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 77.4       |\n",
      "|    n_updates            | 46420      |\n",
      "|    policy_gradient_loss | -0.0308    |\n",
      "|    std                  | 0.316      |\n",
      "|    value_loss           | 272        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2671120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.47\n",
      "Num timesteps: 2681120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 114.28\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 113        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 440        |\n",
      "|    iterations           | 1310       |\n",
      "|    time_elapsed         | 6092       |\n",
      "|    total_timesteps      | 2682880    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.49984843 |\n",
      "|    clip_fraction        | 0.687      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.7       |\n",
      "|    explained_variance   | 0.812      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 69.3       |\n",
      "|    n_updates            | 46520      |\n",
      "|    policy_gradient_loss | -0.00501   |\n",
      "|    std                  | 0.303      |\n",
      "|    value_loss           | 228        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2691120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 110.00\n",
      "Num timesteps: 2701120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 141.13\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 128        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 440        |\n",
      "|    iterations           | 1320       |\n",
      "|    time_elapsed         | 6136       |\n",
      "|    total_timesteps      | 2703360    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.44632128 |\n",
      "|    clip_fraction        | 0.585      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.69      |\n",
      "|    explained_variance   | 0.782      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 45         |\n",
      "|    n_updates            | 46620      |\n",
      "|    policy_gradient_loss | -0.00339   |\n",
      "|    std                  | 0.305      |\n",
      "|    value_loss           | 304        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2711120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 117.78\n",
      "Num timesteps: 2721120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.49\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 149        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 440        |\n",
      "|    iterations           | 1330       |\n",
      "|    time_elapsed         | 6181       |\n",
      "|    total_timesteps      | 2723840    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.20862052 |\n",
      "|    clip_fraction        | 0.581      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.757     |\n",
      "|    explained_variance   | 0.779      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 102        |\n",
      "|    n_updates            | 46720      |\n",
      "|    policy_gradient_loss | -0.0255    |\n",
      "|    std                  | 0.31       |\n",
      "|    value_loss           | 277        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2731120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.17\n",
      "Num timesteps: 2741120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 123.92\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 141        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 440        |\n",
      "|    iterations           | 1340       |\n",
      "|    time_elapsed         | 6225       |\n",
      "|    total_timesteps      | 2744320    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.30218485 |\n",
      "|    clip_fraction        | 0.663      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.791     |\n",
      "|    explained_variance   | 0.746      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 52.2       |\n",
      "|    n_updates            | 46820      |\n",
      "|    policy_gradient_loss | -0.0101    |\n",
      "|    std                  | 0.315      |\n",
      "|    value_loss           | 192        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2751120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 141.95\n",
      "Num timesteps: 2761120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 117.34\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 147        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 440        |\n",
      "|    iterations           | 1350       |\n",
      "|    time_elapsed         | 6269       |\n",
      "|    total_timesteps      | 2764800    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.49989063 |\n",
      "|    clip_fraction        | 0.698      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.696     |\n",
      "|    explained_variance   | 0.792      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 44.6       |\n",
      "|    n_updates            | 46920      |\n",
      "|    policy_gradient_loss | -0.00429   |\n",
      "|    std                  | 0.303      |\n",
      "|    value_loss           | 237        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2771120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 123.78\n",
      "Num timesteps: 2781120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 126.76\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 129        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 441        |\n",
      "|    iterations           | 1360       |\n",
      "|    time_elapsed         | 6313       |\n",
      "|    total_timesteps      | 2785280    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.24182864 |\n",
      "|    clip_fraction        | 0.6        |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.72      |\n",
      "|    explained_variance   | 0.771      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 52.1       |\n",
      "|    n_updates            | 47020      |\n",
      "|    policy_gradient_loss | -0.0308    |\n",
      "|    std                  | 0.306      |\n",
      "|    value_loss           | 238        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 2791120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 133.71\n",
      "Num timesteps: 2801120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.19\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 115       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 441       |\n",
      "|    iterations           | 1370      |\n",
      "|    time_elapsed         | 6358      |\n",
      "|    total_timesteps      | 2805760   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 5.1125007 |\n",
      "|    clip_fraction        | 0.852     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.84     |\n",
      "|    explained_variance   | 0.783     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 76        |\n",
      "|    n_updates            | 47120     |\n",
      "|    policy_gradient_loss | 0.127     |\n",
      "|    std                  | 0.324     |\n",
      "|    value_loss           | 223       |\n",
      "---------------------------------------\n",
      "Num timesteps: 2811120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 116.13\n",
      "Num timesteps: 2821120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 122.23\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 133       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 441       |\n",
      "|    iterations           | 1380      |\n",
      "|    time_elapsed         | 6402      |\n",
      "|    total_timesteps      | 2826240   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.6804065 |\n",
      "|    clip_fraction        | 0.653     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.873    |\n",
      "|    explained_variance   | 0.695     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 42.9      |\n",
      "|    n_updates            | 47220     |\n",
      "|    policy_gradient_loss | 0.00021   |\n",
      "|    std                  | 0.323     |\n",
      "|    value_loss           | 232       |\n",
      "---------------------------------------\n",
      "Num timesteps: 2831120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.13\n",
      "Num timesteps: 2841120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 120.55\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 112       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 441       |\n",
      "|    iterations           | 1390      |\n",
      "|    time_elapsed         | 6446      |\n",
      "|    total_timesteps      | 2846720   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.3391341 |\n",
      "|    clip_fraction        | 0.641     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.739    |\n",
      "|    explained_variance   | 0.794     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 37.9      |\n",
      "|    n_updates            | 47320     |\n",
      "|    policy_gradient_loss | -0.0205   |\n",
      "|    std                  | 0.307     |\n",
      "|    value_loss           | 172       |\n",
      "---------------------------------------\n",
      "Num timesteps: 2851120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.45\n",
      "Num timesteps: 2861120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 117.86\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 117       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 441       |\n",
      "|    iterations           | 1400      |\n",
      "|    time_elapsed         | 6490      |\n",
      "|    total_timesteps      | 2867200   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.3180843 |\n",
      "|    clip_fraction        | 0.592     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.798    |\n",
      "|    explained_variance   | 0.716     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 82.8      |\n",
      "|    n_updates            | 47420     |\n",
      "|    policy_gradient_loss | -0.0159   |\n",
      "|    std                  | 0.317     |\n",
      "|    value_loss           | 365       |\n",
      "---------------------------------------\n",
      "Num timesteps: 2871120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 123.87\n",
      "Num timesteps: 2881120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 118.33\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 116       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 441       |\n",
      "|    iterations           | 1410      |\n",
      "|    time_elapsed         | 6534      |\n",
      "|    total_timesteps      | 2887680   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.5682774 |\n",
      "|    clip_fraction        | 0.701     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.831    |\n",
      "|    explained_variance   | 0.853     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 40.9      |\n",
      "|    n_updates            | 47520     |\n",
      "|    policy_gradient_loss | -0.001    |\n",
      "|    std                  | 0.318     |\n",
      "|    value_loss           | 161       |\n",
      "---------------------------------------\n",
      "Num timesteps: 2891120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 121.17\n",
      "Num timesteps: 2901120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.20\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 132        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 442        |\n",
      "|    iterations           | 1420       |\n",
      "|    time_elapsed         | 6579       |\n",
      "|    total_timesteps      | 2908160    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.18137012 |\n",
      "|    clip_fraction        | 0.585      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.785     |\n",
      "|    explained_variance   | 0.65       |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 76.3       |\n",
      "|    n_updates            | 47620      |\n",
      "|    policy_gradient_loss | -0.0236    |\n",
      "|    std                  | 0.31       |\n",
      "|    value_loss           | 335        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2911120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 141.74\n",
      "Num timesteps: 2921120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.16\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 123        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 442        |\n",
      "|    iterations           | 1430       |\n",
      "|    time_elapsed         | 6623       |\n",
      "|    total_timesteps      | 2928640    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.47905958 |\n",
      "|    clip_fraction        | 0.694      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.766     |\n",
      "|    explained_variance   | 0.812      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 36.9       |\n",
      "|    n_updates            | 47720      |\n",
      "|    policy_gradient_loss | -8.42e-07  |\n",
      "|    std                  | 0.313      |\n",
      "|    value_loss           | 150        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2931120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 131.15\n",
      "Num timesteps: 2941120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 122.27\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 138        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 442        |\n",
      "|    iterations           | 1440       |\n",
      "|    time_elapsed         | 6667       |\n",
      "|    total_timesteps      | 2949120    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.57300013 |\n",
      "|    clip_fraction        | 0.694      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.867     |\n",
      "|    explained_variance   | 0.756      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 57.9       |\n",
      "|    n_updates            | 47820      |\n",
      "|    policy_gradient_loss | 0.0214     |\n",
      "|    std                  | 0.327      |\n",
      "|    value_loss           | 198        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 2951120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.47\n",
      "Num timesteps: 2961120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 127.29\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 117        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 442        |\n",
      "|    iterations           | 1450       |\n",
      "|    time_elapsed         | 6711       |\n",
      "|    total_timesteps      | 2969600    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.37740672 |\n",
      "|    clip_fraction        | 0.614      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.06      |\n",
      "|    explained_variance   | 0.659      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 47.3       |\n",
      "|    n_updates            | 47920      |\n",
      "|    policy_gradient_loss | -0.0161    |\n",
      "|    std                  | 0.344      |\n",
      "|    value_loss           | 241        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2971120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 110.68\n",
      "Num timesteps: 2981120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 120.19\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 118        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 442        |\n",
      "|    iterations           | 1460       |\n",
      "|    time_elapsed         | 6755       |\n",
      "|    total_timesteps      | 2990080    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.41719973 |\n",
      "|    clip_fraction        | 0.622      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.951     |\n",
      "|    explained_variance   | 0.776      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 67.9       |\n",
      "|    n_updates            | 48020      |\n",
      "|    policy_gradient_loss | -0.0193    |\n",
      "|    std                  | 0.328      |\n",
      "|    value_loss           | 239        |\n",
      "----------------------------------------\n",
      "Num timesteps: 2991120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 125.96\n",
      "Num timesteps: 3001120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 124.57\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 117        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 442        |\n",
      "|    iterations           | 1470       |\n",
      "|    time_elapsed         | 6799       |\n",
      "|    total_timesteps      | 3010560    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.26562217 |\n",
      "|    clip_fraction        | 0.638      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.964     |\n",
      "|    explained_variance   | 0.766      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 127        |\n",
      "|    n_updates            | 48120      |\n",
      "|    policy_gradient_loss | -0.0185    |\n",
      "|    std                  | 0.334      |\n",
      "|    value_loss           | 292        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3011120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 112.81\n",
      "Num timesteps: 3021120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.15\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 129        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 442        |\n",
      "|    iterations           | 1480       |\n",
      "|    time_elapsed         | 6844       |\n",
      "|    total_timesteps      | 3031040    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.46086898 |\n",
      "|    clip_fraction        | 0.684      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.976     |\n",
      "|    explained_variance   | 0.803      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 47.8       |\n",
      "|    n_updates            | 48220      |\n",
      "|    policy_gradient_loss | 0.0141     |\n",
      "|    std                  | 0.338      |\n",
      "|    value_loss           | 196        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3031120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 129.44\n",
      "Num timesteps: 3041120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 129.55\n",
      "Num timesteps: 3051120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 121.79\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 125        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 443        |\n",
      "|    iterations           | 1490       |\n",
      "|    time_elapsed         | 6888       |\n",
      "|    total_timesteps      | 3051520    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.30730695 |\n",
      "|    clip_fraction        | 0.643      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.971     |\n",
      "|    explained_variance   | 0.643      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 34.3       |\n",
      "|    n_updates            | 48320      |\n",
      "|    policy_gradient_loss | -0.0123    |\n",
      "|    std                  | 0.333      |\n",
      "|    value_loss           | 221        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3061120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 129.34\n",
      "Num timesteps: 3071120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.22\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 117        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 443        |\n",
      "|    iterations           | 1500       |\n",
      "|    time_elapsed         | 6932       |\n",
      "|    total_timesteps      | 3072000    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.22646464 |\n",
      "|    clip_fraction        | 0.572      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.03      |\n",
      "|    explained_variance   | 0.791      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 85.1       |\n",
      "|    n_updates            | 48420      |\n",
      "|    policy_gradient_loss | -0.0166    |\n",
      "|    std                  | 0.341      |\n",
      "|    value_loss           | 314        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3081120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.09\n",
      "Num timesteps: 3091120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 125.10\n",
      "--------------------------------------\n",
      "| rollout/                |          |\n",
      "|    ep_len_mean          | 30       |\n",
      "|    ep_rew_mean          | 129      |\n",
      "| time/                   |          |\n",
      "|    fps                  | 443      |\n",
      "|    iterations           | 1510     |\n",
      "|    time_elapsed         | 6976     |\n",
      "|    total_timesteps      | 3092480  |\n",
      "| train/                  |          |\n",
      "|    approx_kl            | 8.342084 |\n",
      "|    clip_fraction        | 0.83     |\n",
      "|    clip_range           | 0.2      |\n",
      "|    entropy_loss         | -1.12    |\n",
      "|    explained_variance   | 0.712    |\n",
      "|    learning_rate        | 0.001    |\n",
      "|    loss                 | 155      |\n",
      "|    n_updates            | 48520    |\n",
      "|    policy_gradient_loss | 0.15     |\n",
      "|    std                  | 0.358    |\n",
      "|    value_loss           | 329      |\n",
      "--------------------------------------\n",
      "Num timesteps: 3101120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.22\n",
      "Num timesteps: 3111120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 141.90\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 133        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 443        |\n",
      "|    iterations           | 1520       |\n",
      "|    time_elapsed         | 7020       |\n",
      "|    total_timesteps      | 3112960    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.19463699 |\n",
      "|    clip_fraction        | 0.578      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.06      |\n",
      "|    explained_variance   | 0.848      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 58.6       |\n",
      "|    n_updates            | 48620      |\n",
      "|    policy_gradient_loss | -0.0243    |\n",
      "|    std                  | 0.344      |\n",
      "|    value_loss           | 243        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 3121120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 118.45\n",
      "Num timesteps: 3131120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 126.02\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 122        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 443        |\n",
      "|    iterations           | 1530       |\n",
      "|    time_elapsed         | 7064       |\n",
      "|    total_timesteps      | 3133440    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.46805215 |\n",
      "|    clip_fraction        | 0.606      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.15      |\n",
      "|    explained_variance   | 0.661      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 86.1       |\n",
      "|    n_updates            | 48720      |\n",
      "|    policy_gradient_loss | -0.00279   |\n",
      "|    std                  | 0.358      |\n",
      "|    value_loss           | 264        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3141120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 131.76\n",
      "Num timesteps: 3151120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 122.34\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 131        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 443        |\n",
      "|    iterations           | 1540       |\n",
      "|    time_elapsed         | 7108       |\n",
      "|    total_timesteps      | 3153920    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.65799165 |\n",
      "|    clip_fraction        | 0.684      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.09      |\n",
      "|    explained_variance   | 0.675      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 50.1       |\n",
      "|    n_updates            | 48820      |\n",
      "|    policy_gradient_loss | 0.00419    |\n",
      "|    std                  | 0.347      |\n",
      "|    value_loss           | 209        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3161120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 133.32\n",
      "Num timesteps: 3171120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 129.96\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 140        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 443        |\n",
      "|    iterations           | 1550       |\n",
      "|    time_elapsed         | 7153       |\n",
      "|    total_timesteps      | 3174400    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.36010337 |\n",
      "|    clip_fraction        | 0.641      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.09      |\n",
      "|    explained_variance   | 0.762      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 49.5       |\n",
      "|    n_updates            | 48920      |\n",
      "|    policy_gradient_loss | -0.0123    |\n",
      "|    std                  | 0.347      |\n",
      "|    value_loss           | 229        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3181120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.83\n",
      "Num timesteps: 3191120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 124.84\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 126       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 443       |\n",
      "|    iterations           | 1560      |\n",
      "|    time_elapsed         | 7197      |\n",
      "|    total_timesteps      | 3194880   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.2146495 |\n",
      "|    clip_fraction        | 0.562     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.17     |\n",
      "|    explained_variance   | 0.755     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 175       |\n",
      "|    n_updates            | 49020     |\n",
      "|    policy_gradient_loss | -0.0132   |\n",
      "|    std                  | 0.359     |\n",
      "|    value_loss           | 273       |\n",
      "---------------------------------------\n",
      "Num timesteps: 3201120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.39\n",
      "Num timesteps: 3211120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 134.57\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 140       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 444       |\n",
      "|    iterations           | 1570      |\n",
      "|    time_elapsed         | 7241      |\n",
      "|    total_timesteps      | 3215360   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.4316234 |\n",
      "|    clip_fraction        | 0.676     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.07     |\n",
      "|    explained_variance   | 0.73      |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 138       |\n",
      "|    n_updates            | 49120     |\n",
      "|    policy_gradient_loss | 0.0165    |\n",
      "|    std                  | 0.347     |\n",
      "|    value_loss           | 305       |\n",
      "---------------------------------------\n",
      "Num timesteps: 3221120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 137.67\n",
      "Num timesteps: 3231120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 133.54\n",
      "--------------------------------------\n",
      "| rollout/                |          |\n",
      "|    ep_len_mean          | 30       |\n",
      "|    ep_rew_mean          | 133      |\n",
      "| time/                   |          |\n",
      "|    fps                  | 444      |\n",
      "|    iterations           | 1580     |\n",
      "|    time_elapsed         | 7285     |\n",
      "|    total_timesteps      | 3235840  |\n",
      "| train/                  |          |\n",
      "|    approx_kl            | 0.284999 |\n",
      "|    clip_fraction        | 0.584    |\n",
      "|    clip_range           | 0.2      |\n",
      "|    entropy_loss         | -1.04    |\n",
      "|    explained_variance   | 0.763    |\n",
      "|    learning_rate        | 0.001    |\n",
      "|    loss                 | 123      |\n",
      "|    n_updates            | 49220    |\n",
      "|    policy_gradient_loss | -0.0283  |\n",
      "|    std                  | 0.343    |\n",
      "|    value_loss           | 367      |\n",
      "--------------------------------------\n",
      "Num timesteps: 3241120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.80\n",
      "Num timesteps: 3251120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 126.25\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 141        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 444        |\n",
      "|    iterations           | 1590       |\n",
      "|    time_elapsed         | 7329       |\n",
      "|    total_timesteps      | 3256320    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.31056762 |\n",
      "|    clip_fraction        | 0.605      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.11      |\n",
      "|    explained_variance   | 0.876      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 59.5       |\n",
      "|    n_updates            | 49320      |\n",
      "|    policy_gradient_loss | -0.0234    |\n",
      "|    std                  | 0.348      |\n",
      "|    value_loss           | 238        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3261120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 117.41\n",
      "Num timesteps: 3271120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 126.01\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 126       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 444       |\n",
      "|    iterations           | 1600      |\n",
      "|    time_elapsed         | 7374      |\n",
      "|    total_timesteps      | 3276800   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.2649529 |\n",
      "|    clip_fraction        | 0.616     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.06     |\n",
      "|    explained_variance   | 0.815     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 93.3      |\n",
      "|    n_updates            | 49420     |\n",
      "|    policy_gradient_loss | -0.024    |\n",
      "|    std                  | 0.345     |\n",
      "|    value_loss           | 286       |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 3281120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.25\n",
      "Num timesteps: 3291120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 124.01\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 143        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 444        |\n",
      "|    iterations           | 1610       |\n",
      "|    time_elapsed         | 7418       |\n",
      "|    total_timesteps      | 3297280    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.35282692 |\n",
      "|    clip_fraction        | 0.651      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.05      |\n",
      "|    explained_variance   | 0.788      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 47.7       |\n",
      "|    n_updates            | 49520      |\n",
      "|    policy_gradient_loss | -0.0118    |\n",
      "|    std                  | 0.341      |\n",
      "|    value_loss           | 218        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3301120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 118.26\n",
      "Num timesteps: 3311120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.87\n",
      "--------------------------------------\n",
      "| rollout/                |          |\n",
      "|    ep_len_mean          | 30       |\n",
      "|    ep_rew_mean          | 134      |\n",
      "| time/                   |          |\n",
      "|    fps                  | 444      |\n",
      "|    iterations           | 1620     |\n",
      "|    time_elapsed         | 7462     |\n",
      "|    total_timesteps      | 3317760  |\n",
      "| train/                  |          |\n",
      "|    approx_kl            | 0.255287 |\n",
      "|    clip_fraction        | 0.576    |\n",
      "|    clip_range           | 0.2      |\n",
      "|    entropy_loss         | -1.04    |\n",
      "|    explained_variance   | 0.767    |\n",
      "|    learning_rate        | 0.001    |\n",
      "|    loss                 | 41.9     |\n",
      "|    n_updates            | 49620    |\n",
      "|    policy_gradient_loss | -0.033   |\n",
      "|    std                  | 0.342    |\n",
      "|    value_loss           | 292      |\n",
      "--------------------------------------\n",
      "Num timesteps: 3321120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 122.60\n",
      "Num timesteps: 3331120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 127.96\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 132        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 444        |\n",
      "|    iterations           | 1630       |\n",
      "|    time_elapsed         | 7506       |\n",
      "|    total_timesteps      | 3338240    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.49625295 |\n",
      "|    clip_fraction        | 0.647      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.08      |\n",
      "|    explained_variance   | 0.916      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 63.4       |\n",
      "|    n_updates            | 49720      |\n",
      "|    policy_gradient_loss | -0.016     |\n",
      "|    std                  | 0.347      |\n",
      "|    value_loss           | 141        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3341120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 120.47\n",
      "Num timesteps: 3351120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 122.34\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 123        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 444        |\n",
      "|    iterations           | 1640       |\n",
      "|    time_elapsed         | 7551       |\n",
      "|    total_timesteps      | 3358720    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.28583902 |\n",
      "|    clip_fraction        | 0.57       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.13      |\n",
      "|    explained_variance   | 0.742      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 73.4       |\n",
      "|    n_updates            | 49820      |\n",
      "|    policy_gradient_loss | -0.0295    |\n",
      "|    std                  | 0.351      |\n",
      "|    value_loss           | 251        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3361120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 126.75\n",
      "Num timesteps: 3371120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 117.43\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 134        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 444        |\n",
      "|    iterations           | 1650       |\n",
      "|    time_elapsed         | 7595       |\n",
      "|    total_timesteps      | 3379200    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.65514493 |\n",
      "|    clip_fraction        | 0.731      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.09      |\n",
      "|    explained_variance   | 0.779      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 38.6       |\n",
      "|    n_updates            | 49920      |\n",
      "|    policy_gradient_loss | 0.0317     |\n",
      "|    std                  | 0.347      |\n",
      "|    value_loss           | 190        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3381120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 131.49\n",
      "Num timesteps: 3391120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.53\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 131        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 445        |\n",
      "|    iterations           | 1660       |\n",
      "|    time_elapsed         | 7639       |\n",
      "|    total_timesteps      | 3399680    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.21743038 |\n",
      "|    clip_fraction        | 0.576      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.979     |\n",
      "|    explained_variance   | 0.864      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 91.2       |\n",
      "|    n_updates            | 50020      |\n",
      "|    policy_gradient_loss | -0.0212    |\n",
      "|    std                  | 0.333      |\n",
      "|    value_loss           | 278        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3401120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.26\n",
      "Num timesteps: 3411120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 123.21\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 140        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 445        |\n",
      "|    iterations           | 1670       |\n",
      "|    time_elapsed         | 7683       |\n",
      "|    total_timesteps      | 3420160    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.24513613 |\n",
      "|    clip_fraction        | 0.584      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.07      |\n",
      "|    explained_variance   | 0.691      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 94.9       |\n",
      "|    n_updates            | 50120      |\n",
      "|    policy_gradient_loss | -0.0321    |\n",
      "|    std                  | 0.343      |\n",
      "|    value_loss           | 241        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3421120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 138.64\n",
      "Num timesteps: 3431120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 145.42\n",
      "--------------------------------------\n",
      "| rollout/                |          |\n",
      "|    ep_len_mean          | 30       |\n",
      "|    ep_rew_mean          | 140      |\n",
      "| time/                   |          |\n",
      "|    fps                  | 445      |\n",
      "|    iterations           | 1680     |\n",
      "|    time_elapsed         | 7727     |\n",
      "|    total_timesteps      | 3440640  |\n",
      "| train/                  |          |\n",
      "|    approx_kl            | 0.273    |\n",
      "|    clip_fraction        | 0.618    |\n",
      "|    clip_range           | 0.2      |\n",
      "|    entropy_loss         | -1.14    |\n",
      "|    explained_variance   | 0.715    |\n",
      "|    learning_rate        | 0.001    |\n",
      "|    loss                 | 70.8     |\n",
      "|    n_updates            | 50220    |\n",
      "|    policy_gradient_loss | -0.0291  |\n",
      "|    std                  | 0.352    |\n",
      "|    value_loss           | 225      |\n",
      "--------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 3441120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 148.12\n",
      "Num timesteps: 3451120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.56\n",
      "Num timesteps: 3461120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 115.24\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 115        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 445        |\n",
      "|    iterations           | 1690       |\n",
      "|    time_elapsed         | 7772       |\n",
      "|    total_timesteps      | 3461120    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.38932618 |\n",
      "|    clip_fraction        | 0.601      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.09      |\n",
      "|    explained_variance   | 0.834      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 78         |\n",
      "|    n_updates            | 50320      |\n",
      "|    policy_gradient_loss | -0.0252    |\n",
      "|    std                  | 0.347      |\n",
      "|    value_loss           | 234        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3471120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 127.21\n",
      "Num timesteps: 3481120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 123.06\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 127        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 445        |\n",
      "|    iterations           | 1700       |\n",
      "|    time_elapsed         | 7816       |\n",
      "|    total_timesteps      | 3481600    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.36880767 |\n",
      "|    clip_fraction        | 0.659      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.1       |\n",
      "|    explained_variance   | 0.809      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 51.6       |\n",
      "|    n_updates            | 50420      |\n",
      "|    policy_gradient_loss | 0.00022    |\n",
      "|    std                  | 0.35       |\n",
      "|    value_loss           | 208        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3491120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 109.69\n",
      "Num timesteps: 3501120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 111.96\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 112       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 445       |\n",
      "|    iterations           | 1710      |\n",
      "|    time_elapsed         | 7860      |\n",
      "|    total_timesteps      | 3502080   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.2829073 |\n",
      "|    clip_fraction        | 0.595     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.06     |\n",
      "|    explained_variance   | 0.811     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 55.2      |\n",
      "|    n_updates            | 50520     |\n",
      "|    policy_gradient_loss | -0.016    |\n",
      "|    std                  | 0.345     |\n",
      "|    value_loss           | 247       |\n",
      "---------------------------------------\n",
      "Num timesteps: 3511120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 126.69\n",
      "Num timesteps: 3521120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 120.03\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 125       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 445       |\n",
      "|    iterations           | 1720      |\n",
      "|    time_elapsed         | 7904      |\n",
      "|    total_timesteps      | 3522560   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.3056226 |\n",
      "|    clip_fraction        | 0.603     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.13     |\n",
      "|    explained_variance   | 0.72      |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 30.7      |\n",
      "|    n_updates            | 50620     |\n",
      "|    policy_gradient_loss | -0.0179   |\n",
      "|    std                  | 0.354     |\n",
      "|    value_loss           | 199       |\n",
      "---------------------------------------\n",
      "Num timesteps: 3531120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.83\n",
      "Num timesteps: 3541120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 122.84\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 122       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 445       |\n",
      "|    iterations           | 1730      |\n",
      "|    time_elapsed         | 7948      |\n",
      "|    total_timesteps      | 3543040   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.2459918 |\n",
      "|    clip_fraction        | 0.622     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.14     |\n",
      "|    explained_variance   | 0.79      |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 149       |\n",
      "|    n_updates            | 50720     |\n",
      "|    policy_gradient_loss | -0.0183   |\n",
      "|    std                  | 0.355     |\n",
      "|    value_loss           | 338       |\n",
      "---------------------------------------\n",
      "Num timesteps: 3551120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 129.12\n",
      "Num timesteps: 3561120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 137.08\n",
      "--------------------------------------\n",
      "| rollout/                |          |\n",
      "|    ep_len_mean          | 30       |\n",
      "|    ep_rew_mean          | 131      |\n",
      "| time/                   |          |\n",
      "|    fps                  | 445      |\n",
      "|    iterations           | 1740     |\n",
      "|    time_elapsed         | 7993     |\n",
      "|    total_timesteps      | 3563520  |\n",
      "| train/                  |          |\n",
      "|    approx_kl            | 0.276583 |\n",
      "|    clip_fraction        | 0.642    |\n",
      "|    clip_range           | 0.2      |\n",
      "|    entropy_loss         | -1.17    |\n",
      "|    explained_variance   | 0.829    |\n",
      "|    learning_rate        | 0.001    |\n",
      "|    loss                 | 93.6     |\n",
      "|    n_updates            | 50820    |\n",
      "|    policy_gradient_loss | -0.0138  |\n",
      "|    std                  | 0.359    |\n",
      "|    value_loss           | 235      |\n",
      "--------------------------------------\n",
      "Num timesteps: 3571120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 140.28\n",
      "Num timesteps: 3581120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 142.59\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 133        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 445        |\n",
      "|    iterations           | 1750       |\n",
      "|    time_elapsed         | 8037       |\n",
      "|    total_timesteps      | 3584000    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.36533615 |\n",
      "|    clip_fraction        | 0.593      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.15      |\n",
      "|    explained_variance   | 0.743      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 74.3       |\n",
      "|    n_updates            | 50920      |\n",
      "|    policy_gradient_loss | -0.017     |\n",
      "|    std                  | 0.358      |\n",
      "|    value_loss           | 283        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3591120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 121.06\n",
      "Num timesteps: 3601120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.41\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 119        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 446        |\n",
      "|    iterations           | 1760       |\n",
      "|    time_elapsed         | 8081       |\n",
      "|    total_timesteps      | 3604480    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.27389348 |\n",
      "|    clip_fraction        | 0.646      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.04      |\n",
      "|    explained_variance   | 0.751      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 36.3       |\n",
      "|    n_updates            | 51020      |\n",
      "|    policy_gradient_loss | -0.00293   |\n",
      "|    std                  | 0.345      |\n",
      "|    value_loss           | 299        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 3611120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.64\n",
      "Num timesteps: 3621120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.54\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 123        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 446        |\n",
      "|    iterations           | 1770       |\n",
      "|    time_elapsed         | 8125       |\n",
      "|    total_timesteps      | 3624960    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.34249324 |\n",
      "|    clip_fraction        | 0.619      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.03      |\n",
      "|    explained_variance   | 0.775      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 63.7       |\n",
      "|    n_updates            | 51120      |\n",
      "|    policy_gradient_loss | -0.00719   |\n",
      "|    std                  | 0.342      |\n",
      "|    value_loss           | 254        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3631120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 117.51\n",
      "Num timesteps: 3641120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 113.79\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 119        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 446        |\n",
      "|    iterations           | 1780       |\n",
      "|    time_elapsed         | 8169       |\n",
      "|    total_timesteps      | 3645440    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.42760652 |\n",
      "|    clip_fraction        | 0.622      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.2       |\n",
      "|    explained_variance   | 0.658      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 51         |\n",
      "|    n_updates            | 51220      |\n",
      "|    policy_gradient_loss | -0.0169    |\n",
      "|    std                  | 0.362      |\n",
      "|    value_loss           | 283        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3651120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 119.65\n",
      "Num timesteps: 3661120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 133.62\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 115       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 446       |\n",
      "|    iterations           | 1790      |\n",
      "|    time_elapsed         | 8213      |\n",
      "|    total_timesteps      | 3665920   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.4280802 |\n",
      "|    clip_fraction        | 0.618     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.15     |\n",
      "|    explained_variance   | 0.791     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 56        |\n",
      "|    n_updates            | 51320     |\n",
      "|    policy_gradient_loss | -0.0106   |\n",
      "|    std                  | 0.355     |\n",
      "|    value_loss           | 226       |\n",
      "---------------------------------------\n",
      "Num timesteps: 3671120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 139.02\n",
      "Num timesteps: 3681120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.96\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 134        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 446        |\n",
      "|    iterations           | 1800       |\n",
      "|    time_elapsed         | 8257       |\n",
      "|    total_timesteps      | 3686400    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.24381292 |\n",
      "|    clip_fraction        | 0.603      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.23      |\n",
      "|    explained_variance   | 0.894      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 37         |\n",
      "|    n_updates            | 51420      |\n",
      "|    policy_gradient_loss | -0.0375    |\n",
      "|    std                  | 0.363      |\n",
      "|    value_loss           | 132        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3691120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.83\n",
      "Num timesteps: 3701120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.99\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 124        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 446        |\n",
      "|    iterations           | 1810       |\n",
      "|    time_elapsed         | 8301       |\n",
      "|    total_timesteps      | 3706880    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.31482452 |\n",
      "|    clip_fraction        | 0.647      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.09      |\n",
      "|    explained_variance   | 0.786      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 47         |\n",
      "|    n_updates            | 51520      |\n",
      "|    policy_gradient_loss | -0.0077    |\n",
      "|    std                  | 0.347      |\n",
      "|    value_loss           | 203        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3711120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 137.43\n",
      "Num timesteps: 3721120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.57\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 128       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 446       |\n",
      "|    iterations           | 1820      |\n",
      "|    time_elapsed         | 8346      |\n",
      "|    total_timesteps      | 3727360   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.4504227 |\n",
      "|    clip_fraction        | 0.64      |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.07     |\n",
      "|    explained_variance   | 0.775     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 86        |\n",
      "|    n_updates            | 51620     |\n",
      "|    policy_gradient_loss | -0.0228   |\n",
      "|    std                  | 0.345     |\n",
      "|    value_loss           | 191       |\n",
      "---------------------------------------\n",
      "Num timesteps: 3731120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 129.83\n",
      "Num timesteps: 3741120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.78\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 119        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 446        |\n",
      "|    iterations           | 1830       |\n",
      "|    time_elapsed         | 8390       |\n",
      "|    total_timesteps      | 3747840    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.18037346 |\n",
      "|    clip_fraction        | 0.584      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.09      |\n",
      "|    explained_variance   | 0.765      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 54.6       |\n",
      "|    n_updates            | 51720      |\n",
      "|    policy_gradient_loss | -0.0207    |\n",
      "|    std                  | 0.348      |\n",
      "|    value_loss           | 348        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3751120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 127.48\n",
      "Num timesteps: 3761120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 127.18\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 131       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 446       |\n",
      "|    iterations           | 1840      |\n",
      "|    time_elapsed         | 8434      |\n",
      "|    total_timesteps      | 3768320   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.4802337 |\n",
      "|    clip_fraction        | 0.672     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.17     |\n",
      "|    explained_variance   | 0.773     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 38.2      |\n",
      "|    n_updates            | 51820     |\n",
      "|    policy_gradient_loss | 0.00601   |\n",
      "|    std                  | 0.354     |\n",
      "|    value_loss           | 188       |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 3771120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 126.09\n",
      "Num timesteps: 3781120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.37\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 137       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 446       |\n",
      "|    iterations           | 1850      |\n",
      "|    time_elapsed         | 8478      |\n",
      "|    total_timesteps      | 3788800   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.9747775 |\n",
      "|    clip_fraction        | 0.764     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.03     |\n",
      "|    explained_variance   | 0.789     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 52.8      |\n",
      "|    n_updates            | 51920     |\n",
      "|    policy_gradient_loss | 0.0577    |\n",
      "|    std                  | 0.344     |\n",
      "|    value_loss           | 174       |\n",
      "---------------------------------------\n",
      "Num timesteps: 3791120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 138.68\n",
      "Num timesteps: 3801120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 144.04\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 131        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 446        |\n",
      "|    iterations           | 1860       |\n",
      "|    time_elapsed         | 8523       |\n",
      "|    total_timesteps      | 3809280    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.48056495 |\n",
      "|    clip_fraction        | 0.657      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.04      |\n",
      "|    explained_variance   | 0.812      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 43         |\n",
      "|    n_updates            | 52020      |\n",
      "|    policy_gradient_loss | 0.00163    |\n",
      "|    std                  | 0.343      |\n",
      "|    value_loss           | 209        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3811120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 146.50\n",
      "Num timesteps: 3821120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 139.86\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 145        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 447        |\n",
      "|    iterations           | 1870       |\n",
      "|    time_elapsed         | 8567       |\n",
      "|    total_timesteps      | 3829760    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.24098136 |\n",
      "|    clip_fraction        | 0.599      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.05      |\n",
      "|    explained_variance   | 0.859      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 100        |\n",
      "|    n_updates            | 52120      |\n",
      "|    policy_gradient_loss | -0.026     |\n",
      "|    std                  | 0.345      |\n",
      "|    value_loss           | 273        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3831120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.21\n",
      "Num timesteps: 3841120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 110.63\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 150        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 447        |\n",
      "|    iterations           | 1880       |\n",
      "|    time_elapsed         | 8611       |\n",
      "|    total_timesteps      | 3850240    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.37931198 |\n",
      "|    clip_fraction        | 0.646      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.971     |\n",
      "|    explained_variance   | 0.841      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 89         |\n",
      "|    n_updates            | 52220      |\n",
      "|    policy_gradient_loss | -0.0216    |\n",
      "|    std                  | 0.333      |\n",
      "|    value_loss           | 230        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3851120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 146.19\n",
      "Num timesteps: 3861120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.25\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 141        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 447        |\n",
      "|    iterations           | 1890       |\n",
      "|    time_elapsed         | 8655       |\n",
      "|    total_timesteps      | 3870720    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.48046798 |\n",
      "|    clip_fraction        | 0.636      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.09      |\n",
      "|    explained_variance   | 0.804      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 73.9       |\n",
      "|    n_updates            | 52320      |\n",
      "|    policy_gradient_loss | -0.0239    |\n",
      "|    std                  | 0.35       |\n",
      "|    value_loss           | 223        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3871120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 146.30\n",
      "Num timesteps: 3881120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 126.27\n",
      "Num timesteps: 3891120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.48\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 131        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 447        |\n",
      "|    iterations           | 1900       |\n",
      "|    time_elapsed         | 8699       |\n",
      "|    total_timesteps      | 3891200    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.17317124 |\n",
      "|    clip_fraction        | 0.595      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.12      |\n",
      "|    explained_variance   | 0.812      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 89         |\n",
      "|    n_updates            | 52420      |\n",
      "|    policy_gradient_loss | -0.0172    |\n",
      "|    std                  | 0.353      |\n",
      "|    value_loss           | 370        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3901120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.47\n",
      "Num timesteps: 3911120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 131.45\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 129        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 447        |\n",
      "|    iterations           | 1910       |\n",
      "|    time_elapsed         | 8743       |\n",
      "|    total_timesteps      | 3911680    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.32969716 |\n",
      "|    clip_fraction        | 0.631      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.12      |\n",
      "|    explained_variance   | 0.818      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 43.2       |\n",
      "|    n_updates            | 52520      |\n",
      "|    policy_gradient_loss | -0.0135    |\n",
      "|    std                  | 0.351      |\n",
      "|    value_loss           | 209        |\n",
      "----------------------------------------\n",
      "Num timesteps: 3921120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.98\n",
      "Num timesteps: 3931120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 146.66\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 138        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 447        |\n",
      "|    iterations           | 1920       |\n",
      "|    time_elapsed         | 8787       |\n",
      "|    total_timesteps      | 3932160    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.34606296 |\n",
      "|    clip_fraction        | 0.643      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.892     |\n",
      "|    explained_variance   | 0.791      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 38.3       |\n",
      "|    n_updates            | 52620      |\n",
      "|    policy_gradient_loss | -0.032     |\n",
      "|    std                  | 0.325      |\n",
      "|    value_loss           | 223        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 3941120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 154.78\n",
      "Num timesteps: 3951120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.47\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 148       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 447       |\n",
      "|    iterations           | 1930      |\n",
      "|    time_elapsed         | 8832      |\n",
      "|    total_timesteps      | 3952640   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.3790453 |\n",
      "|    clip_fraction        | 0.629     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.886    |\n",
      "|    explained_variance   | 0.785     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 54.9      |\n",
      "|    n_updates            | 52720     |\n",
      "|    policy_gradient_loss | -0.0145   |\n",
      "|    std                  | 0.326     |\n",
      "|    value_loss           | 283       |\n",
      "---------------------------------------\n",
      "Num timesteps: 3961120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 134.37\n",
      "Num timesteps: 3971120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.29\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 141       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 447       |\n",
      "|    iterations           | 1940      |\n",
      "|    time_elapsed         | 8876      |\n",
      "|    total_timesteps      | 3973120   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.8762715 |\n",
      "|    clip_fraction        | 0.654     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.878    |\n",
      "|    explained_variance   | 0.69      |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 47.8      |\n",
      "|    n_updates            | 52820     |\n",
      "|    policy_gradient_loss | -0.0138   |\n",
      "|    std                  | 0.324     |\n",
      "|    value_loss           | 279       |\n",
      "---------------------------------------\n",
      "Num timesteps: 3981120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 139.26\n",
      "Num timesteps: 3991120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 150.87\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 138       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 447       |\n",
      "|    iterations           | 1950      |\n",
      "|    time_elapsed         | 8920      |\n",
      "|    total_timesteps      | 3993600   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.4839314 |\n",
      "|    clip_fraction        | 0.641     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.882    |\n",
      "|    explained_variance   | 0.683     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 145       |\n",
      "|    n_updates            | 52920     |\n",
      "|    policy_gradient_loss | 0.014     |\n",
      "|    std                  | 0.327     |\n",
      "|    value_loss           | 565       |\n",
      "---------------------------------------\n",
      "Num timesteps: 4001120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 144.76\n",
      "Num timesteps: 4011120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 151.13\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 146       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 447       |\n",
      "|    iterations           | 1960      |\n",
      "|    time_elapsed         | 8964      |\n",
      "|    total_timesteps      | 4014080   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 4.4091516 |\n",
      "|    clip_fraction        | 0.849     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.815    |\n",
      "|    explained_variance   | 0.819     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 44.9      |\n",
      "|    n_updates            | 53020     |\n",
      "|    policy_gradient_loss | 0.134     |\n",
      "|    std                  | 0.321     |\n",
      "|    value_loss           | 204       |\n",
      "---------------------------------------\n",
      "Num timesteps: 4021120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 140.72\n",
      "Num timesteps: 4031120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 145.19\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 140        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 447        |\n",
      "|    iterations           | 1970       |\n",
      "|    time_elapsed         | 9008       |\n",
      "|    total_timesteps      | 4034560    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.48149103 |\n",
      "|    clip_fraction        | 0.698      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.815     |\n",
      "|    explained_variance   | 0.872      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 46.3       |\n",
      "|    n_updates            | 53120      |\n",
      "|    policy_gradient_loss | -0.00192   |\n",
      "|    std                  | 0.32       |\n",
      "|    value_loss           | 188        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4041120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.74\n",
      "Num timesteps: 4051120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 156.04\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 133        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 447        |\n",
      "|    iterations           | 1980       |\n",
      "|    time_elapsed         | 9052       |\n",
      "|    total_timesteps      | 4055040    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.94973075 |\n",
      "|    clip_fraction        | 0.68       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.753     |\n",
      "|    explained_variance   | 0.812      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 56.3       |\n",
      "|    n_updates            | 53220      |\n",
      "|    policy_gradient_loss | 0.00443    |\n",
      "|    std                  | 0.313      |\n",
      "|    value_loss           | 236        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4061120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 142.43\n",
      "Num timesteps: 4071120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 142.46\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 129       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 448       |\n",
      "|    iterations           | 1990      |\n",
      "|    time_elapsed         | 9097      |\n",
      "|    total_timesteps      | 4075520   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.2601498 |\n",
      "|    clip_fraction        | 0.6       |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.808    |\n",
      "|    explained_variance   | 0.789     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 60        |\n",
      "|    n_updates            | 53320     |\n",
      "|    policy_gradient_loss | -0.0232   |\n",
      "|    std                  | 0.319     |\n",
      "|    value_loss           | 295       |\n",
      "---------------------------------------\n",
      "Num timesteps: 4081120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 154.39\n",
      "Num timesteps: 4091120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.68\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 149       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 448       |\n",
      "|    iterations           | 2000      |\n",
      "|    time_elapsed         | 9141      |\n",
      "|    total_timesteps      | 4096000   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.3746281 |\n",
      "|    clip_fraction        | 0.652     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.835    |\n",
      "|    explained_variance   | 0.758     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 90.7      |\n",
      "|    n_updates            | 53420     |\n",
      "|    policy_gradient_loss | -0.0196   |\n",
      "|    std                  | 0.321     |\n",
      "|    value_loss           | 251       |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 4101120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 157.09\n",
      "Num timesteps: 4111120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 133.56\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 139       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 448       |\n",
      "|    iterations           | 2010      |\n",
      "|    time_elapsed         | 9185      |\n",
      "|    total_timesteps      | 4116480   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.4201376 |\n",
      "|    clip_fraction        | 0.691     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.904    |\n",
      "|    explained_variance   | 0.831     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 42.5      |\n",
      "|    n_updates            | 53520     |\n",
      "|    policy_gradient_loss | -0.00282  |\n",
      "|    std                  | 0.329     |\n",
      "|    value_loss           | 181       |\n",
      "---------------------------------------\n",
      "Num timesteps: 4121120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.77\n",
      "Num timesteps: 4131120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 145.11\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 122        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 448        |\n",
      "|    iterations           | 2020       |\n",
      "|    time_elapsed         | 9229       |\n",
      "|    total_timesteps      | 4136960    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.31267452 |\n",
      "|    clip_fraction        | 0.64       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.839     |\n",
      "|    explained_variance   | 0.847      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 81.9       |\n",
      "|    n_updates            | 53620      |\n",
      "|    policy_gradient_loss | -0.0202    |\n",
      "|    std                  | 0.319      |\n",
      "|    value_loss           | 239        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4141120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 146.95\n",
      "Num timesteps: 4151120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 142.72\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 139        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 448        |\n",
      "|    iterations           | 2030       |\n",
      "|    time_elapsed         | 9273       |\n",
      "|    total_timesteps      | 4157440    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.22326489 |\n",
      "|    clip_fraction        | 0.585      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.93      |\n",
      "|    explained_variance   | 0.78       |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 80.9       |\n",
      "|    n_updates            | 53720      |\n",
      "|    policy_gradient_loss | -0.034     |\n",
      "|    std                  | 0.33       |\n",
      "|    value_loss           | 326        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4161120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 141.92\n",
      "Num timesteps: 4171120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 146.50\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 137        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 448        |\n",
      "|    iterations           | 2040       |\n",
      "|    time_elapsed         | 9317       |\n",
      "|    total_timesteps      | 4177920    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.29524505 |\n",
      "|    clip_fraction        | 0.635      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.944     |\n",
      "|    explained_variance   | 0.861      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 101        |\n",
      "|    n_updates            | 53820      |\n",
      "|    policy_gradient_loss | -0.018     |\n",
      "|    std                  | 0.331      |\n",
      "|    value_loss           | 298        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4181120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.29\n",
      "Num timesteps: 4191120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 137.38\n",
      "--------------------------------------\n",
      "| rollout/                |          |\n",
      "|    ep_len_mean          | 30       |\n",
      "|    ep_rew_mean          | 145      |\n",
      "| time/                   |          |\n",
      "|    fps                  | 448      |\n",
      "|    iterations           | 2050     |\n",
      "|    time_elapsed         | 9362     |\n",
      "|    total_timesteps      | 4198400  |\n",
      "| train/                  |          |\n",
      "|    approx_kl            | 0.515468 |\n",
      "|    clip_fraction        | 0.631    |\n",
      "|    clip_range           | 0.2      |\n",
      "|    entropy_loss         | -1.05    |\n",
      "|    explained_variance   | 0.834    |\n",
      "|    learning_rate        | 0.001    |\n",
      "|    loss                 | 60.7     |\n",
      "|    n_updates            | 53920    |\n",
      "|    policy_gradient_loss | -0.0119  |\n",
      "|    std                  | 0.345    |\n",
      "|    value_loss           | 184      |\n",
      "--------------------------------------\n",
      "Num timesteps: 4201120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.47\n",
      "Num timesteps: 4211120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 171.65\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 135        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 448        |\n",
      "|    iterations           | 2060       |\n",
      "|    time_elapsed         | 9406       |\n",
      "|    total_timesteps      | 4218880    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.38903096 |\n",
      "|    clip_fraction        | 0.658      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.833     |\n",
      "|    explained_variance   | 0.84       |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 59.2       |\n",
      "|    n_updates            | 54020      |\n",
      "|    policy_gradient_loss | -0.0178    |\n",
      "|    std                  | 0.321      |\n",
      "|    value_loss           | 213        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4221120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 138.08\n",
      "Num timesteps: 4231120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 153.72\n",
      "--------------------------------------\n",
      "| rollout/                |          |\n",
      "|    ep_len_mean          | 30       |\n",
      "|    ep_rew_mean          | 156      |\n",
      "| time/                   |          |\n",
      "|    fps                  | 448      |\n",
      "|    iterations           | 2070     |\n",
      "|    time_elapsed         | 9450     |\n",
      "|    total_timesteps      | 4239360  |\n",
      "| train/                  |          |\n",
      "|    approx_kl            | 8.645811 |\n",
      "|    clip_fraction        | 0.696    |\n",
      "|    clip_range           | 0.2      |\n",
      "|    entropy_loss         | -0.994   |\n",
      "|    explained_variance   | 0.824    |\n",
      "|    learning_rate        | 0.001    |\n",
      "|    loss                 | 39.5     |\n",
      "|    n_updates            | 54120    |\n",
      "|    policy_gradient_loss | 0.0809   |\n",
      "|    std                  | 0.341    |\n",
      "|    value_loss           | 243      |\n",
      "--------------------------------------\n",
      "Num timesteps: 4241120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 157.61\n",
      "Num timesteps: 4251120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 150.26\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 136        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 448        |\n",
      "|    iterations           | 2080       |\n",
      "|    time_elapsed         | 9494       |\n",
      "|    total_timesteps      | 4259840    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.55240524 |\n",
      "|    clip_fraction        | 0.719      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.954     |\n",
      "|    explained_variance   | 0.834      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 56.2       |\n",
      "|    n_updates            | 54220      |\n",
      "|    policy_gradient_loss | 0.0326     |\n",
      "|    std                  | 0.335      |\n",
      "|    value_loss           | 169        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 4261120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 137.85\n",
      "Num timesteps: 4271120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 156.62\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 148        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 448        |\n",
      "|    iterations           | 2090       |\n",
      "|    time_elapsed         | 9538       |\n",
      "|    total_timesteps      | 4280320    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.23805878 |\n",
      "|    clip_fraction        | 0.594      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.993     |\n",
      "|    explained_variance   | 0.856      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 67.8       |\n",
      "|    n_updates            | 54320      |\n",
      "|    policy_gradient_loss | -0.0327    |\n",
      "|    std                  | 0.337      |\n",
      "|    value_loss           | 284        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4281120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 141.81\n",
      "Num timesteps: 4291120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.89\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 148       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 448       |\n",
      "|    iterations           | 2100      |\n",
      "|    time_elapsed         | 9583      |\n",
      "|    total_timesteps      | 4300800   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.4544838 |\n",
      "|    clip_fraction        | 0.644     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.938    |\n",
      "|    explained_variance   | 0.745     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 61.3      |\n",
      "|    n_updates            | 54420     |\n",
      "|    policy_gradient_loss | -0.0151   |\n",
      "|    std                  | 0.33      |\n",
      "|    value_loss           | 247       |\n",
      "---------------------------------------\n",
      "Num timesteps: 4301120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 151.45\n",
      "Num timesteps: 4311120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 149.84\n",
      "Num timesteps: 4321120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 142.52\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 147        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 448        |\n",
      "|    iterations           | 2110       |\n",
      "|    time_elapsed         | 9627       |\n",
      "|    total_timesteps      | 4321280    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.23591924 |\n",
      "|    clip_fraction        | 0.57       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.907     |\n",
      "|    explained_variance   | 0.779      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 137        |\n",
      "|    n_updates            | 54520      |\n",
      "|    policy_gradient_loss | -0.0213    |\n",
      "|    std                  | 0.327      |\n",
      "|    value_loss           | 360        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4331120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.43\n",
      "Num timesteps: 4341120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 134.56\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 137        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 448        |\n",
      "|    iterations           | 2120       |\n",
      "|    time_elapsed         | 9671       |\n",
      "|    total_timesteps      | 4341760    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.36486816 |\n",
      "|    clip_fraction        | 0.619      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.995     |\n",
      "|    explained_variance   | 0.859      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 49.1       |\n",
      "|    n_updates            | 54620      |\n",
      "|    policy_gradient_loss | -0.0257    |\n",
      "|    std                  | 0.336      |\n",
      "|    value_loss           | 179        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4351120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 140.77\n",
      "Num timesteps: 4361120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 129.21\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 125        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 448        |\n",
      "|    iterations           | 2130       |\n",
      "|    time_elapsed         | 9715       |\n",
      "|    total_timesteps      | 4362240    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.31998175 |\n",
      "|    clip_fraction        | 0.565      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.8       |\n",
      "|    explained_variance   | 0.547      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 222        |\n",
      "|    n_updates            | 54720      |\n",
      "|    policy_gradient_loss | -0.0139    |\n",
      "|    std                  | 0.314      |\n",
      "|    value_loss           | 555        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4371120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 129.54\n",
      "Num timesteps: 4381120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 142.46\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 130        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 449        |\n",
      "|    iterations           | 2140       |\n",
      "|    time_elapsed         | 9759       |\n",
      "|    total_timesteps      | 4382720    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.18839702 |\n",
      "|    clip_fraction        | 0.569      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.729     |\n",
      "|    explained_variance   | 0.794      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 230        |\n",
      "|    n_updates            | 54820      |\n",
      "|    policy_gradient_loss | -0.0344    |\n",
      "|    std                  | 0.305      |\n",
      "|    value_loss           | 499        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4391120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 120.73\n",
      "Num timesteps: 4401120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.11\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 128       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 449       |\n",
      "|    iterations           | 2150      |\n",
      "|    time_elapsed         | 9804      |\n",
      "|    total_timesteps      | 4403200   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 1.2884336 |\n",
      "|    clip_fraction        | 0.763     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.841    |\n",
      "|    explained_variance   | 0.743     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 72.7      |\n",
      "|    n_updates            | 54920     |\n",
      "|    policy_gradient_loss | 0.0512    |\n",
      "|    std                  | 0.324     |\n",
      "|    value_loss           | 225       |\n",
      "---------------------------------------\n",
      "Num timesteps: 4411120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 141.51\n",
      "Num timesteps: 4421120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 138.67\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 137       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 449       |\n",
      "|    iterations           | 2160      |\n",
      "|    time_elapsed         | 9848      |\n",
      "|    total_timesteps      | 4423680   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.3553065 |\n",
      "|    clip_fraction        | 0.588     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.1      |\n",
      "|    explained_variance   | 0.721     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 139       |\n",
      "|    n_updates            | 55020     |\n",
      "|    policy_gradient_loss | -0.0247   |\n",
      "|    std                  | 0.346     |\n",
      "|    value_loss           | 399       |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 4431120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 144.98\n",
      "Num timesteps: 4441120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.21\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 150        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 449        |\n",
      "|    iterations           | 2170       |\n",
      "|    time_elapsed         | 9892       |\n",
      "|    total_timesteps      | 4444160    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.20316765 |\n",
      "|    clip_fraction        | 0.569      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.917     |\n",
      "|    explained_variance   | 0.851      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 113        |\n",
      "|    n_updates            | 55120      |\n",
      "|    policy_gradient_loss | -0.0363    |\n",
      "|    std                  | 0.327      |\n",
      "|    value_loss           | 340        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4451120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 126.82\n",
      "Num timesteps: 4461120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.66\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 147       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 449       |\n",
      "|    iterations           | 2180      |\n",
      "|    time_elapsed         | 9936      |\n",
      "|    total_timesteps      | 4464640   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.3079535 |\n",
      "|    clip_fraction        | 0.603     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.881    |\n",
      "|    explained_variance   | 0.702     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 70.4      |\n",
      "|    n_updates            | 55220     |\n",
      "|    policy_gradient_loss | -0.0265   |\n",
      "|    std                  | 0.323     |\n",
      "|    value_loss           | 306       |\n",
      "---------------------------------------\n",
      "Num timesteps: 4471120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 144.55\n",
      "Num timesteps: 4481120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 139.74\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 136        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 449        |\n",
      "|    iterations           | 2190       |\n",
      "|    time_elapsed         | 9981       |\n",
      "|    total_timesteps      | 4485120    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.60934675 |\n",
      "|    clip_fraction        | 0.688      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.759     |\n",
      "|    explained_variance   | 0.746      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 51.8       |\n",
      "|    n_updates            | 55320      |\n",
      "|    policy_gradient_loss | 0.0136     |\n",
      "|    std                  | 0.313      |\n",
      "|    value_loss           | 353        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4491120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.45\n",
      "Num timesteps: 4501120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 145.68\n",
      "--------------------------------------\n",
      "| rollout/                |          |\n",
      "|    ep_len_mean          | 30       |\n",
      "|    ep_rew_mean          | 132      |\n",
      "| time/                   |          |\n",
      "|    fps                  | 449      |\n",
      "|    iterations           | 2200     |\n",
      "|    time_elapsed         | 10025    |\n",
      "|    total_timesteps      | 4505600  |\n",
      "| train/                  |          |\n",
      "|    approx_kl            | 1.976882 |\n",
      "|    clip_fraction        | 0.724    |\n",
      "|    clip_range           | 0.2      |\n",
      "|    entropy_loss         | -0.781   |\n",
      "|    explained_variance   | 0.754    |\n",
      "|    learning_rate        | 0.001    |\n",
      "|    loss                 | 56.1     |\n",
      "|    n_updates            | 55420    |\n",
      "|    policy_gradient_loss | 0.0224   |\n",
      "|    std                  | 0.313    |\n",
      "|    value_loss           | 229      |\n",
      "--------------------------------------\n",
      "Num timesteps: 4511120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 144.20\n",
      "Num timesteps: 4521120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 140.76\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 143       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 449       |\n",
      "|    iterations           | 2210      |\n",
      "|    time_elapsed         | 10069     |\n",
      "|    total_timesteps      | 4526080   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.5179342 |\n",
      "|    clip_fraction        | 0.675     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.994    |\n",
      "|    explained_variance   | 0.685     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 57.2      |\n",
      "|    n_updates            | 55520     |\n",
      "|    policy_gradient_loss | 0.0115    |\n",
      "|    std                  | 0.34      |\n",
      "|    value_loss           | 297       |\n",
      "---------------------------------------\n",
      "Num timesteps: 4531120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 119.63\n",
      "Num timesteps: 4541120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.29\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 129        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 449        |\n",
      "|    iterations           | 2220       |\n",
      "|    time_elapsed         | 10113      |\n",
      "|    total_timesteps      | 4546560    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.35641277 |\n",
      "|    clip_fraction        | 0.644      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.04      |\n",
      "|    explained_variance   | 0.776      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 105        |\n",
      "|    n_updates            | 55620      |\n",
      "|    policy_gradient_loss | -0.00985   |\n",
      "|    std                  | 0.342      |\n",
      "|    value_loss           | 203        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4551120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.18\n",
      "Num timesteps: 4561120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 122.70\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 144        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 449        |\n",
      "|    iterations           | 2230       |\n",
      "|    time_elapsed         | 10157      |\n",
      "|    total_timesteps      | 4567040    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.30911708 |\n",
      "|    clip_fraction        | 0.654      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.991     |\n",
      "|    explained_variance   | 0.755      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 66.5       |\n",
      "|    n_updates            | 55720      |\n",
      "|    policy_gradient_loss | -0.0141    |\n",
      "|    std                  | 0.332      |\n",
      "|    value_loss           | 212        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4571120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.38\n",
      "Num timesteps: 4581120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.98\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 130        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 449        |\n",
      "|    iterations           | 2240       |\n",
      "|    time_elapsed         | 10201      |\n",
      "|    total_timesteps      | 4587520    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.46678805 |\n",
      "|    clip_fraction        | 0.694      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.903     |\n",
      "|    explained_variance   | 0.783      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 63         |\n",
      "|    n_updates            | 55820      |\n",
      "|    policy_gradient_loss | 0.00818    |\n",
      "|    std                  | 0.326      |\n",
      "|    value_loss           | 250        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 4591120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 146.45\n",
      "Num timesteps: 4601120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 124.88\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 130        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 449        |\n",
      "|    iterations           | 2250       |\n",
      "|    time_elapsed         | 10245      |\n",
      "|    total_timesteps      | 4608000    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.18609178 |\n",
      "|    clip_fraction        | 0.581      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.98      |\n",
      "|    explained_variance   | 0.77       |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 192        |\n",
      "|    n_updates            | 55920      |\n",
      "|    policy_gradient_loss | -0.0298    |\n",
      "|    std                  | 0.334      |\n",
      "|    value_loss           | 370        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4611120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 118.88\n",
      "Num timesteps: 4621120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 116.01\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 126        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 449        |\n",
      "|    iterations           | 2260       |\n",
      "|    time_elapsed         | 10289      |\n",
      "|    total_timesteps      | 4628480    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.33608192 |\n",
      "|    clip_fraction        | 0.589      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.12      |\n",
      "|    explained_variance   | 0.683      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 57.9       |\n",
      "|    n_updates            | 56020      |\n",
      "|    policy_gradient_loss | -0.0184    |\n",
      "|    std                  | 0.352      |\n",
      "|    value_loss           | 316        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4631120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.28\n",
      "Num timesteps: 4641120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.07\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 135        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 449        |\n",
      "|    iterations           | 2270       |\n",
      "|    time_elapsed         | 10334      |\n",
      "|    total_timesteps      | 4648960    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.45617086 |\n",
      "|    clip_fraction        | 0.662      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.968     |\n",
      "|    explained_variance   | 0.828      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 112        |\n",
      "|    n_updates            | 56120      |\n",
      "|    policy_gradient_loss | -0.0043    |\n",
      "|    std                  | 0.332      |\n",
      "|    value_loss           | 255        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4651120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 139.70\n",
      "Num timesteps: 4661120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 137.51\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 136        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 449        |\n",
      "|    iterations           | 2280       |\n",
      "|    time_elapsed         | 10378      |\n",
      "|    total_timesteps      | 4669440    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.37924588 |\n",
      "|    clip_fraction        | 0.662      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.901     |\n",
      "|    explained_variance   | 0.799      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 62.6       |\n",
      "|    n_updates            | 56220      |\n",
      "|    policy_gradient_loss | -0.00953   |\n",
      "|    std                  | 0.327      |\n",
      "|    value_loss           | 286        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4671120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.51\n",
      "Num timesteps: 4681120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.75\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 124       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 449       |\n",
      "|    iterations           | 2290      |\n",
      "|    time_elapsed         | 10422     |\n",
      "|    total_timesteps      | 4689920   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.2613392 |\n",
      "|    clip_fraction        | 0.598     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.913    |\n",
      "|    explained_variance   | 0.675     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 72.9      |\n",
      "|    n_updates            | 56320     |\n",
      "|    policy_gradient_loss | -0.0235   |\n",
      "|    std                  | 0.325     |\n",
      "|    value_loss           | 438       |\n",
      "---------------------------------------\n",
      "Num timesteps: 4691120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.86\n",
      "Num timesteps: 4701120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 115.49\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 142       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 450       |\n",
      "|    iterations           | 2300      |\n",
      "|    time_elapsed         | 10466     |\n",
      "|    total_timesteps      | 4710400   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.8020536 |\n",
      "|    clip_fraction        | 0.727     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.909    |\n",
      "|    explained_variance   | 0.828     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 44.9      |\n",
      "|    n_updates            | 56420     |\n",
      "|    policy_gradient_loss | 0.0293    |\n",
      "|    std                  | 0.329     |\n",
      "|    value_loss           | 260       |\n",
      "---------------------------------------\n",
      "Num timesteps: 4711120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 145.72\n",
      "Num timesteps: 4721120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 119.40\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 120        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 450        |\n",
      "|    iterations           | 2310       |\n",
      "|    time_elapsed         | 10510      |\n",
      "|    total_timesteps      | 4730880    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.37816343 |\n",
      "|    clip_fraction        | 0.662      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.895     |\n",
      "|    explained_variance   | 0.779      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 52.4       |\n",
      "|    n_updates            | 56520      |\n",
      "|    policy_gradient_loss | -0.015     |\n",
      "|    std                  | 0.326      |\n",
      "|    value_loss           | 188        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4731120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 116.78\n",
      "Num timesteps: 4741120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 127.31\n",
      "Num timesteps: 4751120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 137.55\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 141        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 450        |\n",
      "|    iterations           | 2320       |\n",
      "|    time_elapsed         | 10555      |\n",
      "|    total_timesteps      | 4751360    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.26567036 |\n",
      "|    clip_fraction        | 0.56       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.932     |\n",
      "|    explained_variance   | 0.72       |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 153        |\n",
      "|    n_updates            | 56620      |\n",
      "|    policy_gradient_loss | -0.0302    |\n",
      "|    std                  | 0.329      |\n",
      "|    value_loss           | 317        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 4761120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 133.52\n",
      "Num timesteps: 4771120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.44\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 138        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 450        |\n",
      "|    iterations           | 2330       |\n",
      "|    time_elapsed         | 10599      |\n",
      "|    total_timesteps      | 4771840    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.33934146 |\n",
      "|    clip_fraction        | 0.603      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.905     |\n",
      "|    explained_variance   | 0.759      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 137        |\n",
      "|    n_updates            | 56720      |\n",
      "|    policy_gradient_loss | -0.0221    |\n",
      "|    std                  | 0.329      |\n",
      "|    value_loss           | 359        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4781120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.09\n",
      "Num timesteps: 4791120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 115.17\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 123        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 450        |\n",
      "|    iterations           | 2340       |\n",
      "|    time_elapsed         | 10643      |\n",
      "|    total_timesteps      | 4792320    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.37506706 |\n",
      "|    clip_fraction        | 0.601      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.951     |\n",
      "|    explained_variance   | 0.69       |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 83.9       |\n",
      "|    n_updates            | 56820      |\n",
      "|    policy_gradient_loss | -0.0246    |\n",
      "|    std                  | 0.331      |\n",
      "|    value_loss           | 289        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4801120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 114.29\n",
      "Num timesteps: 4811120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 133.82\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 133       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 450       |\n",
      "|    iterations           | 2350      |\n",
      "|    time_elapsed         | 10688     |\n",
      "|    total_timesteps      | 4812800   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.2060563 |\n",
      "|    clip_fraction        | 0.557     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.981    |\n",
      "|    explained_variance   | 0.732     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 74.8      |\n",
      "|    n_updates            | 56920     |\n",
      "|    policy_gradient_loss | -0.0369   |\n",
      "|    std                  | 0.335     |\n",
      "|    value_loss           | 350       |\n",
      "---------------------------------------\n",
      "Num timesteps: 4821120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 150.40\n",
      "Num timesteps: 4831120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 133.72\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 122        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 450        |\n",
      "|    iterations           | 2360       |\n",
      "|    time_elapsed         | 10732      |\n",
      "|    total_timesteps      | 4833280    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.20762134 |\n",
      "|    clip_fraction        | 0.556      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.1       |\n",
      "|    explained_variance   | 0.732      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 67         |\n",
      "|    n_updates            | 57020      |\n",
      "|    policy_gradient_loss | -0.0161    |\n",
      "|    std                  | 0.348      |\n",
      "|    value_loss           | 380        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4841120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 140.48\n",
      "Num timesteps: 4851120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 115.29\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 122        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 450        |\n",
      "|    iterations           | 2370       |\n",
      "|    time_elapsed         | 10776      |\n",
      "|    total_timesteps      | 4853760    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.24087515 |\n",
      "|    clip_fraction        | 0.598      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.05      |\n",
      "|    explained_variance   | 0.773      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 92         |\n",
      "|    n_updates            | 57120      |\n",
      "|    policy_gradient_loss | -0.0125    |\n",
      "|    std                  | 0.342      |\n",
      "|    value_loss           | 299        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4861120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 124.60\n",
      "Num timesteps: 4871120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 131.75\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 106        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 450        |\n",
      "|    iterations           | 2380       |\n",
      "|    time_elapsed         | 10820      |\n",
      "|    total_timesteps      | 4874240    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.49000204 |\n",
      "|    clip_fraction        | 0.632      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.877     |\n",
      "|    explained_variance   | 0.703      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 81.9       |\n",
      "|    n_updates            | 57220      |\n",
      "|    policy_gradient_loss | -0.00956   |\n",
      "|    std                  | 0.324      |\n",
      "|    value_loss           | 285        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4881120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 114.40\n",
      "Num timesteps: 4891120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 127.39\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 136        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 450        |\n",
      "|    iterations           | 2390       |\n",
      "|    time_elapsed         | 10864      |\n",
      "|    total_timesteps      | 4894720    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.18538675 |\n",
      "|    clip_fraction        | 0.535      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.833     |\n",
      "|    explained_variance   | 0.758      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 100        |\n",
      "|    n_updates            | 57320      |\n",
      "|    policy_gradient_loss | -0.0314    |\n",
      "|    std                  | 0.319      |\n",
      "|    value_loss           | 388        |\n",
      "----------------------------------------\n",
      "Num timesteps: 4901120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 134.37\n",
      "Num timesteps: 4911120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 118.79\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 126        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 450        |\n",
      "|    iterations           | 2400       |\n",
      "|    time_elapsed         | 10908      |\n",
      "|    total_timesteps      | 4915200    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.38749164 |\n",
      "|    clip_fraction        | 0.639      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.97      |\n",
      "|    explained_variance   | 0.781      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 67         |\n",
      "|    n_updates            | 57420      |\n",
      "|    policy_gradient_loss | -0.00962   |\n",
      "|    std                  | 0.335      |\n",
      "|    value_loss           | 261        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 4921120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.71\n",
      "Num timesteps: 4931120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 122.84\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 118       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 450       |\n",
      "|    iterations           | 2410      |\n",
      "|    time_elapsed         | 10952     |\n",
      "|    total_timesteps      | 4935680   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.2032316 |\n",
      "|    clip_fraction        | 0.598     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.874    |\n",
      "|    explained_variance   | 0.812     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 143       |\n",
      "|    n_updates            | 57520     |\n",
      "|    policy_gradient_loss | -0.0373   |\n",
      "|    std                  | 0.324     |\n",
      "|    value_loss           | 459       |\n",
      "---------------------------------------\n",
      "Num timesteps: 4941120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 141.91\n",
      "Num timesteps: 4951120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.09\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 137       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 450       |\n",
      "|    iterations           | 2420      |\n",
      "|    time_elapsed         | 10997     |\n",
      "|    total_timesteps      | 4956160   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.1332745 |\n",
      "|    clip_fraction        | 0.476     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.876    |\n",
      "|    explained_variance   | 0.626     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 165       |\n",
      "|    n_updates            | 57620     |\n",
      "|    policy_gradient_loss | -0.0328   |\n",
      "|    std                  | 0.322     |\n",
      "|    value_loss           | 469       |\n",
      "---------------------------------------\n",
      "Num timesteps: 4961120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 127.64\n",
      "Num timesteps: 4971120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.65\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 129       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 450       |\n",
      "|    iterations           | 2430      |\n",
      "|    time_elapsed         | 11041     |\n",
      "|    total_timesteps      | 4976640   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.5557818 |\n",
      "|    clip_fraction        | 0.67      |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.872    |\n",
      "|    explained_variance   | 0.632     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 22.9      |\n",
      "|    n_updates            | 57720     |\n",
      "|    policy_gradient_loss | -0.00813  |\n",
      "|    std                  | 0.322     |\n",
      "|    value_loss           | 223       |\n",
      "---------------------------------------\n",
      "Num timesteps: 4981120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 116.89\n",
      "Num timesteps: 4991120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 129.36\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 131       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 450       |\n",
      "|    iterations           | 2440      |\n",
      "|    time_elapsed         | 11085     |\n",
      "|    total_timesteps      | 4997120   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.2324221 |\n",
      "|    clip_fraction        | 0.566     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.947    |\n",
      "|    explained_variance   | 0.665     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 229       |\n",
      "|    n_updates            | 57820     |\n",
      "|    policy_gradient_loss | -0.0243   |\n",
      "|    std                  | 0.333     |\n",
      "|    value_loss           | 519       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5001120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 125.19\n",
      "Num timesteps: 5011120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 122.19\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 87.1       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 450        |\n",
      "|    iterations           | 2450       |\n",
      "|    time_elapsed         | 11129      |\n",
      "|    total_timesteps      | 5017600    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.23237737 |\n",
      "|    clip_fraction        | 0.546      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.994     |\n",
      "|    explained_variance   | 0.693      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 53.9       |\n",
      "|    n_updates            | 57920      |\n",
      "|    policy_gradient_loss | -0.0356    |\n",
      "|    std                  | 0.334      |\n",
      "|    value_loss           | 187        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5021120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 103.64\n",
      "Num timesteps: 5031120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 114.78\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 100        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 450        |\n",
      "|    iterations           | 2460       |\n",
      "|    time_elapsed         | 11174      |\n",
      "|    total_timesteps      | 5038080    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.22005576 |\n",
      "|    clip_fraction        | 0.618      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.82      |\n",
      "|    explained_variance   | 0.798      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 56.2       |\n",
      "|    n_updates            | 58020      |\n",
      "|    policy_gradient_loss | -0.0194    |\n",
      "|    std                  | 0.318      |\n",
      "|    value_loss           | 270        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5041120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 121.68\n",
      "Num timesteps: 5051120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 123.04\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 105        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 450        |\n",
      "|    iterations           | 2470       |\n",
      "|    time_elapsed         | 11218      |\n",
      "|    total_timesteps      | 5058560    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.21967681 |\n",
      "|    clip_fraction        | 0.595      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.822     |\n",
      "|    explained_variance   | 0.714      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 139        |\n",
      "|    n_updates            | 58120      |\n",
      "|    policy_gradient_loss | -0.0163    |\n",
      "|    std                  | 0.319      |\n",
      "|    value_loss           | 375        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5061120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 105.48\n",
      "Num timesteps: 5071120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 124.04\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 129       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 450       |\n",
      "|    iterations           | 2480      |\n",
      "|    time_elapsed         | 11262     |\n",
      "|    total_timesteps      | 5079040   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.6921386 |\n",
      "|    clip_fraction        | 0.676     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.906    |\n",
      "|    explained_variance   | 0.778     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 68.6      |\n",
      "|    n_updates            | 58220     |\n",
      "|    policy_gradient_loss | 0.00263   |\n",
      "|    std                  | 0.331     |\n",
      "|    value_loss           | 242       |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 5081120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.85\n",
      "Num timesteps: 5091120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 115.26\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 117        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 451        |\n",
      "|    iterations           | 2490       |\n",
      "|    time_elapsed         | 11306      |\n",
      "|    total_timesteps      | 5099520    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.21412724 |\n",
      "|    clip_fraction        | 0.58       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.812     |\n",
      "|    explained_variance   | 0.79       |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 110        |\n",
      "|    n_updates            | 58320      |\n",
      "|    policy_gradient_loss | -0.0125    |\n",
      "|    std                  | 0.318      |\n",
      "|    value_loss           | 321        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5101120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 111.56\n",
      "Num timesteps: 5111120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.78\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 103        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 451        |\n",
      "|    iterations           | 2500       |\n",
      "|    time_elapsed         | 11350      |\n",
      "|    total_timesteps      | 5120000    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.35693556 |\n",
      "|    clip_fraction        | 0.637      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.697     |\n",
      "|    explained_variance   | 0.812      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 57.7       |\n",
      "|    n_updates            | 58420      |\n",
      "|    policy_gradient_loss | -0.0241    |\n",
      "|    std                  | 0.305      |\n",
      "|    value_loss           | 186        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5121120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 104.67\n",
      "Num timesteps: 5131120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 100.52\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 117       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 451       |\n",
      "|    iterations           | 2510      |\n",
      "|    time_elapsed         | 11395     |\n",
      "|    total_timesteps      | 5140480   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 1.2237597 |\n",
      "|    clip_fraction        | 0.777     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.807    |\n",
      "|    explained_variance   | 0.697     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 79.3      |\n",
      "|    n_updates            | 58520     |\n",
      "|    policy_gradient_loss | 0.0572    |\n",
      "|    std                  | 0.318     |\n",
      "|    value_loss           | 321       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5141120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 113.25\n",
      "Num timesteps: 5151120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 108.34\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 104        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 451        |\n",
      "|    iterations           | 2520       |\n",
      "|    time_elapsed         | 11439      |\n",
      "|    total_timesteps      | 5160960    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.23288204 |\n",
      "|    clip_fraction        | 0.598      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.791     |\n",
      "|    explained_variance   | 0.804      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 107        |\n",
      "|    n_updates            | 58620      |\n",
      "|    policy_gradient_loss | -0.0246    |\n",
      "|    std                  | 0.315      |\n",
      "|    value_loss           | 380        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5161120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 103.22\n",
      "Num timesteps: 5171120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 108.80\n",
      "Num timesteps: 5181120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 115.57\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 118       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 451       |\n",
      "|    iterations           | 2530      |\n",
      "|    time_elapsed         | 11483     |\n",
      "|    total_timesteps      | 5181440   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.3788085 |\n",
      "|    clip_fraction        | 0.639     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.876    |\n",
      "|    explained_variance   | 0.643     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 63        |\n",
      "|    n_updates            | 58720     |\n",
      "|    policy_gradient_loss | -0.0136   |\n",
      "|    std                  | 0.322     |\n",
      "|    value_loss           | 267       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5191120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 102.00\n",
      "Num timesteps: 5201120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 105.68\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 104       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 451       |\n",
      "|    iterations           | 2540      |\n",
      "|    time_elapsed         | 11527     |\n",
      "|    total_timesteps      | 5201920   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.5084584 |\n",
      "|    clip_fraction        | 0.675     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.745    |\n",
      "|    explained_variance   | 0.658     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 88.2      |\n",
      "|    n_updates            | 58820     |\n",
      "|    policy_gradient_loss | -0.00799  |\n",
      "|    std                  | 0.311     |\n",
      "|    value_loss           | 293       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5211120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 123.42\n",
      "Num timesteps: 5221120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 118.17\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 127        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 451        |\n",
      "|    iterations           | 2550       |\n",
      "|    time_elapsed         | 11571      |\n",
      "|    total_timesteps      | 5222400    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.74834824 |\n",
      "|    clip_fraction        | 0.762      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.01      |\n",
      "|    explained_variance   | 0.762      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 62.9       |\n",
      "|    n_updates            | 58920      |\n",
      "|    policy_gradient_loss | 0.0526     |\n",
      "|    std                  | 0.344      |\n",
      "|    value_loss           | 204        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5231120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 113.03\n",
      "Num timesteps: 5241120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 127.86\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 127        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 451        |\n",
      "|    iterations           | 2560       |\n",
      "|    time_elapsed         | 11616      |\n",
      "|    total_timesteps      | 5242880    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.35573167 |\n",
      "|    clip_fraction        | 0.616      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.09      |\n",
      "|    explained_variance   | 0.682      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 82         |\n",
      "|    n_updates            | 59020      |\n",
      "|    policy_gradient_loss | -0.00279   |\n",
      "|    std                  | 0.35       |\n",
      "|    value_loss           | 288        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 5251120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 106.31\n",
      "Num timesteps: 5261120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 124.37\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 119        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 451        |\n",
      "|    iterations           | 2570       |\n",
      "|    time_elapsed         | 11660      |\n",
      "|    total_timesteps      | 5263360    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.17219836 |\n",
      "|    clip_fraction        | 0.568      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.09      |\n",
      "|    explained_variance   | 0.783      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 128        |\n",
      "|    n_updates            | 59120      |\n",
      "|    policy_gradient_loss | -0.0303    |\n",
      "|    std                  | 0.345      |\n",
      "|    value_loss           | 338        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5271120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 117.89\n",
      "Num timesteps: 5281120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 102.95\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 105       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 451       |\n",
      "|    iterations           | 2580      |\n",
      "|    time_elapsed         | 11704     |\n",
      "|    total_timesteps      | 5283840   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.5948355 |\n",
      "|    clip_fraction        | 0.675     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.02     |\n",
      "|    explained_variance   | 0.772     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 48.1      |\n",
      "|    n_updates            | 59220     |\n",
      "|    policy_gradient_loss | 0.00221   |\n",
      "|    std                  | 0.34      |\n",
      "|    value_loss           | 250       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5291120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 120.15\n",
      "Num timesteps: 5301120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 124.55\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 128       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 451       |\n",
      "|    iterations           | 2590      |\n",
      "|    time_elapsed         | 11748     |\n",
      "|    total_timesteps      | 5304320   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.3706809 |\n",
      "|    clip_fraction        | 0.595     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.1      |\n",
      "|    explained_variance   | 0.704     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 56.2      |\n",
      "|    n_updates            | 59320     |\n",
      "|    policy_gradient_loss | -0.0214   |\n",
      "|    std                  | 0.347     |\n",
      "|    value_loss           | 279       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5311120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 114.71\n",
      "Num timesteps: 5321120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 115.45\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 117       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 451       |\n",
      "|    iterations           | 2600      |\n",
      "|    time_elapsed         | 11792     |\n",
      "|    total_timesteps      | 5324800   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.2059978 |\n",
      "|    clip_fraction        | 0.595     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.11     |\n",
      "|    explained_variance   | 0.666     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 103       |\n",
      "|    n_updates            | 59420     |\n",
      "|    policy_gradient_loss | -0.0232   |\n",
      "|    std                  | 0.348     |\n",
      "|    value_loss           | 339       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5331120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 108.83\n",
      "Num timesteps: 5341120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 112.58\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 119        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 451        |\n",
      "|    iterations           | 2610       |\n",
      "|    time_elapsed         | 11836      |\n",
      "|    total_timesteps      | 5345280    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.24072942 |\n",
      "|    clip_fraction        | 0.599      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.972     |\n",
      "|    explained_variance   | 0.69       |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 109        |\n",
      "|    n_updates            | 59520      |\n",
      "|    policy_gradient_loss | -0.0291    |\n",
      "|    std                  | 0.333      |\n",
      "|    value_loss           | 456        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5351120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 112.41\n",
      "Num timesteps: 5361120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 115.75\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 112        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 451        |\n",
      "|    iterations           | 2620       |\n",
      "|    time_elapsed         | 11880      |\n",
      "|    total_timesteps      | 5365760    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.45091873 |\n",
      "|    clip_fraction        | 0.644      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.885     |\n",
      "|    explained_variance   | 0.758      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 49         |\n",
      "|    n_updates            | 59620      |\n",
      "|    policy_gradient_loss | -0.00794   |\n",
      "|    std                  | 0.327      |\n",
      "|    value_loss           | 276        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5371120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 99.70\n",
      "Num timesteps: 5381120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 131.43\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 126        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 451        |\n",
      "|    iterations           | 2630       |\n",
      "|    time_elapsed         | 11925      |\n",
      "|    total_timesteps      | 5386240    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.41725433 |\n",
      "|    clip_fraction        | 0.617      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.978     |\n",
      "|    explained_variance   | 0.795      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 91.2       |\n",
      "|    n_updates            | 59720      |\n",
      "|    policy_gradient_loss | -0.00629   |\n",
      "|    std                  | 0.338      |\n",
      "|    value_loss           | 244        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5391120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 119.98\n",
      "Num timesteps: 5401120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 105.12\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 111        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 451        |\n",
      "|    iterations           | 2640       |\n",
      "|    time_elapsed         | 11969      |\n",
      "|    total_timesteps      | 5406720    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.39796636 |\n",
      "|    clip_fraction        | 0.647      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.05      |\n",
      "|    explained_variance   | 0.795      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 59         |\n",
      "|    n_updates            | 59820      |\n",
      "|    policy_gradient_loss | -0.00978   |\n",
      "|    std                  | 0.343      |\n",
      "|    value_loss           | 238        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 5411120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 120.61\n",
      "Num timesteps: 5421120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 133.50\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 107        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 451        |\n",
      "|    iterations           | 2650       |\n",
      "|    time_elapsed         | 12013      |\n",
      "|    total_timesteps      | 5427200    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.39515918 |\n",
      "|    clip_fraction        | 0.648      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.06      |\n",
      "|    explained_variance   | 0.658      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 52.6       |\n",
      "|    n_updates            | 59920      |\n",
      "|    policy_gradient_loss | -0.00669   |\n",
      "|    std                  | 0.344      |\n",
      "|    value_loss           | 243        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5431120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 120.62\n",
      "Num timesteps: 5441120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 117.78\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 124       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 451       |\n",
      "|    iterations           | 2660      |\n",
      "|    time_elapsed         | 12057     |\n",
      "|    total_timesteps      | 5447680   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.8262378 |\n",
      "|    clip_fraction        | 0.678     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.03     |\n",
      "|    explained_variance   | 0.488     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 154       |\n",
      "|    n_updates            | 60020     |\n",
      "|    policy_gradient_loss | 0.0235    |\n",
      "|    std                  | 0.34      |\n",
      "|    value_loss           | 387       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5451120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 138.83\n",
      "Num timesteps: 5461120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 120.04\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 121       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 451       |\n",
      "|    iterations           | 2670      |\n",
      "|    time_elapsed         | 12101     |\n",
      "|    total_timesteps      | 5468160   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.5684738 |\n",
      "|    clip_fraction        | 0.67      |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.02     |\n",
      "|    explained_variance   | 0.766     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 69.3      |\n",
      "|    n_updates            | 60120     |\n",
      "|    policy_gradient_loss | -0.0118   |\n",
      "|    std                  | 0.337     |\n",
      "|    value_loss           | 228       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5471120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 121.14\n",
      "Num timesteps: 5481120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 126.84\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 119       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 451       |\n",
      "|    iterations           | 2680      |\n",
      "|    time_elapsed         | 12146     |\n",
      "|    total_timesteps      | 5488640   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.2849971 |\n",
      "|    clip_fraction        | 0.598     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.97     |\n",
      "|    explained_variance   | 0.734     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 123       |\n",
      "|    n_updates            | 60220     |\n",
      "|    policy_gradient_loss | -0.0182   |\n",
      "|    std                  | 0.334     |\n",
      "|    value_loss           | 369       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5491120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.62\n",
      "Num timesteps: 5501120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.84\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 116        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 451        |\n",
      "|    iterations           | 2690       |\n",
      "|    time_elapsed         | 12190      |\n",
      "|    total_timesteps      | 5509120    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.39487082 |\n",
      "|    clip_fraction        | 0.637      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.01      |\n",
      "|    explained_variance   | 0.751      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 45.4       |\n",
      "|    n_updates            | 60320      |\n",
      "|    policy_gradient_loss | -0.0126    |\n",
      "|    std                  | 0.338      |\n",
      "|    value_loss           | 208        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5511120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 112.49\n",
      "Num timesteps: 5521120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 121.37\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 119        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 451        |\n",
      "|    iterations           | 2700       |\n",
      "|    time_elapsed         | 12234      |\n",
      "|    total_timesteps      | 5529600    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.26223505 |\n",
      "|    clip_fraction        | 0.579      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.843     |\n",
      "|    explained_variance   | 0.74       |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 128        |\n",
      "|    n_updates            | 60420      |\n",
      "|    policy_gradient_loss | -0.0212    |\n",
      "|    std                  | 0.321      |\n",
      "|    value_loss           | 480        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5531120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 122.95\n",
      "Num timesteps: 5541120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 125.68\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 121        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 452        |\n",
      "|    iterations           | 2710       |\n",
      "|    time_elapsed         | 12278      |\n",
      "|    total_timesteps      | 5550080    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.62732136 |\n",
      "|    clip_fraction        | 0.672      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.9       |\n",
      "|    explained_variance   | 0.815      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 66.6       |\n",
      "|    n_updates            | 60520      |\n",
      "|    policy_gradient_loss | -0.016     |\n",
      "|    std                  | 0.328      |\n",
      "|    value_loss           | 194        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5551120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 118.17\n",
      "Num timesteps: 5561120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 141.41\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 127       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 452       |\n",
      "|    iterations           | 2720      |\n",
      "|    time_elapsed         | 12322     |\n",
      "|    total_timesteps      | 5570560   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 1.0358548 |\n",
      "|    clip_fraction        | 0.73      |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.794    |\n",
      "|    explained_variance   | 0.737     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 68.5      |\n",
      "|    n_updates            | 60620     |\n",
      "|    policy_gradient_loss | 0.0327    |\n",
      "|    std                  | 0.317     |\n",
      "|    value_loss           | 236       |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 5571120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 129.72\n",
      "Num timesteps: 5581120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 119.02\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 119        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 452        |\n",
      "|    iterations           | 2730       |\n",
      "|    time_elapsed         | 12366      |\n",
      "|    total_timesteps      | 5591040    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.89426106 |\n",
      "|    clip_fraction        | 0.74       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.785     |\n",
      "|    explained_variance   | 0.669      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 75.4       |\n",
      "|    n_updates            | 60720      |\n",
      "|    policy_gradient_loss | 0.0466     |\n",
      "|    std                  | 0.318      |\n",
      "|    value_loss           | 278        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5591120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 113.21\n",
      "Num timesteps: 5601120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 124.52\n",
      "Num timesteps: 5611120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 125.91\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 115       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 452       |\n",
      "|    iterations           | 2740      |\n",
      "|    time_elapsed         | 12411     |\n",
      "|    total_timesteps      | 5611520   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.4993099 |\n",
      "|    clip_fraction        | 0.672     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.832    |\n",
      "|    explained_variance   | 0.787     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 129       |\n",
      "|    n_updates            | 60820     |\n",
      "|    policy_gradient_loss | -0.00665  |\n",
      "|    std                  | 0.318     |\n",
      "|    value_loss           | 360       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5621120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 123.54\n",
      "Num timesteps: 5631120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 117.72\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 117        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 452        |\n",
      "|    iterations           | 2750       |\n",
      "|    time_elapsed         | 12455      |\n",
      "|    total_timesteps      | 5632000    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.98755383 |\n",
      "|    clip_fraction        | 0.681      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.825     |\n",
      "|    explained_variance   | 0.714      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 87.9       |\n",
      "|    n_updates            | 60920      |\n",
      "|    policy_gradient_loss | -0.0217    |\n",
      "|    std                  | 0.317      |\n",
      "|    value_loss           | 382        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5641120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 117.39\n",
      "Num timesteps: 5651120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 112.25\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 128        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 452        |\n",
      "|    iterations           | 2760       |\n",
      "|    time_elapsed         | 12499      |\n",
      "|    total_timesteps      | 5652480    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.38132387 |\n",
      "|    clip_fraction        | 0.672      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.872     |\n",
      "|    explained_variance   | 0.783      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 38.5       |\n",
      "|    n_updates            | 61020      |\n",
      "|    policy_gradient_loss | -0.00173   |\n",
      "|    std                  | 0.322      |\n",
      "|    value_loss           | 218        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5661120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 122.10\n",
      "Num timesteps: 5671120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 107.87\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 133        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 452        |\n",
      "|    iterations           | 2770       |\n",
      "|    time_elapsed         | 12543      |\n",
      "|    total_timesteps      | 5672960    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.49706322 |\n",
      "|    clip_fraction        | 0.681      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.03      |\n",
      "|    explained_variance   | 0.878      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 82.2       |\n",
      "|    n_updates            | 61120      |\n",
      "|    policy_gradient_loss | 0.000954   |\n",
      "|    std                  | 0.342      |\n",
      "|    value_loss           | 186        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5681120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 114.70\n",
      "Num timesteps: 5691120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.57\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 122       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 452       |\n",
      "|    iterations           | 2780      |\n",
      "|    time_elapsed         | 12587     |\n",
      "|    total_timesteps      | 5693440   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 1.1285388 |\n",
      "|    clip_fraction        | 0.609     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.922    |\n",
      "|    explained_variance   | 0.717     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 82.2      |\n",
      "|    n_updates            | 61220     |\n",
      "|    policy_gradient_loss | -0.00736  |\n",
      "|    std                  | 0.33      |\n",
      "|    value_loss           | 394       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5701120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.79\n",
      "Num timesteps: 5711120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 134.99\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 127       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 452       |\n",
      "|    iterations           | 2790      |\n",
      "|    time_elapsed         | 12632     |\n",
      "|    total_timesteps      | 5713920   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.5722415 |\n",
      "|    clip_fraction        | 0.689     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.937    |\n",
      "|    explained_variance   | 0.816     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 37.2      |\n",
      "|    n_updates            | 61320     |\n",
      "|    policy_gradient_loss | 0.00324   |\n",
      "|    std                  | 0.331     |\n",
      "|    value_loss           | 195       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5721120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.25\n",
      "Num timesteps: 5731120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 125.60\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 131        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 452        |\n",
      "|    iterations           | 2800       |\n",
      "|    time_elapsed         | 12676      |\n",
      "|    total_timesteps      | 5734400    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.56597614 |\n",
      "|    clip_fraction        | 0.66       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.951     |\n",
      "|    explained_variance   | 0.832      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 114        |\n",
      "|    n_updates            | 61420      |\n",
      "|    policy_gradient_loss | -0.014     |\n",
      "|    std                  | 0.331      |\n",
      "|    value_loss           | 290        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 5741120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.83\n",
      "Num timesteps: 5751120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 116.63\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 124        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 452        |\n",
      "|    iterations           | 2810       |\n",
      "|    time_elapsed         | 12720      |\n",
      "|    total_timesteps      | 5754880    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.37308145 |\n",
      "|    clip_fraction        | 0.646      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.996     |\n",
      "|    explained_variance   | 0.857      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 69.6       |\n",
      "|    n_updates            | 61520      |\n",
      "|    policy_gradient_loss | -0.0219    |\n",
      "|    std                  | 0.335      |\n",
      "|    value_loss           | 245        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5761120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 114.27\n",
      "Num timesteps: 5771120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 119.90\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 114       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 452       |\n",
      "|    iterations           | 2820      |\n",
      "|    time_elapsed         | 12764     |\n",
      "|    total_timesteps      | 5775360   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.4521563 |\n",
      "|    clip_fraction        | 0.649     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.996    |\n",
      "|    explained_variance   | 0.788     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 65.9      |\n",
      "|    n_updates            | 61620     |\n",
      "|    policy_gradient_loss | -0.0211   |\n",
      "|    std                  | 0.336     |\n",
      "|    value_loss           | 237       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5781120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 119.73\n",
      "Num timesteps: 5791120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 159.53\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 144       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 452       |\n",
      "|    iterations           | 2830      |\n",
      "|    time_elapsed         | 12808     |\n",
      "|    total_timesteps      | 5795840   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.6062249 |\n",
      "|    clip_fraction        | 0.676     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.907    |\n",
      "|    explained_variance   | 0.801     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 118       |\n",
      "|    n_updates            | 61720     |\n",
      "|    policy_gradient_loss | -0.0042   |\n",
      "|    std                  | 0.331     |\n",
      "|    value_loss           | 290       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5801120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 125.94\n",
      "Num timesteps: 5811120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 117.07\n",
      "--------------------------------------\n",
      "| rollout/                |          |\n",
      "|    ep_len_mean          | 30       |\n",
      "|    ep_rew_mean          | 130      |\n",
      "| time/                   |          |\n",
      "|    fps                  | 452      |\n",
      "|    iterations           | 2840     |\n",
      "|    time_elapsed         | 12852    |\n",
      "|    total_timesteps      | 5816320  |\n",
      "| train/                  |          |\n",
      "|    approx_kl            | 3.556471 |\n",
      "|    clip_fraction        | 0.751    |\n",
      "|    clip_range           | 0.2      |\n",
      "|    entropy_loss         | -1.01    |\n",
      "|    explained_variance   | 0.82     |\n",
      "|    learning_rate        | 0.001    |\n",
      "|    loss                 | 83.3     |\n",
      "|    n_updates            | 61820    |\n",
      "|    policy_gradient_loss | 0.0461   |\n",
      "|    std                  | 0.34     |\n",
      "|    value_loss           | 249      |\n",
      "--------------------------------------\n",
      "Num timesteps: 5821120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 150.97\n",
      "Num timesteps: 5831120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 144.83\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 153       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 452       |\n",
      "|    iterations           | 2850      |\n",
      "|    time_elapsed         | 12897     |\n",
      "|    total_timesteps      | 5836800   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.3741936 |\n",
      "|    clip_fraction        | 0.605     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.972    |\n",
      "|    explained_variance   | 0.83      |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 69.9      |\n",
      "|    n_updates            | 61920     |\n",
      "|    policy_gradient_loss | -0.0273   |\n",
      "|    std                  | 0.334     |\n",
      "|    value_loss           | 288       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5841120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 122.31\n",
      "Num timesteps: 5851120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.69\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 136        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 452        |\n",
      "|    iterations           | 2860       |\n",
      "|    time_elapsed         | 12941      |\n",
      "|    total_timesteps      | 5857280    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.35146272 |\n",
      "|    clip_fraction        | 0.641      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.876     |\n",
      "|    explained_variance   | 0.738      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 63.3       |\n",
      "|    n_updates            | 62020      |\n",
      "|    policy_gradient_loss | -0.0105    |\n",
      "|    std                  | 0.322      |\n",
      "|    value_loss           | 402        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5861120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 140.43\n",
      "Num timesteps: 5871120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.88\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 142        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 452        |\n",
      "|    iterations           | 2870       |\n",
      "|    time_elapsed         | 12985      |\n",
      "|    total_timesteps      | 5877760    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.47398287 |\n",
      "|    clip_fraction        | 0.684      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.796     |\n",
      "|    explained_variance   | 0.818      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 78.7       |\n",
      "|    n_updates            | 62120      |\n",
      "|    policy_gradient_loss | 0.0135     |\n",
      "|    std                  | 0.314      |\n",
      "|    value_loss           | 282        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5881120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 139.19\n",
      "Num timesteps: 5891120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 139.18\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 128        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 452        |\n",
      "|    iterations           | 2880       |\n",
      "|    time_elapsed         | 13029      |\n",
      "|    total_timesteps      | 5898240    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.41482556 |\n",
      "|    clip_fraction        | 0.66       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.882     |\n",
      "|    explained_variance   | 0.813      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 86         |\n",
      "|    n_updates            | 62220      |\n",
      "|    policy_gradient_loss | -0.0132    |\n",
      "|    std                  | 0.324      |\n",
      "|    value_loss           | 287        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 5901120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 124.87\n",
      "Num timesteps: 5911120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.88\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 143       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 452       |\n",
      "|    iterations           | 2890      |\n",
      "|    time_elapsed         | 13073     |\n",
      "|    total_timesteps      | 5918720   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 1.7511499 |\n",
      "|    clip_fraction        | 0.772     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.86     |\n",
      "|    explained_variance   | 0.88      |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 52        |\n",
      "|    n_updates            | 62320     |\n",
      "|    policy_gradient_loss | 0.034     |\n",
      "|    std                  | 0.327     |\n",
      "|    value_loss           | 173       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5921120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 125.44\n",
      "Num timesteps: 5931120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.07\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 143       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 452       |\n",
      "|    iterations           | 2900      |\n",
      "|    time_elapsed         | 13118     |\n",
      "|    total_timesteps      | 5939200   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.3337309 |\n",
      "|    clip_fraction        | 0.649     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.938    |\n",
      "|    explained_variance   | 0.851     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 59.6      |\n",
      "|    n_updates            | 62420     |\n",
      "|    policy_gradient_loss | -0.0216   |\n",
      "|    std                  | 0.332     |\n",
      "|    value_loss           | 181       |\n",
      "---------------------------------------\n",
      "Num timesteps: 5941120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.75\n",
      "Num timesteps: 5951120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 121.36\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 139        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 452        |\n",
      "|    iterations           | 2910       |\n",
      "|    time_elapsed         | 13162      |\n",
      "|    total_timesteps      | 5959680    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.41022834 |\n",
      "|    clip_fraction        | 0.655      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.94      |\n",
      "|    explained_variance   | 0.838      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 112        |\n",
      "|    n_updates            | 62520      |\n",
      "|    policy_gradient_loss | -0.00538   |\n",
      "|    std                  | 0.327      |\n",
      "|    value_loss           | 232        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5961120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 133.88\n",
      "Num timesteps: 5971120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 123.55\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 133        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 452        |\n",
      "|    iterations           | 2920       |\n",
      "|    time_elapsed         | 13206      |\n",
      "|    total_timesteps      | 5980160    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.23553863 |\n",
      "|    clip_fraction        | 0.59       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.943     |\n",
      "|    explained_variance   | 0.799      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 58.2       |\n",
      "|    n_updates            | 62620      |\n",
      "|    policy_gradient_loss | -0.0231    |\n",
      "|    std                  | 0.331      |\n",
      "|    value_loss           | 266        |\n",
      "----------------------------------------\n",
      "Num timesteps: 5981120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 125.98\n",
      "Num timesteps: 5991120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.16\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 135        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 452        |\n",
      "|    iterations           | 2930       |\n",
      "|    time_elapsed         | 13250      |\n",
      "|    total_timesteps      | 6000640    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.34143752 |\n",
      "|    clip_fraction        | 0.605      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.03      |\n",
      "|    explained_variance   | 0.819      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 88.7       |\n",
      "|    n_updates            | 62720      |\n",
      "|    policy_gradient_loss | -0.0264    |\n",
      "|    std                  | 0.338      |\n",
      "|    value_loss           | 335        |\n",
      "----------------------------------------\n",
      "Num timesteps: 6001120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 128.16\n",
      "Num timesteps: 6011120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 132.66\n",
      "Num timesteps: 6021120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 135.81\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 136        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 452        |\n",
      "|    iterations           | 2940       |\n",
      "|    time_elapsed         | 13294      |\n",
      "|    total_timesteps      | 6021120    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.38261035 |\n",
      "|    clip_fraction        | 0.662      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.956     |\n",
      "|    explained_variance   | 0.767      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 65         |\n",
      "|    n_updates            | 62820      |\n",
      "|    policy_gradient_loss | -0.0114    |\n",
      "|    std                  | 0.334      |\n",
      "|    value_loss           | 243        |\n",
      "----------------------------------------\n",
      "Num timesteps: 6031120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 138.26\n",
      "Num timesteps: 6041120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 137.14\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 140        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 452        |\n",
      "|    iterations           | 2950       |\n",
      "|    time_elapsed         | 13339      |\n",
      "|    total_timesteps      | 6041600    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.39586097 |\n",
      "|    clip_fraction        | 0.665      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.933     |\n",
      "|    explained_variance   | 0.711      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 286        |\n",
      "|    n_updates            | 62920      |\n",
      "|    policy_gradient_loss | -0.0004    |\n",
      "|    std                  | 0.332      |\n",
      "|    value_loss           | 317        |\n",
      "----------------------------------------\n",
      "Num timesteps: 6051120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.36\n",
      "Num timesteps: 6061120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.67\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 129       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 452       |\n",
      "|    iterations           | 2960      |\n",
      "|    time_elapsed         | 13383     |\n",
      "|    total_timesteps      | 6062080   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.8544436 |\n",
      "|    clip_fraction        | 0.728     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.996    |\n",
      "|    explained_variance   | 0.75      |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 69.3      |\n",
      "|    n_updates            | 63020     |\n",
      "|    policy_gradient_loss | 0.0178    |\n",
      "|    std                  | 0.339     |\n",
      "|    value_loss           | 266       |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 6071120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 125.48\n",
      "Num timesteps: 6081120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 143.42\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 141        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 452        |\n",
      "|    iterations           | 2970       |\n",
      "|    time_elapsed         | 13427      |\n",
      "|    total_timesteps      | 6082560    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.40532106 |\n",
      "|    clip_fraction        | 0.644      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.03      |\n",
      "|    explained_variance   | 0.709      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 95.2       |\n",
      "|    n_updates            | 63120      |\n",
      "|    policy_gradient_loss | -0.024     |\n",
      "|    std                  | 0.34       |\n",
      "|    value_loss           | 288        |\n",
      "----------------------------------------\n",
      "Num timesteps: 6091120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 122.06\n",
      "Num timesteps: 6101120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 140.63\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 134       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 453       |\n",
      "|    iterations           | 2980      |\n",
      "|    time_elapsed         | 13471     |\n",
      "|    total_timesteps      | 6103040   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.4031201 |\n",
      "|    clip_fraction        | 0.656     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.845    |\n",
      "|    explained_variance   | 0.71      |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 83.3      |\n",
      "|    n_updates            | 63220     |\n",
      "|    policy_gradient_loss | -0.00363  |\n",
      "|    std                  | 0.321     |\n",
      "|    value_loss           | 378       |\n",
      "---------------------------------------\n",
      "Num timesteps: 6111120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 129.20\n",
      "Num timesteps: 6121120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 138.17\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 136        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 453        |\n",
      "|    iterations           | 2990       |\n",
      "|    time_elapsed         | 13516      |\n",
      "|    total_timesteps      | 6123520    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.30636546 |\n",
      "|    clip_fraction        | 0.636      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.957     |\n",
      "|    explained_variance   | 0.776      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 97.1       |\n",
      "|    n_updates            | 63320      |\n",
      "|    policy_gradient_loss | -0.0141    |\n",
      "|    std                  | 0.333      |\n",
      "|    value_loss           | 332        |\n",
      "----------------------------------------\n",
      "Num timesteps: 6131120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 131.51\n",
      "Num timesteps: 6141120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 119.50\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 140        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 453        |\n",
      "|    iterations           | 3000       |\n",
      "|    time_elapsed         | 13560      |\n",
      "|    total_timesteps      | 6144000    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.41404215 |\n",
      "|    clip_fraction        | 0.69       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.89      |\n",
      "|    explained_variance   | 0.851      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 86.6       |\n",
      "|    n_updates            | 63420      |\n",
      "|    policy_gradient_loss | -0.00525   |\n",
      "|    std                  | 0.324      |\n",
      "|    value_loss           | 237        |\n",
      "----------------------------------------\n",
      "Num timesteps: 6151120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 117.31\n",
      "Num timesteps: 6161120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 138.42\n",
      "--------------------------------------\n",
      "| rollout/                |          |\n",
      "|    ep_len_mean          | 30       |\n",
      "|    ep_rew_mean          | 144      |\n",
      "| time/                   |          |\n",
      "|    fps                  | 453      |\n",
      "|    iterations           | 3010     |\n",
      "|    time_elapsed         | 13604    |\n",
      "|    total_timesteps      | 6164480  |\n",
      "| train/                  |          |\n",
      "|    approx_kl            | 16.08535 |\n",
      "|    clip_fraction        | 0.793    |\n",
      "|    clip_range           | 0.2      |\n",
      "|    entropy_loss         | -0.989   |\n",
      "|    explained_variance   | 0.814    |\n",
      "|    learning_rate        | 0.001    |\n",
      "|    loss                 | 51       |\n",
      "|    n_updates            | 63520    |\n",
      "|    policy_gradient_loss | 0.0959   |\n",
      "|    std                  | 0.338    |\n",
      "|    value_loss           | 200      |\n",
      "--------------------------------------\n",
      "Num timesteps: 6171120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 136.72\n",
      "Num timesteps: 6181120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 123.38\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 131       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 453       |\n",
      "|    iterations           | 3020      |\n",
      "|    time_elapsed         | 13648     |\n",
      "|    total_timesteps      | 6184960   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.3538341 |\n",
      "|    clip_fraction        | 0.608     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.05     |\n",
      "|    explained_variance   | 0.706     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 69.1      |\n",
      "|    n_updates            | 63620     |\n",
      "|    policy_gradient_loss | -0.0313   |\n",
      "|    std                  | 0.34      |\n",
      "|    value_loss           | 328       |\n",
      "---------------------------------------\n",
      "Num timesteps: 6191120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.41\n",
      "Num timesteps: 6201120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 142.69\n",
      "--------------------------------------\n",
      "| rollout/                |          |\n",
      "|    ep_len_mean          | 30       |\n",
      "|    ep_rew_mean          | 144      |\n",
      "| time/                   |          |\n",
      "|    fps                  | 453      |\n",
      "|    iterations           | 3030     |\n",
      "|    time_elapsed         | 13692    |\n",
      "|    total_timesteps      | 6205440  |\n",
      "| train/                  |          |\n",
      "|    approx_kl            | 1.175431 |\n",
      "|    clip_fraction        | 0.659    |\n",
      "|    clip_range           | 0.2      |\n",
      "|    entropy_loss         | -1.04    |\n",
      "|    explained_variance   | 0.828    |\n",
      "|    learning_rate        | 0.001    |\n",
      "|    loss                 | 73.8     |\n",
      "|    n_updates            | 63720    |\n",
      "|    policy_gradient_loss | 0.00219  |\n",
      "|    std                  | 0.341    |\n",
      "|    value_loss           | 233      |\n",
      "--------------------------------------\n",
      "Num timesteps: 6211120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 129.49\n",
      "Num timesteps: 6221120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 122.15\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 135        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 453        |\n",
      "|    iterations           | 3040       |\n",
      "|    time_elapsed         | 13736      |\n",
      "|    total_timesteps      | 6225920    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.31549138 |\n",
      "|    clip_fraction        | 0.603      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.974     |\n",
      "|    explained_variance   | 0.741      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 72.8       |\n",
      "|    n_updates            | 63820      |\n",
      "|    policy_gradient_loss | -0.0193    |\n",
      "|    std                  | 0.333      |\n",
      "|    value_loss           | 269        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num timesteps: 6231120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 147.53\n",
      "Num timesteps: 6241120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 130.18\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 146       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 453       |\n",
      "|    iterations           | 3050      |\n",
      "|    time_elapsed         | 13781     |\n",
      "|    total_timesteps      | 6246400   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.7345991 |\n",
      "|    clip_fraction        | 0.656     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.07     |\n",
      "|    explained_variance   | 0.725     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 73.1      |\n",
      "|    n_updates            | 63920     |\n",
      "|    policy_gradient_loss | -0.0137   |\n",
      "|    std                  | 0.343     |\n",
      "|    value_loss           | 272       |\n",
      "---------------------------------------\n",
      "Num timesteps: 6251120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 133.55\n",
      "Num timesteps: 6261120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 124.64\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 135        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 453        |\n",
      "|    iterations           | 3060       |\n",
      "|    time_elapsed         | 13825      |\n",
      "|    total_timesteps      | 6266880    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.22254404 |\n",
      "|    clip_fraction        | 0.573      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.06      |\n",
      "|    explained_variance   | 0.697      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 158        |\n",
      "|    n_updates            | 64020      |\n",
      "|    policy_gradient_loss | -0.0361    |\n",
      "|    std                  | 0.344      |\n",
      "|    value_loss           | 410        |\n",
      "----------------------------------------\n",
      "Num timesteps: 6271120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 140.77\n",
      "Num timesteps: 6281120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 123.98\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 145        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 453        |\n",
      "|    iterations           | 3070       |\n",
      "|    time_elapsed         | 13869      |\n",
      "|    total_timesteps      | 6287360    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.27705795 |\n",
      "|    clip_fraction        | 0.583      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.973     |\n",
      "|    explained_variance   | 0.753      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 167        |\n",
      "|    n_updates            | 64120      |\n",
      "|    policy_gradient_loss | -0.0166    |\n",
      "|    std                  | 0.332      |\n",
      "|    value_loss           | 418        |\n",
      "----------------------------------------\n",
      "Num timesteps: 6291120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 139.40\n",
      "Num timesteps: 6301120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 133.80\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 137        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 453        |\n",
      "|    iterations           | 3080       |\n",
      "|    time_elapsed         | 13913      |\n",
      "|    total_timesteps      | 6307840    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.24394783 |\n",
      "|    clip_fraction        | 0.575      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.07      |\n",
      "|    explained_variance   | 0.757      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 69.6       |\n",
      "|    n_updates            | 64220      |\n",
      "|    policy_gradient_loss | -0.0283    |\n",
      "|    std                  | 0.344      |\n",
      "|    value_loss           | 398        |\n",
      "----------------------------------------\n",
      "Num timesteps: 6311120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 126.24\n",
      "Num timesteps: 6321120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 126.44\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 30        |\n",
      "|    ep_rew_mean          | 121       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 453       |\n",
      "|    iterations           | 3090      |\n",
      "|    time_elapsed         | 13958     |\n",
      "|    total_timesteps      | 6328320   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.5190901 |\n",
      "|    clip_fraction        | 0.672     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -1.21     |\n",
      "|    explained_variance   | 0.779     |\n",
      "|    learning_rate        | 0.001     |\n",
      "|    loss                 | 50.4      |\n",
      "|    n_updates            | 64320     |\n",
      "|    policy_gradient_loss | 0.00153   |\n",
      "|    std                  | 0.36      |\n",
      "|    value_loss           | 233       |\n",
      "---------------------------------------\n",
      "Num timesteps: 6331120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 140.99\n",
      "Num timesteps: 6341120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 124.49\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 128        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 453        |\n",
      "|    iterations           | 3100       |\n",
      "|    time_elapsed         | 14002      |\n",
      "|    total_timesteps      | 6348800    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.23577726 |\n",
      "|    clip_fraction        | 0.612      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.12      |\n",
      "|    explained_variance   | 0.74       |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 127        |\n",
      "|    n_updates            | 64420      |\n",
      "|    policy_gradient_loss | -0.0243    |\n",
      "|    std                  | 0.349      |\n",
      "|    value_loss           | 371        |\n",
      "----------------------------------------\n",
      "Num timesteps: 6351120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 113.86\n",
      "Num timesteps: 6361120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 126.44\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 30         |\n",
      "|    ep_rew_mean          | 132        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 453        |\n",
      "|    iterations           | 3110       |\n",
      "|    time_elapsed         | 14047      |\n",
      "|    total_timesteps      | 6369280    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.32181036 |\n",
      "|    clip_fraction        | 0.658      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.19      |\n",
      "|    explained_variance   | 0.803      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 149        |\n",
      "|    n_updates            | 64520      |\n",
      "|    policy_gradient_loss | 0.00232    |\n",
      "|    std                  | 0.36       |\n",
      "|    value_loss           | 295        |\n",
      "----------------------------------------\n",
      "Num timesteps: 6371120\n",
      "Best mean reward: 191.41 - Last mean reward per episode: 127.55\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-253-1c598d8c83c1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5e7\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlog_interval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.pyenv/versions/3.7.3/envs/venv-884/lib/python3.7/site-packages/stable_baselines3/ppo/ppo.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, eval_env, eval_freq, n_eval_episodes, tb_log_name, eval_log_path, reset_num_timesteps)\u001b[0m\n\u001b[1;32m    287\u001b[0m             \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    288\u001b[0m             \u001b[0meval_log_path\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0meval_log_path\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 289\u001b[0;31m             \u001b[0mreset_num_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreset_num_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    290\u001b[0m         )\n",
      "\u001b[0;32m~/.pyenv/versions/3.7.3/envs/venv-884/lib/python3.7/site-packages/stable_baselines3/common/on_policy_algorithm.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, eval_env, eval_freq, n_eval_episodes, tb_log_name, eval_log_path, reset_num_timesteps)\u001b[0m\n\u001b[1;32m    225\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_timesteps\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    226\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 227\u001b[0;31m             \u001b[0mcontinue_training\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollect_rollouts\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrollout_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_rollout_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_steps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    228\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    229\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mcontinue_training\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.pyenv/versions/3.7.3/envs/venv-884/lib/python3.7/site-packages/stable_baselines3/common/on_policy_algorithm.py\u001b[0m in \u001b[0;36mcollect_rollouts\u001b[0;34m(self, env, callback, rollout_buffer, n_rollout_steps)\u001b[0m\n\u001b[1;32m    166\u001b[0m                 \u001b[0mclipped_actions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mactions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhigh\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 168\u001b[0;31m             \u001b[0mnew_obs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrewards\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdones\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minfos\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclipped_actions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    169\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    170\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_timesteps\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_envs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.pyenv/versions/3.7.3/envs/venv-884/lib/python3.7/site-packages/stable_baselines3/common/vec_env/base_vec_env.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, actions)\u001b[0m\n\u001b[1;32m    161\u001b[0m         \"\"\"\n\u001b[1;32m    162\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mactions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 163\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    164\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    165\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_images\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mSequence\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.pyenv/versions/3.7.3/envs/venv-884/lib/python3.7/site-packages/stable_baselines3/common/vec_env/dummy_vec_env.py\u001b[0m in \u001b[0;36mstep_wait\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     42\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0menv_idx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_envs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m             obs, self.buf_rews[env_idx], self.buf_dones[env_idx], self.buf_infos[env_idx] = self.envs[env_idx].step(\n\u001b[0;32m---> 44\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0menv_idx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m             )\n\u001b[1;32m     46\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuf_dones\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0menv_idx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.pyenv/versions/3.7.3/envs/venv-884/lib/python3.7/site-packages/stable_baselines3/common/monitor.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, action)\u001b[0m\n\u001b[1;32m     95\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mneeds_reset\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Tried to step environment that needs reset\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 97\u001b[0;31m         \u001b[0mobservation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minfo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     98\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrewards\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreward\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-215-1c7823c8f189>\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, action_)\u001b[0m\n\u001b[1;32m     80\u001b[0m         \u001b[0mdone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnet_worth\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msteps\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepisode_length\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     81\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 82\u001b[0;31m         \u001b[0mobs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_obs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstock_obs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbalance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mholdings\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     83\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcum_rew\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mrew\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-215-1c7823c8f189>\u001b[0m in \u001b[0;36mget_obs\u001b[0;34m(self, stock_obs, balance, holdings, index)\u001b[0m\n\u001b[1;32m     99\u001b[0m         \u001b[0mix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mindex\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mtic\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtickers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m             \u001b[0mfeature_vals\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeature_vals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtic\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mix\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.pyenv/versions/3.7.3/envs/venv-884/lib/python3.7/site-packages/pandas/core/series.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    875\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    876\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 877\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    878\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    879\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_get_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.pyenv/versions/3.7.3/envs/venv-884/lib/python3.7/site-packages/pandas/core/series.py\u001b[0m in \u001b[0;36m_get_with\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    915\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    916\u001b[0m         \u001b[0;31m# handle the dup indexing case GH#4246\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 917\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    918\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    919\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_get_values_tuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.pyenv/versions/3.7.3/envs/venv-884/lib/python3.7/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    893\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m             \u001b[0mmaybe_callable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_if_callable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 895\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmaybe_callable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    896\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_is_scalar_access\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.pyenv/versions/3.7.3/envs/venv-884/lib/python3.7/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_getitem_axis\u001b[0;34m(self, key, axis)\u001b[0m\n\u001b[1;32m   1111\u001b[0m                     \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Cannot index with multidimensional key\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1112\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1113\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_iterable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1114\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1115\u001b[0m             \u001b[0;31m# nested tuple slicing\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.pyenv/versions/3.7.3/envs/venv-884/lib/python3.7/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_getitem_iterable\u001b[0;34m(self, key, axis)\u001b[0m\n\u001b[1;32m   1051\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1052\u001b[0m         \u001b[0;31m# A collection of keys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1053\u001b[0;31m         \u001b[0mkeyarr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_listlike_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mraise_missing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1054\u001b[0m         return self.obj._reindex_with_indexers(\n\u001b[1;32m   1055\u001b[0m             \u001b[0;34m{\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mkeyarr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mallow_dups\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.pyenv/versions/3.7.3/envs/venv-884/lib/python3.7/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_get_listlike_indexer\u001b[0;34m(self, key, axis, raise_missing)\u001b[0m\n\u001b[1;32m   1260\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_index_as_unique\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1261\u001b[0m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_indexer_for\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkeyarr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1262\u001b[0;31m             \u001b[0mkeyarr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreindex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkeyarr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1263\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1264\u001b[0m             \u001b[0mkeyarr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_indexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reindex_non_unique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkeyarr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.pyenv/versions/3.7.3/envs/venv-884/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mreindex\u001b[0;34m(self, target, method, level, limit, tolerance)\u001b[0m\n\u001b[1;32m   3501\u001b[0m             \u001b[0mtarget\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3502\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3503\u001b[0;31m             \u001b[0mtarget\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mensure_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3504\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3505\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlevel\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.pyenv/versions/3.7.3/envs/venv-884/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mensure_index\u001b[0;34m(index_like, copy)\u001b[0m\n\u001b[1;32m   5915\u001b[0m             \u001b[0mindex_like\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcopy_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex_like\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5916\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5917\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mIndex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex_like\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5918\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5919\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.pyenv/versions/3.7.3/envs/venv-884/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36m__new__\u001b[0;34m(cls, data, dtype, copy, name, tupleize_cols, **kwargs)\u001b[0m\n\u001b[1;32m    252\u001b[0m     ) -> \"Index\":\n\u001b[1;32m    253\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 254\u001b[0;31m         \u001b[0;32mfrom\u001b[0m \u001b[0mpandas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindexes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrange\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mRangeIndex\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    255\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    256\u001b[0m         \u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmaybe_extract_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#Learn\n",
    "model.learn(total_timesteps=int(5e7), callback = callback, log_interval = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "083e5e03",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}