Cartpole_baselines.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4d8f2f76",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import gym\n",
    "from gym.wrappers import Monitor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c4259290",
   "metadata": {},
   "outputs": [],
   "source": [
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "from stable_baselines3.common.evaluation import evaluate_policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "607471d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "env_name='CartPole-v0'\n",
    "env=Monitor(gym.make(env_name),'./',force=True)\n",
    "env=DummyVecEnv([lambda:env])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8f130553",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_env(env, episodes, random=True, model=\"\"):\n",
    "    total_rewards=[]\n",
    "    if random:\n",
    "        for _ in range(episodes):\n",
    "            ep_rewards=0\n",
    "            env.reset()\n",
    "            while True:\n",
    "                action=env.action_space.sample()\n",
    "                next_state, reward,done, info=env.step(action)\n",
    "                ep_rewards+=reward\n",
    "                if done:\n",
    "                    total_rewards.append(ep_rewards)\n",
    "                    break\n",
    "    else:\n",
    "        if model==\"\":\n",
    "            print('Please enter the agent model')\n",
    "            \n",
    "        else:\n",
    "            for _ in range(episodes):\n",
    "                ep_rewards=0\n",
    "                state=env.reset()\n",
    "                while True:\n",
    "                    action,_=model.predict(state)\n",
    "                    next_state, reward,done, info=env.step(action)\n",
    "                    ep_rewards+=reward\n",
    "                    state=next_state\n",
    "                    if done:\n",
    "                        total_rewards.append(ep_rewards)\n",
    "                        break\n",
    "        return total_rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "54b6398a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_experiment(env_name,env,random=True, episodes=5, model=\"\"):\n",
    "    render_path=os.path.join(env_name,'render','base_performance') if random else os.path.join(env_name,'render','model_performance')\n",
    "    env=Monitor(gym.make(env_name),render_path,force=True)\n",
    "    if random:\n",
    "        total_rewards=run_env(env, 5)\n",
    "    else:\n",
    "        total_rewards=run_env(env,episodes=episodes, random=False,model=model)\n",
    "    return env, total_rewards"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44b52ac7",
   "metadata": {},
   "source": [
    "### Model Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0102114e",
   "metadata": {},
   "outputs": [],
   "source": [
    "log_path=os.path.join(env_name,\"PPO\",\"logs\")\n",
    "print(log_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ccbef4e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cpu device\n"
     ]
    }
   ],
   "source": [
    "model=PPO(\"MlpPolicy\",env,verbose=1,tensorboard_log=log_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7e1ccbbe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logging to CartPole-v0\\PPO\\logs\\PPO_3\n",
      "-----------------------------\n",
      "| time/              |      |\n",
      "|    fps             | 719  |\n",
      "|    iterations      | 1    |\n",
      "|    time_elapsed    | 2    |\n",
      "|    total_timesteps | 2048 |\n",
      "-----------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 825         |\n",
      "|    iterations           | 2           |\n",
      "|    time_elapsed         | 4           |\n",
      "|    total_timesteps      | 4096        |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.009532657 |\n",
      "|    clip_fraction        | 0.102       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.686      |\n",
      "|    explained_variance   | -0.00303    |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 5.86        |\n",
      "|    n_updates            | 10          |\n",
      "|    policy_gradient_loss | -0.0152     |\n",
      "|    value_loss           | 45          |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 898         |\n",
      "|    iterations           | 3           |\n",
      "|    time_elapsed         | 6           |\n",
      "|    total_timesteps      | 6144        |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.009648826 |\n",
      "|    clip_fraction        | 0.0677      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.669      |\n",
      "|    explained_variance   | 0.0943      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 16.5        |\n",
      "|    n_updates            | 20          |\n",
      "|    policy_gradient_loss | -0.0204     |\n",
      "|    value_loss           | 40.6        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 850         |\n",
      "|    iterations           | 4           |\n",
      "|    time_elapsed         | 9           |\n",
      "|    total_timesteps      | 8192        |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.007982429 |\n",
      "|    clip_fraction        | 0.0776      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.636      |\n",
      "|    explained_variance   | 0.229       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 23.8        |\n",
      "|    n_updates            | 30          |\n",
      "|    policy_gradient_loss | -0.0183     |\n",
      "|    value_loss           | 52.5        |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 895          |\n",
      "|    iterations           | 5            |\n",
      "|    time_elapsed         | 11           |\n",
      "|    total_timesteps      | 10240        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0076353545 |\n",
      "|    clip_fraction        | 0.0681       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.608       |\n",
      "|    explained_variance   | 0.26         |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 27           |\n",
      "|    n_updates            | 40           |\n",
      "|    policy_gradient_loss | -0.016       |\n",
      "|    value_loss           | 66.4         |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 921          |\n",
      "|    iterations           | 6            |\n",
      "|    time_elapsed         | 13           |\n",
      "|    total_timesteps      | 12288        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0100845685 |\n",
      "|    clip_fraction        | 0.096        |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.584       |\n",
      "|    explained_variance   | 0.383        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 29.6         |\n",
      "|    n_updates            | 50           |\n",
      "|    policy_gradient_loss | -0.0157      |\n",
      "|    value_loss           | 64.6         |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 947          |\n",
      "|    iterations           | 7            |\n",
      "|    time_elapsed         | 15           |\n",
      "|    total_timesteps      | 14336        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0058560087 |\n",
      "|    clip_fraction        | 0.0665       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.578       |\n",
      "|    explained_variance   | 0.549        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 29.5         |\n",
      "|    n_updates            | 60           |\n",
      "|    policy_gradient_loss | -0.0119      |\n",
      "|    value_loss           | 57.4         |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 968         |\n",
      "|    iterations           | 8           |\n",
      "|    time_elapsed         | 16          |\n",
      "|    total_timesteps      | 16384       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.005947152 |\n",
      "|    clip_fraction        | 0.0413      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.555      |\n",
      "|    explained_variance   | 0.66        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 16.6        |\n",
      "|    n_updates            | 70          |\n",
      "|    policy_gradient_loss | -0.00675    |\n",
      "|    value_loss           | 39.7        |\n",
      "-----------------------------------------\n",
      "---------------------------------------\n",
      "| time/                   |           |\n",
      "|    fps                  | 981       |\n",
      "|    iterations           | 9         |\n",
      "|    time_elapsed         | 18        |\n",
      "|    total_timesteps      | 18432     |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.0068229 |\n",
      "|    clip_fraction        | 0.073     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.563    |\n",
      "|    explained_variance   | 0.928     |\n",
      "|    learning_rate        | 0.0003    |\n",
      "|    loss                 | 1.79      |\n",
      "|    n_updates            | 80        |\n",
      "|    policy_gradient_loss | -0.0112   |\n",
      "|    value_loss           | 16.5      |\n",
      "---------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 990          |\n",
      "|    iterations           | 10           |\n",
      "|    time_elapsed         | 20           |\n",
      "|    total_timesteps      | 20480        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0047639776 |\n",
      "|    clip_fraction        | 0.0197       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.547       |\n",
      "|    explained_variance   | 0.884        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.53         |\n",
      "|    n_updates            | 90           |\n",
      "|    policy_gradient_loss | -0.00155     |\n",
      "|    value_loss           | 14.1         |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 1001         |\n",
      "|    iterations           | 11           |\n",
      "|    time_elapsed         | 22           |\n",
      "|    total_timesteps      | 22528        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0056492407 |\n",
      "|    clip_fraction        | 0.0249       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.536       |\n",
      "|    explained_variance   | 0.893        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 3.85         |\n",
      "|    n_updates            | 100          |\n",
      "|    policy_gradient_loss | -0.0067      |\n",
      "|    value_loss           | 18.2         |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 1008        |\n",
      "|    iterations           | 12          |\n",
      "|    time_elapsed         | 24          |\n",
      "|    total_timesteps      | 24576       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.008848859 |\n",
      "|    clip_fraction        | 0.064       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.532      |\n",
      "|    explained_variance   | 0.917       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.487       |\n",
      "|    n_updates            | 110         |\n",
      "|    policy_gradient_loss | -0.00516    |\n",
      "|    value_loss           | 5.51        |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 902         |\n",
      "|    iterations           | 13          |\n",
      "|    time_elapsed         | 29          |\n",
      "|    total_timesteps      | 26624       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.004110072 |\n",
      "|    clip_fraction        | 0.0343      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.535      |\n",
      "|    explained_variance   | 0.912       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.961       |\n",
      "|    n_updates            | 120         |\n",
      "|    policy_gradient_loss | -0.00804    |\n",
      "|    value_loss           | 5.55        |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 913          |\n",
      "|    iterations           | 14           |\n",
      "|    time_elapsed         | 31           |\n",
      "|    total_timesteps      | 28672        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0033406718 |\n",
      "|    clip_fraction        | 0.0193       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.525       |\n",
      "|    explained_variance   | 0.166        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 0.157        |\n",
      "|    n_updates            | 130          |\n",
      "|    policy_gradient_loss | -0.00238     |\n",
      "|    value_loss           | 2.46         |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 924         |\n",
      "|    iterations           | 15          |\n",
      "|    time_elapsed         | 33          |\n",
      "|    total_timesteps      | 30720       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.002979925 |\n",
      "|    clip_fraction        | 0.0174      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.523      |\n",
      "|    explained_variance   | 0.00231     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.201       |\n",
      "|    n_updates            | 140         |\n",
      "|    policy_gradient_loss | -0.000585   |\n",
      "|    value_loss           | 1.63        |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 934          |\n",
      "|    iterations           | 16           |\n",
      "|    time_elapsed         | 35           |\n",
      "|    total_timesteps      | 32768        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0025089094 |\n",
      "|    clip_fraction        | 0.0162       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.494       |\n",
      "|    explained_variance   | 0.123        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 0.0481       |\n",
      "|    n_updates            | 150          |\n",
      "|    policy_gradient_loss | -0.00196     |\n",
      "|    value_loss           | 1.02         |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 943         |\n",
      "|    iterations           | 17          |\n",
      "|    time_elapsed         | 36          |\n",
      "|    total_timesteps      | 34816       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.006226251 |\n",
      "|    clip_fraction        | 0.0468      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.491      |\n",
      "|    explained_variance   | 0.295       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.155       |\n",
      "|    n_updates            | 160         |\n",
      "|    policy_gradient_loss | -0.00338    |\n",
      "|    value_loss           | 0.678       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 950         |\n",
      "|    iterations           | 18          |\n",
      "|    time_elapsed         | 38          |\n",
      "|    total_timesteps      | 36864       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.008445769 |\n",
      "|    clip_fraction        | 0.0709      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.502      |\n",
      "|    explained_variance   | 0.116       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0778      |\n",
      "|    n_updates            | 170         |\n",
      "|    policy_gradient_loss | -0.00483    |\n",
      "|    value_loss           | 0.414       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 957         |\n",
      "|    iterations           | 19          |\n",
      "|    time_elapsed         | 40          |\n",
      "|    total_timesteps      | 38912       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.007361469 |\n",
      "|    clip_fraction        | 0.0671      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.504      |\n",
      "|    explained_variance   | 0.216       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0372      |\n",
      "|    n_updates            | 180         |\n",
      "|    policy_gradient_loss | -0.00739    |\n",
      "|    value_loss           | 0.277       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 964         |\n",
      "|    iterations           | 20          |\n",
      "|    time_elapsed         | 42          |\n",
      "|    total_timesteps      | 40960       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.007096479 |\n",
      "|    clip_fraction        | 0.0713      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.506      |\n",
      "|    explained_variance   | 0.297       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.038       |\n",
      "|    n_updates            | 190         |\n",
      "|    policy_gradient_loss | -0.00592    |\n",
      "|    value_loss           | 0.175       |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x1759060ab38>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.learn(40000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "78aa1601",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'CartPole-v0\\\\PPO\\\\models'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_path=os.path.join(env_name,\"PPO\",\"models\")\n",
    "model_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "41f2fb04",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_4688\\3192530043.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel_path\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": [
    "model.save(model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e685e9f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be5b13b5",
   "metadata": {},
   "source": [
    "### Evaluation of Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "407fd5ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "env_name='CartPole-v0'\n",
    "env=Monitor(gym.make(env_name),'./',force=True)\n",
    "env=DummyVecEnv([lambda:env])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "390d363b",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path=os.path.join(env_name,\"PPO\",\"models\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0d79104e",
   "metadata": {},
   "outputs": [],
   "source": [
    "model=PPO.load(model_path, env)\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "25276cbc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\ai_gym\\lib\\site-packages\\stable_baselines3\\common\\evaluation.py:69: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  UserWarning,\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(200.0, 0.0)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_policy(model,env,render=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6af17c22",
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fab9874",
   "metadata": {},
   "outputs": [],
   "source": [
    "del model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee550936",
   "metadata": {},
   "source": [
    "### Test model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbf34276",
   "metadata": {},
   "outputs": [],
   "source": [
    "env_name='CartPole-v0'\n",
    "env=Monitor(gym.make(env_name),'./',force=True)\n",
    "env=DummyVecEnv([lambda:env])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9abef2cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path=os.path.join(env_name,\"PPO\",\"models\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2853de5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "model=PPO.load(model_path, env)\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1bbc3b9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "_, total_rewards = run_experiment(env_name,env,random=False, episodes=5, model=model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d93bbd1d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[200.0, 200.0, 200.0, 200.0, 200.0]\n"
     ]
    }
   ],
   "source": [
    "print(total_rewards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fca5e3dd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}