3a_Breakout-v0_final_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7d1fa20c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys\n",
    "import gc\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98159a81",
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.append('../')\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c8c767df",
   "metadata": {},
   "outputs": [],
   "source": [
    "from get_algos import get_all_algos , run_experiment ,create_paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c9fc3531",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "from sb3_contrib import RecurrentPPO\n",
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.vec_env import VecFrameStack\n",
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "from stable_baselines3.common.env_util import make_atari_env"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1d33e3c",
   "metadata": {},
   "source": [
    "### 1. Explore breakout environment\n",
    "<ul>\n",
    "    <li>Explore the environment</li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1aad3196",
   "metadata": {},
   "outputs": [],
   "source": [
    "env_name='Breakout-v0'\n",
    "env=gym.make(env_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "df2cd276",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action space:  Discrete(4)\n",
      "Observation space shape:  Box([[[0 0 0]\n",
      "  [0 0 0]\n",
      "  [0 0 0]\n",
      "  ...\n",
      "  [0 0 0]\n",
      "  [0 0 0]\n",
      "  [0 0 0]]\n",
      "\n",
      " [[0 0 0]\n",
      "  [0 0 0]\n",
      "  [0 0 0]\n",
      "  ...\n",
      "  [0 0 0]\n",
      "  [0 0 0]\n",
      "  [0 0 0]]\n",
      "\n",
      " [[0 0 0]\n",
      "  [0 0 0]\n",
      "  [0 0 0]\n",
      "  ...\n",
      "  [0 0 0]\n",
      "  [0 0 0]\n",
      "  [0 0 0]]\n",
      "\n",
      " ...\n",
      "\n",
      " [[0 0 0]\n",
      "  [0 0 0]\n",
      "  [0 0 0]\n",
      "  ...\n",
      "  [0 0 0]\n",
      "  [0 0 0]\n",
      "  [0 0 0]]\n",
      "\n",
      " [[0 0 0]\n",
      "  [0 0 0]\n",
      "  [0 0 0]\n",
      "  ...\n",
      "  [0 0 0]\n",
      "  [0 0 0]\n",
      "  [0 0 0]]\n",
      "\n",
      " [[0 0 0]\n",
      "  [0 0 0]\n",
      "  [0 0 0]\n",
      "  ...\n",
      "  [0 0 0]\n",
      "  [0 0 0]\n",
      "  [0 0 0]]], [[[255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  ...\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]]\n",
      "\n",
      " [[255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  ...\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]]\n",
      "\n",
      " [[255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  ...\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]]\n",
      "\n",
      " ...\n",
      "\n",
      " [[255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  ...\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]]\n",
      "\n",
      " [[255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  ...\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]]\n",
      "\n",
      " [[255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  ...\n",
      "  [255 255 255]\n",
      "  [255 255 255]\n",
      "  [255 255 255]]], (210, 160, 3), uint8)\n",
      "Action space shape:  ()\n",
      "Observation space shape:  (210, 160, 3)\n"
     ]
    }
   ],
   "source": [
    "print('Action space: ',env.action_space)\n",
    "print('Observation space shape: ',env.observation_space)\n",
    "print('Action space shape: ',env.action_space.shape)\n",
    "print('Observation space shape: ',env.observation_space.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a195559a",
   "metadata": {},
   "source": [
    "#Taking a look at how breakout looks and simulate random actions\n",
    "env.reset()\n",
    "rewards_all=[]\n",
    "for epi in range(50):\n",
    "    state=env.reset()\n",
    "    rewards=0\n",
    "    while True:\n",
    "        action=env.action_space.sample()\n",
    "        state,reward,done,_=env.step(action)\n",
    "        env.render()\n",
    "        rewards+=reward\n",
    "        if done:\n",
    "            rewards_all.append(rewards)\n",
    "            if (epi+1)%5==0:\n",
    "                print('Episode: ',epi+1,' Reward: ',rewards)\n",
    "            break  \n",
    "env.close()\n",
    "print('Average rewards for random action Breakout-v0 is: ',sum(rewards_all)/len(rewards_all))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54c5cdd5",
   "metadata": {},
   "source": [
    "### 2. Training\n",
    "<ol>\n",
    "    <li>Make the atari env using make_atari_env</li>\n",
    "    <li>Parallelize the learning using VecFrameStack</li>\n",
    "    <li>Create train function for all appropriate algos</li>\n",
    "    <li>Train</li>\n",
    "    <li>Store model</li>\n",
    "</ol> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "00e701da",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_breakout(algo_name, env_name,n_steps,device='cuda'):\n",
    "    policy='CnnLstmPolicy' if algo_name=='RecurrentPPO' else 'CnnPolicy'\n",
    "    log_path, render_path, model_path=create_paths(algo_name,env_name,n_steps)\n",
    "    env=make_atari_env(env_name,n_envs=6)\n",
    "    env=VecFrameStack(env, n_stack=6)\n",
    "    model=eval(algo_name)(policy,env, tensorboard_log=log_path,device=device)\n",
    "    model.learn(n_steps)\n",
    "    model.save(os.path.join(model_path,env_name+\"_\"+algo_name+\"_\"+\"model\"))\n",
    "    del model\n",
    "    env.close()\n",
    "\n",
    "    return log_path, render_path, model_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "65fae03b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['PPO', 'RecurrentPPO']\n"
     ]
    }
   ],
   "source": [
    "algo_list=['PPO','RecurrentPPO']\n",
    "print(algo_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ebcf18c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "models_records={}\n",
    "n_steps=5000000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "52178f03",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_experiment(algo_list, env_name,n_steps):\n",
    "    for algo in reversed(algo_list):\n",
    "        print('******* Training ',algo,' *******')\n",
    "        log_path, render_path, model_path=train_breakout(algo, env_name,n_steps)\n",
    "        gc.collect()\n",
    "        torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0a4d7aa0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "******* Training  RecurrentPPO  *******\n",
      "******* Training  PPO  *******\n"
     ]
    }
   ],
   "source": [
    "train_experiment(algo_list, env_name,n_steps)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}