PPO_Algorithm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SWEN 711 - Spring 24 - Term Project\n",
    "\n",
    "### Code written by: Hemanth Chebiyam, Sanjeev Vijayakumar\n",
    "\n",
    "### PPO Algorithm on OpenAI Gym Car Racing Environment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Import Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gymnasium as gym\n",
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Test Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "environment_name = \"CarRacing-v2\"\n",
    "env = gym.make(environment_name, render_mode=\"human\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[[0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         ...,\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0]],\n",
       " \n",
       "        [[0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         ...,\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0]],\n",
       " \n",
       "        [[0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         ...,\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0]],\n",
       " \n",
       "        ...,\n",
       " \n",
       "        [[0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         ...,\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0]],\n",
       " \n",
       "        [[0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         ...,\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0]],\n",
       " \n",
       "        [[0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         ...,\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0],\n",
       "         [0, 0, 0]]], dtype=uint8),\n",
       " {})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Box([-1.  0.  0.], 1.0, (3,), float32)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.action_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Box(0, 255, (96, 96, 3), uint8)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.observation_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Simulating without Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode:1 Score:-29.32862190812757\n",
      "Episode:2 Score:-29.328621908127605\n",
      "Episode:3 Score:-29.328621908127623\n",
      "Episode:4 Score:-29.328621908127566\n",
      "Episode:5 Score:-25.79505300353385\n"
     ]
    }
   ],
   "source": [
    "environment_name = \"CarRacing-v2\"\n",
    "env = gym.make(environment_name, render_mode=\"human\")\n",
    "\n",
    "episodes = 5\n",
    "for episode in range(episodes):\n",
    "    observation, info = env.reset(seed=42)\n",
    "    score = 0 \n",
    "    done = False\n",
    "\n",
    "    while not done:\n",
    "        action = env.action_space.sample()  # this is where you would insert your policy\n",
    "        observation, reward, terminated, truncated, info = env.step(action)\n",
    "        score += reward\n",
    "\n",
    "        if terminated or truncated:\n",
    "            break\n",
    "\n",
    "    print('Episode:{} Score:{}'.format(episode + 1, score))\n",
    "\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Simulating after basic training (~500k timesteps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make(environment_name, render_mode=\"human\")\n",
    "env = DummyVecEnv([lambda: env])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "ppo_path = os.path.join('Training', 'Saved Models', 'PPO_500k_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\heman\\miniconda3\\envs\\gymenv\\Lib\\site-packages\\stable_baselines3\\common\\save_util.py:167: UserWarning: Could not deserialize object lr_schedule. Consider using `custom_objects` argument to replace this object.\n",
      "Exception: code expected at least 16 arguments, got 15\n",
      "  warnings.warn(\n",
      "c:\\Users\\heman\\miniconda3\\envs\\gymenv\\Lib\\site-packages\\stable_baselines3\\common\\save_util.py:167: UserWarning: Could not deserialize object clip_range. Consider using `custom_objects` argument to replace this object.\n",
      "Exception: code expected at least 16 arguments, got 15\n",
      "  warnings.warn(\n",
      "c:\\Users\\heman\\miniconda3\\envs\\gymenv\\Lib\\site-packages\\stable_baselines3\\common\\vec_env\\patch_gym.py:95: UserWarning: You loaded a model that was trained using OpenAI Gym. We strongly recommend transitioning to Gymnasium by saving that model again.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wrapping the env in a VecTransposeImage.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\heman\\miniconda3\\envs\\gymenv\\Lib\\site-packages\\stable_baselines3\\common\\base_class.py:751: UserWarning: You are probably loading a model saved with SB3 < 1.7.0, we deactivated exact_match so you can save the model again to avoid issues in the future (see https://github.com/DLR-RM/stable-baselines3/issues/1233 for more info). Original error: Error(s) in loading state_dict for ActorCriticCnnPolicy:\n",
      "\tMissing key(s) in state_dict: \"pi_features_extractor.cnn.0.weight\", \"pi_features_extractor.cnn.0.bias\", \"pi_features_extractor.cnn.2.weight\", \"pi_features_extractor.cnn.2.bias\", \"pi_features_extractor.cnn.4.weight\", \"pi_features_extractor.cnn.4.bias\", \"pi_features_extractor.linear.0.weight\", \"pi_features_extractor.linear.0.bias\", \"vf_features_extractor.cnn.0.weight\", \"vf_features_extractor.cnn.0.bias\", \"vf_features_extractor.cnn.2.weight\", \"vf_features_extractor.cnn.2.bias\", \"vf_features_extractor.cnn.4.weight\", \"vf_features_extractor.cnn.4.bias\", \"vf_features_extractor.linear.0.weight\", \"vf_features_extractor.linear.0.bias\".  \n",
      "Note: the model should still work fine, this only a warning.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "model = PPO.load(ppo_path, env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode:1 Score:-4.593639575972384\n",
      "Episode:2 Score:83.74558303887123\n",
      "Episode:3 Score:48.409893992936006\n",
      "Episode:4 Score:-15.194346289753488\n",
      "Episode:5 Score:94.34628975265443\n"
     ]
    }
   ],
   "source": [
    "environment_name = \"CarRacing-v2\"\n",
    "env = gym.make(environment_name, render_mode=\"human\")\n",
    "\n",
    "episodes = 5\n",
    "for episode in range(episodes):\n",
    "    observation, info = env.reset(seed=42)\n",
    "    score = 0 \n",
    "    done = False\n",
    "\n",
    "    while not done:\n",
    "        action, _ = model.predict(observation)  # this is where you would insert your policy\n",
    "        observation, reward, terminated, truncated, info = env.step(action)\n",
    "        score += reward\n",
    "\n",
    "        if terminated or truncated:\n",
    "            break\n",
    "\n",
    "    print('Episode:{} Score:{}'.format(episode + 1, score))\n",
    "\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Simulating after more training (~2 mil timesteps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make(environment_name, render_mode=\"human\")\n",
    "env = DummyVecEnv([lambda: env])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "ppo_path = os.path.join('Training', 'Saved Models', 'PPO_2mil_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\heman\\miniconda3\\envs\\gymenv\\Lib\\site-packages\\stable_baselines3\\common\\save_util.py:167: UserWarning: Could not deserialize object lr_schedule. Consider using `custom_objects` argument to replace this object.\n",
      "Exception: code expected at least 16 arguments, got 15\n",
      "  warnings.warn(\n",
      "c:\\Users\\heman\\miniconda3\\envs\\gymenv\\Lib\\site-packages\\stable_baselines3\\common\\save_util.py:167: UserWarning: Could not deserialize object clip_range. Consider using `custom_objects` argument to replace this object.\n",
      "Exception: code expected at least 16 arguments, got 15\n",
      "  warnings.warn(\n",
      "c:\\Users\\heman\\miniconda3\\envs\\gymenv\\Lib\\site-packages\\stable_baselines3\\common\\vec_env\\patch_gym.py:95: UserWarning: You loaded a model that was trained using OpenAI Gym. We strongly recommend transitioning to Gymnasium by saving that model again.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wrapping the env in a VecTransposeImage.\n"
     ]
    }
   ],
   "source": [
    "model = PPO.load(ppo_path, env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode:1 Score:620.8480565370928\n",
      "Episode:2 Score:525.4416961130704\n",
      "Episode:3 Score:574.9116607773736\n",
      "Episode:4 Score:589.0459363957509\n",
      "Episode:5 Score:571.3780918727856\n"
     ]
    }
   ],
   "source": [
    "environment_name = \"CarRacing-v2\"\n",
    "env = gym.make(environment_name, render_mode=\"human\")\n",
    "episodes = 5\n",
    "for episode in range(episodes):\n",
    "    observation, info = env.reset(seed=42)\n",
    "    score = 0 \n",
    "    done = False\n",
    "\n",
    "    while not done:\n",
    "        action, _ = model.predict(observation)  # this is where you would insert your policy\n",
    "        observation, reward, terminated, truncated, info = env.step(action)\n",
    "        score += reward\n",
    "\n",
    "        if terminated or truncated:\n",
    "            break\n",
    "\n",
    "    print('Episode:{} Score:{}'.format(episode + 1, score))\n",
    "\n",
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}