script_2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Banana Navigation\n",
    "\n",
    "## Import Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mtensorflow 1.7.1 has requirement numpy>=1.13.3, but you'll have numpy 1.12.1 which is incompatible.\u001b[0m\r\n",
      "\u001b[31mipython 6.5.0 has requirement prompt-toolkit<2.0.0,>=1.0.15, but you'll have prompt-toolkit 3.0.7 which is incompatible.\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "# If you are runnning on Udacity's VM please run their script\n",
    "vm_var = True\n",
    "!pip -q install ./python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from unityagents import UnityEnvironment\n",
    "import numpy as np\n",
    "import random\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from collections import deque, namedtuple\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize Unity Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:unityagents:\n",
      "'Academy' started successfully!\n",
      "Unity Academy name: Academy\n",
      "        Number of Brains: 1\n",
      "        Number of External Brains : 1\n",
      "        Lesson number : 0\n",
      "        Reset Parameters :\n",
      "\t\t\n",
      "Unity brain name: BananaBrain\n",
      "        Number of Visual Observations (per agent): 0\n",
      "        Vector Observation space type: continuous\n",
      "        Vector Observation space size (per agent): 37\n",
      "        Number of stacked Vector Observation: 1\n",
      "        Vector Action space type: discrete\n",
      "        Vector Action space size (per agent): 4\n",
      "        Vector Action descriptions: , , , \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of agents: 1\n",
      "Number of actions: 4\n",
      "The states look like: [ 1.          0.          0.          0.          0.84408134  0.          0.\n",
      "  1.          0.          0.0748472   0.          1.          0.          0.\n",
      "  0.25755     1.          0.          0.          0.          0.74177343\n",
      "  0.          1.          0.          0.          0.25854847  0.          0.\n",
      "  1.          0.          0.09355672  0.          1.          0.          0.\n",
      "  0.31969345  0.          0.        ]\n",
      "The states size: 37\n"
     ]
    }
   ],
   "source": [
    "if vm_var:\n",
    "    env = UnityEnvironment(file_name=\"/data/Banana_Linux_NoVis/Banana.x86_64\")\n",
    "# else:\n",
    "#     env = UnityEnvironment(filename=\"data/Banana_Linux_NoVis/Banana.x86_64\")\n",
    "\n",
    "# Obtain the default brain\n",
    "brain_name = env.brain_names[0]\n",
    "brain = env.brains[brain_name]\n",
    "\n",
    "# Reset the environment\n",
    "env_info = env.reset(train_mode=True)[brain_name]\n",
    "# Display number of agents in the environment\n",
    "print('Number of agents:', len(env_info.agents))\n",
    "\n",
    "# Obtain and display action size \n",
    "action_size = brain.vector_action_space_size\n",
    "print('Number of actions:', action_size)\n",
    "\n",
    "# Obtain and display state space\n",
    "state = env_info.vector_observations[0]\n",
    "print('The states look like:', state)\n",
    "\n",
    "# Obtain and display state size\n",
    "state_size = len(state)\n",
    "print('The states size:', state_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Deep Learning Model\n",
    "\n",
    "2 fully connected hidden layers with 64 neurons and ReLu as activation function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Qmodel(state_size, action_size, seed):\n",
    "    model = nn.Sequential(nn.Linear(state_size, 128),\n",
    "                         nn.ReLU(),\n",
    "                         nn.Linear(128, 128),\n",
    "                         nn.ReLU(),\n",
    "                         nn.Linear(128,action_size))\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Replay Buffer\n",
    "\n",
    "Replay buffer with random sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer:\n",
    "    \"\"\"Fixed-size buffer to store experience tuples.\"\"\"\n",
    "\n",
    "    def __init__(self, action_size, buffer_size, batch_size, seed):\n",
    "        \"\"\"Initialize a ReplayBuffer object.\n",
    "\n",
    "        Params\n",
    "        ======\n",
    "            action_size (int): dimension of each action\n",
    "            buffer_size (int): maximum size of buffer\n",
    "            batch_size (int): size of each training batch\n",
    "            seed (int): random seed\n",
    "        \"\"\"\n",
    "        self.action_size = action_size\n",
    "        self.memory = deque(maxlen=buffer_size)  \n",
    "        self.batch_size = batch_size\n",
    "        self.experience = namedtuple(\"Experience\", field_names=[\"state\", \"action\", \"reward\", \"next_state\", \"done\"])\n",
    "        self.seed = random.seed(seed)\n",
    "    \n",
    "    def add(self, state, action, reward, next_state, done):\n",
    "        \"\"\"Add a new experience to memory.\"\"\"\n",
    "        e = self.experience(state, action, reward, next_state, done)\n",
    "        self.memory.append(e)\n",
    "    \n",
    "    def sample(self):\n",
    "        \"\"\"Randomly sample a batch of experiences from memory.\"\"\"\n",
    "        experiences = random.sample(self.memory, k=self.batch_size)\n",
    "\n",
    "        states = torch.from_numpy(np.vstack([e.state for e in experiences if e is not None])).float().to(device)\n",
    "        actions = torch.from_numpy(np.vstack([e.action for e in experiences if e is not None])).long().to(device)\n",
    "        rewards = torch.from_numpy(np.vstack([e.reward for e in experiences if e is not None])).float().to(device)\n",
    "        next_states = torch.from_numpy(np.vstack([e.next_state for e in experiences if e is not None])).float().to(device)\n",
    "        dones = torch.from_numpy(np.vstack([e.done for e in experiences if e is not None]).astype(np.uint8)).float().to(device)\n",
    "  \n",
    "        return (states, actions, rewards, next_states, dones)\n",
    "\n",
    "    def __len__(self):\n",
    "        \"\"\"Return the current size of internal memory.\"\"\"\n",
    "        return len(self.memory)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "BUFFER_SIZE = int(1e5)  # replay buffer size\n",
    "BATCH_SIZE = 128         # minibatch size\n",
    "GAMMA = 0.999            # discount factor\n",
    "TAU = 5e-3              # for soft update of target parameters\n",
    "LR = 5e-4               # learning rate \n",
    "UPDATE_EVERY = 16        # how often to update the network\n",
    "\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "\n",
    "class Agent():\n",
    "    def __init__(self,state_size, action_size, seed):\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        self.seed = random.seed(seed)\n",
    "        self.q_local = Qmodel(state_size,action_size, seed).to(device)\n",
    "        self.q_target = Qmodel(state_size,action_size, seed).to(device)\n",
    "        self.optimizer = optim.Adam(self.q_local.parameters(), lr=LR)\n",
    "        \n",
    "        self.buffer = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed)\n",
    "        self.t_step = 0\n",
    "        return\n",
    "    \n",
    "    def step(self, state, action, reward, next_state, done):\n",
    "        self.buffer.add(state, action, reward, next_state, done)\n",
    "        \n",
    "        self.t_step = (self.t_step +1) % UPDATE_EVERY        \n",
    "        if self.t_step == 0:\n",
    "            if len(self.buffer)>BATCH_SIZE:\n",
    "                self.learn()\n",
    "        return\n",
    "    \n",
    "    def act(self,state, eps = 0.0):\n",
    "        state = torch.from_numpy(state).float().unsqueeze(0).to(device)\n",
    "        self.q_local.eval()\n",
    "        with torch.no_grad():\n",
    "            action = self.q_local(state)\n",
    "        self.q_local.train()\n",
    "        if random.random() > eps:\n",
    "            action = np.argmax(action.cpu().data.numpy())\n",
    "        else:\n",
    "            action = np.random.choice(np.arange(self.action_size))\n",
    "        return action\n",
    "                               \n",
    "    def learn(self):\n",
    "        states, actions, rewards, next_states, dones = self.buffer.sample()\n",
    "        q_targets_next = self.q_target(next_states).detach().max(1)[0].unsqueeze(1)\n",
    "        q_targets = rewards + (GAMMA * q_targets_next * (1 - dones))\n",
    "        q_pred = self.q_local(states).gather(1,actions)\n",
    "        loss = F.mse_loss(q_pred,q_targets)\n",
    "        self.optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        self.optimizer.step()\n",
    "        self.soft_update()\n",
    "        return\n",
    "        \n",
    "    def soft_update(self):\n",
    "        for target_p, local_p in zip(self.q_target.parameters(), self.q_local.parameters()):\n",
    "            target_p.data.copy_(TAU * local_p.data + (1 - TAU)*target_p.data)\n",
    "        return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Training Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dqn(n_episodes=3000, max_t=1000, eps_start=1.0, eps_end=0.01, eps_decay=0.995):\n",
    "    scores = []\n",
    "    scores_recent = []\n",
    "    eps = eps_start\n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        env_info = env.reset(train_mode=True)[brain_name]\n",
    "        state = env_info.vector_observations[0]            # get the initial state\n",
    "        score = 0\n",
    "        for i_t in range(max_t):                           # considered as episodic with 100 steps\n",
    "            action = agent.act(state, eps)                 # select an action\n",
    "            env_info = env.step(action)[brain_name]        # send the action to the environment\n",
    "            next_state = env_info.vector_observations[0]   # get the next state\n",
    "            reward = env_info.rewards[0]                   # get the reward\n",
    "            done = env_info.local_done[0]                  # see if episode has finished\n",
    "            agent.step(state, action, reward, next_state, done)  # Sample & Learn step of Q learning\n",
    "            score += reward                                # update the score\n",
    "            state = next_state                             # roll over the state to next time step\n",
    "            if done:                                       # exit loop if episode finished\n",
    "                break\n",
    "        scores.append(score)\n",
    "        scores_recent.append(score)\n",
    "        eps = max(eps_end, eps*eps_decay)                  # decrease epsilon\n",
    "        print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_recent)), end=\"\")\n",
    "        if i_episode % 100 == 0:\n",
    "            print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_recent)))\n",
    "        if np.mean(scores_recent)>=13.0:\n",
    "            print('\\nEnvironment solved in {:d} episodes!\\tAverage Score: {:.2f}'.format(i_episode-100, np.mean(scores_recent)))\n",
    "            torch.save(agent.q_local.state_dict(), 'my_weights.pth')\n",
    "            break\n",
    "    return scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run The Code!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 100\tAverage Score: 2.87\n",
      "Episode 200\tAverage Score: 6.37\n",
      "Episode 300\tAverage Score: 8.29\n",
      "Episode 400\tAverage Score: 9.72\n",
      "Episode 500\tAverage Score: 10.82\n",
      "Episode 600\tAverage Score: 11.58\n",
      "Episode 700\tAverage Score: 12.10\n",
      "Episode 800\tAverage Score: 12.50\n",
      "Episode 900\tAverage Score: 12.93\n",
      "Episode 928\tAverage Score: 13.00\n",
      "Environment solved in 828 episodes!\tAverage Score: 13.00\n"
     ]
    }
   ],
   "source": [
    "# Provide parameters to agent\n",
    "agent = Agent(state_size, action_size, seed=0)\n",
    "# Obtain scores\n",
    "scores = dqn(2000, eps_decay=0.930)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot the scores\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(len(scores)), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}