003 Week 3 Value Function Approximation and DQN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lectures\n",
    " - [Value functions approximation - RL by David Silver](https://www.youtube.com/watch?v=UoPei5o4fps&list=PLqYmG7hTraZDM-OYHWgPebj2MfCFzFObQ&index=6)\n",
    "    - Differentiable function approximators\n",
    "    - Incremental methods\n",
    "    - Batch methods (DQN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading:\n",
    " - [Playing Atari with Deep Reinforcement Learning](https://arxiv.org/pdf/1312.5602.pdf)\n",
    " - [Human-level control through deep reinforcement\n",
    "learning](https://storage.googleapis.com/deepmind-media/dqn/DQNNaturePaper.pdf)\n",
    "  - Preprocessing.\n",
    "  - Deep Q-learning.\n",
    "  - Hyperparameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Coding\n",
    "This week we will apply Deep Q-Networks (DQN) to Pong."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Explore the environment:\n",
    "- https://gym.openai.com/envs/Pong-v0/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### install env: \n",
    " - pip install atari-py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.misc import imresize\n",
    "import random \n",
    "import torch\n",
    "import os\n",
    "import time\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Preprocessor():\n",
    "    def __init__(self, game, h, w, device, n_frame=4):\n",
    "        self.game = game\n",
    "        self.h, self.w = h, w\n",
    "        self.set_box(game)\n",
    "        self.n_frame = n_frame\n",
    "        self.history = []\n",
    "        self.device = device\n",
    "        self.delta_value = 256\n",
    "    def set_box(self, game):\n",
    "        if self.game == 'pong':\n",
    "            self.box = [35, 193, 0, self.w]\n",
    "        \n",
    "    def phi(self, rbg, to_tensor=True):\n",
    "        stack = None\n",
    "        frame = self.rgb2gray(rbg)\n",
    "        self.history.append(frame)\n",
    "        if len(self.history) == self.n_frame:\n",
    "            stack = np.stack(self.history)\n",
    "            if to_tensor:\n",
    "                stack = self.to_tensor([stack])\n",
    "        self.history = self.history[-(self.n_frame-1):]\n",
    "        return stack\n",
    "    \n",
    "    def to_tensor(self, frames):\n",
    "        frame = torch.from_numpy(np.stack(frames)).float().to(self.device)\n",
    "        return frame\n",
    "    \n",
    "    def rgb2gray(self, rgb):\n",
    "        rgb = rgb[self.box[0]:self.box[1], self.box[2]: self.box[3]]\n",
    "        rgb = np.array(Image.fromarray(rgb).resize((84,84)))\n",
    "        gray = np.dot(rgb[...,:3], [0.2989, 0.5870, 0.1140])\n",
    "        gray = gray / self.delta_value\n",
    "        return gray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor = Preprocessor('pong', 210, 160, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make(\"Pong-v0\")\n",
    "observation = env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(210, 160, 3)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "observation.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 NOOP\n",
      "1 FIRE\n",
      "2 RIGHT\n",
      "3 LEFT\n",
      "4 RIGHTFIRE\n",
      "5 LEFTFIRE\n"
     ]
    }
   ],
   "source": [
    "for action_id, name in enumerate(env.unwrapped.get_action_meanings()):\n",
    "    print(action_id, name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "NOOP, RIGHT, LEFT= 0, 2, 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAKAAAADSCAIAAABCR1ywAAACh0lEQVR4nO3bPUoDURhAUSMD2lu4CDdgaZeVWNq6GVeSzjIbcBEp0sfOJsg0EgPODLmeUwXyMw8uHw/mTVZXP3h5ffjpLS7Ias6Qz0+nr/X2/jHDSuZx+Nyc/MztzXrSNVxP+ussTuA4geOGpS483mt/szdfuvFe+5u9+a+Y4DiB4wSOEzhO4DiB4wSOEzhO4DiB4wSOW+xe9H+4/zw25/3nMRMcJ3CcwHGzPpPF/ExwnMBxw3a3X3oNTMgExwkcJ3CcwHECxwkcJ3CcwHECxw2P93dLr4EJmeA4geMEjhM4TuA4geMEjhM4TuA4geOOgbe7vafvkkxwnMBxAscd/x/s0LDKBMcJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHDcsvQBOO3xuvl/f3qzP+q4JjhM4TuA4geMEjhM4TuA4geMEjhM4TuA4geMEjhM4TuA458EX4Nwz4DETHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHCdwnMBxAscJHDdsd/ul18CETHCcwHECxwkcJ3CcwHECxwkcJ3CcwHECxwkcJ3CcwHECx30Bu1ci6JJ73gAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<PIL.Image.Image image mode=RGB size=160x210 at 0x7F7838017710>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Image.fromarray(observation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "ob = env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.0, False, {'ale.lives': 0}, (84, 84))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADJJJREFUeJzt3V2MXPV5x/Hvr14cAgmyTQpyMSlYskiiqpjISqHkgjpJS2kEuUhaUCKlVVrfpCppKwXTXrRUipRIVV4uqkgWJEVVykscmlhcJLUc0vbKwcakBYyDSSje4NhUQN4uUB2eXsxxu3WG7Nndmd09/n8/0mjmnDkz53909Jtz5uzs86SqkNSWX1jpAUhafgZfapDBlxpk8KUGGXypQQZfapDBlxq0pOAnuT7JkSRHk+yc1KAkTVcW+wOeJGuAbwPvAmaBh4FbquqJyQ1P0jTMLOG1bwOOVtV3AJLcC9wEvGrwk/gzQWnKqirzLbOUU/1LgGNzpme7eZJWuaUc8cd9qvzMET3JDmDHEtYjacKWEvxZ4NI505uA585cqKp2AbvAU31ptVjKqf7DwJYklydZC9wM7JnMsCRN06KP+FV1KskfA18D1gCfq6rHJzYySVOz6D/nLWplnupLUzftq/qSBsrgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNWje4Cf5XJKTSR6bM29Dkr1Jnuru1093mJImqc8R/++B68+YtxPYV1VbgH3dtKSBmDf4VfWvwAtnzL4JuLt7fDfwngmPS9IULfY7/sVVdRygu79ockOSNG1LaajRi510pNVnsUf8E0k2AnT3J19twaraVVXbqmrbItclacIWG/w9wAe7xx8EvjKZ4UhaDvM21EhyD3Ad8AbgBPBXwJeB+4E3As8C76uqMy8AjnsvG2pIU9anoYaddKSzjJ10JI1l8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxrUp5POpUkeSnI4yeNJbu3m201HGqg+Nfc2Ahur6pEkrwcOMmqg8fvAC1X18SQ7gfVVdds872XpLWnKJlJ6q6qOV9Uj3eMfAYeBS7CbjjRYC2qokeQy4CpgP2d000kytpuODTWk1ad3ld0krwP+BfhYVT2Q5KWqWjfn+Rer6ud+z/dUX5q+iVXZTXIO8CXgC1X1QDe7dzcdSatLn6v6Ae4CDlfVJ+c8ZTcdaaD6XNV/O/BvwH8Ar3Sz/4LR9/wFddPxVF+aPjvpSA2yk46ksQy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSg/rU3Ds3yTeTfKvrpHNHN//yJPu7Tjr3JVk7/eFKmoQ+R/yXge1VdSWwFbg+ydXAJ4BPVdUW4EXgQ9MbpqRJ6tNJp6rqx93kOd2tgO3A7m6+nXSkAelbV39NkkcZ1c7fCzwNvFRVp7pFZhm11Rr32h1JDiQ5MIkBS1q6XsGvqp9W1VZgE/A24M3jFnuV1+6qqm1VtW3xw5Q0SQu6ql9VLwHfAK4G1iU53XtvE/DcZIcmaVr6XNX/xSTrusevBd7JqGPuQ8B7u8XspCMNSJ9OOr/K6OLdGkYfFPdX1d8k2QzcC2wADgEfqKqX53kvG2qoWeOyNupQN/H12ElHWi1WU/D95Z7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UoN7B70psH0ryYDdtJx1poBZyxL+VUZHN0+ykIw1U34Yam4DfAe7spoOddKTB6nvE/zTwUeCVbvpC7KQjDVafuvrvBk5W1cG5s8csaicd6edI8jO3lTIz/yJcC9yY5AbgXOACRmcA65LMdEd9O+lIA9KnW+7tVbWpqi4Dbga+XlXvx0460mAt5e/4twF/luQoo+/8d01mSJKmzU460lnGTjqSxjL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81qE/NPZI8A/wI+Clwqqq2JdkA3AdcBjwD/G5VvTidYUqapIUc8X+jqrbOqZa7E9jXNdTY101LGoClnOrfxKiRBthQQxqUvsEv4J+THEyyo5t3cVUdB+juL5rGACVNXq/v+MC1VfVckouAvUme7LuC7oNix7wLSlo2C66ym+SvgR8DfwRcV1XHk2wEvlFVV8zzWqvsSlM2kSq7Sc5P8vrTj4HfBB4D9jBqpAE21JAGZd4jfpLNwD91kzPAP1bVx5JcCNwPvBF4FnhfVb0wz3t5xJemrM8R34Ya0lnGhhqSxjL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDeoV/CTrkuxO8mSSw0muSbIhyd4kT3X366c9WEmT0feI/xngq1X1JuBK4DB20pEGq0+xzQuAbwGba87CSY5geW1p1ZlUzb3NwPPA55McSnJnV2bbTjrSQPUJ/gzwVuCzVXUV8BMWcFqfZEeSA0kOLHKMkiasT/Bngdmq2t9N72b0QXCiO8Wnuz857sVVtauqts3psitphc0b/Kr6PnAsyenv7+8AnsBOOtJg9WqokWQrcCewFvgO8AeMPjTspCOtMnbSkRpkJx1JYxl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBs0b/CRXJHl0zu2HST5iJx1puBZUeivJGuB7wK8BHwZeqKqPJ9kJrK+q2+Z5vaW3pCmbRumtdwBPV9V/AjcBd3fz7wbes8D3krRCFhr8m4F7usd20pEGqnfwk6wFbgS+uJAV2ElHWn0WcsT/beCRqjrRTdtJRxqohQT/Fv7vNB/spCMNVt9OOucBxxi1yv5BN+9C7KQjrTp20pEaZCcdSWMZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQb1Cn6SP03yeJLHktyT5NwklyfZ33XSua+rwitpAPq00LoE+BNgW1X9CrCGUX39TwCfqqotwIvAh6Y5UEmT0/dUfwZ4bZIZ4DzgOLAd2N09bycdaUDmDX5VfQ/4W0aVdI8DPwAOAi9V1alusVngkmkNUtJk9TnVX8+oT97lwC8B5zNqrnGmsRV07aQjrT4zPZZ5J/DdqnoeIMkDwK8D65LMdEf9TcBz415cVbuAXd1rLa8trQJ9vuM/C1yd5LwkYdQx9wngIeC93TJ20pEGpG8nnTuA3wNOAYeAP2T0nf5eYEM37wNV9fI87+MRX5oyO+lIDbKTjqSxDL7UIIMvNcjgSw3q83f8Sfov4Cfd/dniDbg9q9XZtC3Qb3t+uc8bLetVfYAkB6pq27KudIrcntXrbNoWmOz2eKovNcjgSw1aieDvWoF1TpPbs3qdTdsCE9yeZf+OL2nleaovNWhZg5/k+iRHkhxNsnM5171USS5N8lCSw139wVu7+RuS7O1qD+7t6hcMRpI1SQ4lebCbHmwtxSTrkuxO8mS3n64Z8v6ZZq3LZQt+kjXA3zEq4vEW4JYkb1mu9U/AKeDPq+rNwNXAh7vx7wT2dbUH93XTQ3IrcHjO9JBrKX4G+GpVvQm4ktF2DXL/TL3WZVUtyw24BvjanOnbgduXa/1T2J6vAO8CjgAbu3kbgSMrPbYFbMMmRmHYDjwIhNEPRGbG7bPVfAMuAL5Ld91qzvxB7h9G//Z+jNG/vc90++e3JrV/lvNU//SGnDbYOn1JLgOuAvYDF1fVcYDu/qKVG9mCfRr4KPBKN30hw62luBl4Hvh899XlziTnM9D9U1OudbmcwR/3P8KD+5NCktcBXwI+UlU/XOnxLFaSdwMnq+rg3NljFh3KPpoB3gp8tqquYvTT8EGc1o+z1FqX81nO4M8Cl86ZftU6fatVknMYhf4LVfVAN/tEko3d8xuBkys1vgW6FrgxyTOMKiltZ3QGsK4row7D2kezwGxV7e+mdzP6IBjq/vnfWpdV9d/A/6t12S2z6P2znMF/GNjSXZVcy+hCxZ5lXP+SdPUG7wIOV9Un5zy1h1HNQRhQ7cGqur2qNlXVZYz2xder6v0MtJZiVX0fOJbkim7W6dqQg9w/TLvW5TJfsLgB+DbwNPCXK30BZYFjfzuj06p/Bx7tbjcw+l68D3iqu9+w0mNdxLZdBzzYPd4MfBM4CnwReM1Kj28B27EVONDtoy8D64e8f4A7gCeBx4B/AF4zqf3jL/ekBvnLPalBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQb9D+VeELyFhaZmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "observation, reward, done, info = env.step(0)\n",
    "frame = preprocessor.rgb2gray(observation)\n",
    "plt.imshow(frame, cmap = \"gray\")\n",
    "reward, done, info, frame.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5748781249999999"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "ob = env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# i = 92\n",
    "# t = D.D_win[i][0][0][0].cpu().numpy()\n",
    "# plt.imshow(t, cmap = \"gray\")\n",
    "# time.sleep(0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# frame.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.0, False, {'ale.lives': 0}, (84, 84), torch.Size([1, 4, 84, 84]))"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADJdJREFUeJzt3V2sHPV5x/Hvrz44BBJkmxTkYlKwZJFEVYHISqHkgjpJS2kEuUhSUCKlVVrfpCppKwXTXrRUipRIVV4uqkgWJEVVykscmiAukloOaXrlYDBpAUMwCcUnODYVkLcLVIenFztuT92lZ845u3vO+P/9SKvdmTO78x+Nfjuzs3ueJ1WFpLb8wmoPQNLsGXypQQZfapDBlxpk8KUGGXypQQZfatCKgp/kmiRPJjmcZNekBiVpurLcH/AkWQd8F3gXMA88CNxYVY9PbniSpmFuBc99G3C4qr4HkOQu4HrgVYOfxJ8JSlNWVVlsmZWc6l8AHFkwPd/Nk7TGreSIP+5d5f8c0ZPsBHauYD2SJmwlwZ8HLlwwvQV47tSFqmo3sBs81ZfWipWc6j8IbEtycZL1wA3AfZMZlqRpWvYRv6pOJPkj4OvAOuDzVfXYxEYmaWqW/XXeslbmqb40ddO+qi9poAy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgxYNfpLPJzme5NEF8zYl2Zvkqe5+43SHKWmS+hzx/w645pR5u4B9VbUN2NdNSxqIRYNfVd8CXjhl9vXAHd3jO4D3THhckqZouZ/xz6+qowDd/XmTG5KkaVtJQ41e7KQjrT3LPeIfS7IZoLs//moLVtXuqtpeVduXuS5JE7bc4N8HfKh7/CHgq5MZjqRZWLShRpI7gauBNwDHgL8EvgLcA7wReBZ4X1WdegFw3GvZUEOasj4NNeykI51m7KQjaSyDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1KA+nXQuTPJAkkNJHktyUzffbjrSQPWpubcZ2FxVDyd5PfAQowYavwe8UFWfSLIL2FhVNy/yWpbekqZsIqW3qupoVT3cPf4JcAi4ALvpSIO1pIYaSS4CLgf2c0o3nSRju+nYUENae3pX2U3yOuCfgY9X1b1JXqqqDQv+/mJV/b+f8z3Vl6ZvYlV2k5wBfBn4YlXd283u3U1H0trS56p+gNuBQ1X1qQV/spuONFB9ruq/HfgX4N+AV7rZf87oc/6Suul4qi9Nn510pAbZSUfSWAZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQX1q7p2Z5NtJvtN10rm1m39xkv1dJ527k6yf/nAlTUKfI/7LwI6quhS4DLgmyRXAJ4FPV9U24EXgw9MbpqRJ6tNJp6rqp93kGd2tgB3Anm6+nXSkAelbV39dkkcY1c7fCzwNvFRVJ7pF5hm11Rr33J1JDiQ5MIkBS1q5XsGvqp9X1WXAFuBtwJvHLfYqz91dVduravvyhylpkpZ0Vb+qXgK+CVwBbEhysvfeFuC5yQ5N0rT0uar/i0k2dI9fC7yTUcfcB4D3dovZSUcakD6ddH6V0cW7dYzeKO6pqr9OshW4C9gEHAQ+WFUvL/JaNtSQpsxOOlKD7KQjaSyDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1KDewe9KbB9Mcn83bScdaaCWcsS/iVGRzZPspCMNVN+GGluA3wFu66aDnXSkwep7xP8M8DHglW76XOykIw1Wn7r67waOV9VDC2ePWdROOtJAzC2+CFcB1yW5FjgTOIfRGcCGJHPdUd9OOtKA9OmWe0tVbamqi4AbgG9U1Qewk440WCv5Hv9m4E+THGb0mf/2yQxJ0rTZSUc6zdhJR9JYBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qUJ8KPJImYNy/wI/q1s6eR3ypQb2O+EmeAX4C/Bw4UVXbk2wC7gYuAp4B3l9VL05nmJImaSlH/N+oqssWVMvdBezrGmrs66YlDcBKTvWvZ9RIA2yoIQ1K3+AX8E9JHkqys5t3flUdBejuz5vGACVNXt+r+ldV1XNJzgP2Jnmi7wq6N4qdiy4oaWaWXGU3yV8BPwX+ELi6qo4m2Qx8s6ouWeS5VtlVs2b1dd5EquwmOTvJ608+Bn4TeBS4j1EjDbChhjQoix7xk2wF/rGbnAP+oao+nuRc4B7gjcCzwPuq6oVFXssjvpq1lo74NtSQZmQtBd9f7kkNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI8trSjKxWKe1xPOJLDTL4UoMMvtQggy81qFfwk2xIsifJE0kOJbkyyaYke5M81d1vnPZgJU1G3yP+Z4GvVdWbgEuBQ9hJRxqsPsU2zwG+A2ytBQsneRLLa0trzqRq7m0Fnge+kORgktu6Mtt20pEGqk/w54C3Ap+rqsuBn7GE0/okO5McSHJgmWOUNGF9gj8PzFfV/m56D6M3gmPdKT7d/fFxT66q3VW1fUGXXUmrbNHgV9UPgSNJTn5+fwfwOHbSkQarV0ONJJcBtwHrge8Bv8/oTcNOOtIaYycdqUF20pE0lsGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9q0KLBT3JJkkcW3H6c5KN20pGGa0mlt5KsA34A/BrwEeCFqvpEkl3Axqq6eZHnW3pLmrJplN56B/B0Vf07cD1wRzf/DuA9S3wtSatkqcG/Abize2wnHWmgegc/yXrgOuBLS1mBnXSktWcpR/zfBh6uqmPdtJ10pIFaSvBv5H9O88FOOtJg9e2kcxZwhFGr7B91887FTjrSmmMnHalBdtKRNJbBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfalCv4Cf5kySPJXk0yZ1JzkxycZL9XSedu7sqvJIGoE8LrQuAPwa2V9WvAOsY1df/JPDpqtoGvAh8eJoDlTQ5fU/154DXJpkDzgKOAjuAPd3f7aQjDciiwa+qHwB/w6iS7lHgR8BDwEtVdaJbbB64YFqDlDRZfU71NzLqk3cx8EvA2Yyaa5xqbAVdO+lIa89cj2XeCXy/qp4HSHIv8OvAhiRz3VF/C/DcuCdX1W5gd/dcy2tLa0Cfz/jPAlckOStJGHXMfRx4AHhvt4yddKQB6dtJ51bgd4ETwEHgDxh9pr8L2NTN+2BVvbzI63jEl6bMTjpSg+ykI2ksgy81yOBLDTL4UoP6fI8/Sf8B/Ky7P128AbdnrTqdtgX6bc8v93mhmV7VB0hyoKq2z3SlU+T2rF2n07bAZLfHU32pQQZfatBqBH/3Kqxzmtyetet02haY4PbM/DO+pNXnqb7UoJkGP8k1SZ5McjjJrlmue6WSXJjkgSSHuvqDN3XzNyXZ29Ue3NvVLxiMJOuSHExyfzc92FqKSTYk2ZPkiW4/XTnk/TPNWpczC36SdcDfMiri8RbgxiRvmdX6J+AE8GdV9WbgCuAj3fh3Afu62oP7uukhuQk4tGB6yLUUPwt8rareBFzKaLsGuX+mXuuyqmZyA64Evr5g+hbgllmtfwrb81XgXcCTwOZu3mbgydUe2xK2YQujMOwA7gfC6Acic+P22Vq+AecA36e7brVg/iD3D6N/ez/C6N/e57r981uT2j+zPNU/uSEnDbZOX5KLgMuB/cD5VXUUoLs/b/VGtmSfAT4GvNJNn8twayluBZ4HvtB9dLktydkMdP/UlGtdzjL44/5HeHBfKSR5HfBl4KNV9ePVHs9yJXk3cLyqHlo4e8yiQ9lHc8Bbgc9V1eWMfho+iNP6cVZa63Ixswz+PHDhgulXrdO3ViU5g1Hov1hV93azjyXZ3P19M3B8tca3RFcB1yV5hlElpR2MzgA2dGXUYVj7aB6Yr6r93fQeRm8EQ90//13rsqr+E/hftS67ZZa9f2YZ/AeBbd1VyfWMLlTcN8P1r0hXb/B24FBVfWrBn+5jVHMQBlR7sKpuqaotVXURo33xjar6AAOtpVhVPwSOJLmkm3WyNuQg9w/TrnU54wsW1wLfBZ4G/mK1L6AscexvZ3Ra9a/AI93tWkafi/cBT3X3m1Z7rMvYtquB+7vHW4FvA4eBLwGvWe3xLWE7LgMOdPvoK8DGIe8f4FbgCeBR4O+B10xq//jLPalB/nJPapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQf8F0vUWtByw6pgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(4):\n",
    "    observation, reward, done, info = env.step(3)\n",
    "    frame = preprocessor.rgb2gray(observation)\n",
    "    frames = preprocessor.phi(observation)\n",
    "frames = preprocessor.phi(observation)\n",
    "plt.imshow(frame, cmap = \"gray\")\n",
    "reward, done, info, frame.shape, frames.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test preprocess:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_images(images,):\n",
    "    assert len(images) == 4\n",
    "    \n",
    "    # Create figure with 3x3 sub-plots.\n",
    "    fig, axes = plt.subplots(2, 2)\n",
    "    fig.subplots_adjust(hspace=0.3, wspace=0.3)\n",
    "\n",
    "    for i, ax in enumerate(axes.flat):\n",
    "        # Plot image.\n",
    "        ax.imshow(images[i], cmap='binary')\n",
    "        # Remove ticks from the plot.\n",
    "        ax.set_xticks([])\n",
    "        ax.set_yticks([])\n",
    "        \n",
    "    plt.show()\n",
    "    \n",
    "def show_image_from_frames(frames):\n",
    "    images = np.array(frames.cpu())[0]\n",
    "    plot_images(images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAADuCAYAAABRejAmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAABABJREFUeJzt2sFt4lAYRlG/UUog63ER9F8BRZB16OFNBUgxI3i54py1F9/i15UtGHPODaDqz+oBAP9DxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIO3jyMOn02nu+/6kKdxzvV632+02Vu94R25+jSM3fyhi+75vl8vlsVU87Hw+r57wttz8Gkdu3uckkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkittAYYxtjrJ4BaSIGpIkYkCZiQJqIAWkiBrzMM37MEjEgTcSANBED0j5WD3hnc87VEyDPmxiQJmJAmogBaSIGpIkYkCZiQJq/WAAv84y/FXkTA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgLQx5/z5w2N8b9v29bw53PF3zvm5esQ7cvPL/PjmD0UM4LfxOQmkiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJA2seRh0+n09z3/UlTuOd6vW63222s3vGO3PwaR27+UMT2fd8ul8tjq3jY+XxePeFtufk1jty8z0kgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kRsoTHGNsZYPQNe5hk3L2JAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkDax+oB72zOuXoCvNQzbt6bGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJA2phz/vzhMb63bft63hzu+Dvn/Fw94h25+WV+fPOHIgbw2/icBNJEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0v4Bo5FaY8Dk+cIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_image_from_frames(frames)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class DQN(nn.Module):\n",
    "    def __init__(self, input_shape, n_actions):\n",
    "        super(DQN, self).__init__()\n",
    "        self.conv = nn.Sequential(\n",
    "            nn.Conv2d(input_shape[0],32,8,stride=4),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(32,64,4, stride=2),\n",
    "            nn.BatchNorm2d(64),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(64,64,3, stride=1),\n",
    "            nn.BatchNorm2d(64),\n",
    "            nn.ReLU())\n",
    "        self.fc = nn.Sequential(\n",
    "            nn.Linear(3136, 512),\n",
    "            nn.ReLU(), \n",
    "            nn.Linear(512, n_actions))\n",
    "    def forward(self, x):\n",
    "        out = self.conv(x)\n",
    "        out = out.view(out.size(0), -1)\n",
    "        out = self.fc(out)\n",
    "        return out\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_optimizer(Q_net, frames, lr=0.1, n_iter=1, is_train = [True, False, False], MSE=nn.MSELoss()):\n",
    "    optimizer = torch.optim.RMSprop(Q_net.parameters(), lr=lr)\n",
    "    for i in range(n_iter):\n",
    "        loss = None\n",
    "        y_hat = Q_net(frames)\n",
    "        print(f\"y_hat {y_hat}.\")\n",
    "        idx = torch.LongTensor([[0]]).to(device)\n",
    "        y_0 = torch.gather(y_hat, 1, idx)\n",
    "        print(f\"y_0 {y_0}.\")\n",
    "        idx = torch.LongTensor([[1]]).to(device)\n",
    "        y_1 = torch.gather(y_hat, 1, idx)\n",
    "        print(f\"y_1 {y_1}\")\n",
    "        idx = torch.LongTensor([[2]]).to(device)\n",
    "        y_2 = torch.gather(y_hat, 1, idx)\n",
    "        print(f\"y_2 {y_2}\")\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        if is_train[0]:\n",
    "            y =  torch.from_numpy(np.array([-1])).float().to(device)\n",
    "            loss1 = MSE(y_0, y).to(device)\n",
    "            if loss:\n",
    "                loss += loss1\n",
    "            else:\n",
    "                loss = loss1\n",
    "        \n",
    "        if is_train[1]:\n",
    "            y =  torch.from_numpy(np.array([1])).float().to(device)\n",
    "            loss2 = MSE(y_1, y).to(device)\n",
    "            if loss:\n",
    "                loss += loss2\n",
    "            else:\n",
    "                loss = loss2\n",
    "                \n",
    "        if is_train[2]:\n",
    "            y =  torch.from_numpy(np.array([1])).float().to(device)\n",
    "            loss3 = MSE(y_2, y).to(device)\n",
    "            if loss:\n",
    "                loss += loss3\n",
    "            else:\n",
    "                loss = loss3\n",
    "                \n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        print(f\"new y_hat {Q_net(frames)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_hat tensor([[ 0.1216,  0.0957, -0.0241]], device='cuda:0', grad_fn=<AddmmBackward>).\n",
      "y_0 tensor([[0.1216]], device='cuda:0', grad_fn=<GatherBackward>).\n",
      "y_1 tensor([[0.0957]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "y_2 tensor([[-0.0241]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "new y_hat tensor([[-96953960.,  96988952.,  96947872.]], device='cuda:0',\n",
      "       grad_fn=<AddmmBackward>)\n",
      "y_hat tensor([[-96953960.,  96988952.,  96947872.]], device='cuda:0',\n",
      "       grad_fn=<AddmmBackward>).\n",
      "y_0 tensor([[-96953960.]], device='cuda:0', grad_fn=<GatherBackward>).\n",
      "y_1 tensor([[96988952.]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "y_2 tensor([[96947872.]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "new y_hat tensor([[ 12239248., -12242504., -12243192.]], device='cuda:0',\n",
      "       grad_fn=<AddmmBackward>)\n"
     ]
    }
   ],
   "source": [
    "Q_net = DQN([4], 3).to(device)\n",
    "run_optimizer(Q_net, frames, lr=1, n_iter=2, is_train=[True, True, True])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_hat tensor([[ 0.0031, -0.0331,  0.0368]], device='cuda:0', grad_fn=<AddmmBackward>).\n",
      "y_0 tensor([[0.0031]], device='cuda:0', grad_fn=<GatherBackward>).\n",
      "y_1 tensor([[-0.0331]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "y_2 tensor([[0.0368]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "new y_hat tensor([[-499.1908,  497.9409,  498.9920]], device='cuda:0',\n",
      "       grad_fn=<AddmmBackward>)\n",
      "y_hat tensor([[-499.1908,  497.9409,  498.9920]], device='cuda:0',\n",
      "       grad_fn=<AddmmBackward>).\n",
      "y_0 tensor([[-499.1908]], device='cuda:0', grad_fn=<GatherBackward>).\n",
      "y_1 tensor([[497.9409]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "y_2 tensor([[498.9920]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "new y_hat tensor([[ 118.9477, -116.9591, -122.7569]], device='cuda:0',\n",
      "       grad_fn=<AddmmBackward>)\n"
     ]
    }
   ],
   "source": [
    "Q_net = DQN([4], 3).to(device)\n",
    "run_optimizer(Q_net, frames, lr=0.01, n_iter=2, is_train=[True, True, True])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_hat tensor([[-0.0186,  0.0989, -0.0904]], device='cuda:0', grad_fn=<AddmmBackward>).\n",
      "y_0 tensor([[-0.0186]], device='cuda:0', grad_fn=<GatherBackward>).\n",
      "y_1 tensor([[0.0989]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "y_2 tensor([[-0.0904]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "new y_hat tensor([[-1.1356,  1.0272,  1.2238]], device='cuda:0', grad_fn=<AddmmBackward>)\n",
      "y_hat tensor([[-1.1356,  1.0272,  1.2238]], device='cuda:0', grad_fn=<AddmmBackward>).\n",
      "y_0 tensor([[-1.1356]], device='cuda:0', grad_fn=<GatherBackward>).\n",
      "y_1 tensor([[1.0272]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "y_2 tensor([[1.2238]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "new y_hat tensor([[-0.6805,  0.9789,  0.4365]], device='cuda:0', grad_fn=<AddmmBackward>)\n"
     ]
    }
   ],
   "source": [
    "Q_net = DQN([4], 3).to(device)\n",
    "run_optimizer(Q_net, frames, lr=0.0001, n_iter=2, is_train=[True, True, True])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_hat tensor([[ 0.1456, -0.1430, -0.0242]], device='cuda:0', grad_fn=<AddmmBackward>).\n",
      "y_0 tensor([[0.1456]], device='cuda:0', grad_fn=<GatherBackward>).\n",
      "y_1 tensor([[-0.1430]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "y_2 tensor([[-0.0242]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "new y_hat tensor([[-0.3034,  0.1838,  0.2527]], device='cuda:0', grad_fn=<AddmmBackward>)\n",
      "y_hat tensor([[-0.3034,  0.1838,  0.2527]], device='cuda:0', grad_fn=<AddmmBackward>).\n",
      "y_0 tensor([[-0.3034]], device='cuda:0', grad_fn=<GatherBackward>).\n",
      "y_1 tensor([[0.1838]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "y_2 tensor([[0.2527]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "new y_hat tensor([[-0.4747,  0.3839,  0.4468]], device='cuda:0', grad_fn=<AddmmBackward>)\n",
      "y_hat tensor([[-0.4747,  0.3839,  0.4468]], device='cuda:0', grad_fn=<AddmmBackward>).\n",
      "y_0 tensor([[-0.4747]], device='cuda:0', grad_fn=<GatherBackward>).\n",
      "y_1 tensor([[0.3839]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "y_2 tensor([[0.4468]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "new y_hat tensor([[-0.5728,  0.5219,  0.5637]], device='cuda:0', grad_fn=<AddmmBackward>)\n",
      "y_hat tensor([[-0.5728,  0.5219,  0.5637]], device='cuda:0', grad_fn=<AddmmBackward>).\n",
      "y_0 tensor([[-0.5728]], device='cuda:0', grad_fn=<GatherBackward>).\n",
      "y_1 tensor([[0.5219]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "y_2 tensor([[0.5637]], device='cuda:0', grad_fn=<GatherBackward>)\n",
      "new y_hat tensor([[-0.6554,  0.6188,  0.6605]], device='cuda:0', grad_fn=<AddmmBackward>)\n"
     ]
    }
   ],
   "source": [
    "Q_net = DQN([4], 3).to(device)\n",
    "run_optimizer(Q_net, frames, lr=0.00003, n_iter=4, is_train=[True, True, True])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# good learning rate is about 0.00003"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data loader:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This DataLoader will save data and generate data for training model.\n",
    "\n",
    "class DataLoader():\n",
    "    def __init__(self, batch_size, win_buffer_start_size=1000, loss_buffer_start_size=3001, buffer_capacity=40000):\n",
    "        self.batch_size = batch_size\n",
    "        self.D_win = []\n",
    "        self.D_lose = []\n",
    "        self.buffer_capacity = buffer_capacity\n",
    "        self.loss_buffer_start_size = loss_buffer_start_size\n",
    "        self.win_buffer_start_size = win_buffer_start_size \n",
    "        \n",
    "    def reset(self):\n",
    "        self.D_win = []\n",
    "        self.D_lose = []\n",
    "        \n",
    "    def add_samples(self, game, win):\n",
    "        if win:\n",
    "            for step in game:\n",
    "                self.D_win.append(step)\n",
    "            if len(self.D_win) > self.buffer_capacity:\n",
    "                self.D_win = self.D_win[-(self.buffer_capacity - 5000):]\n",
    "        else:\n",
    "            for step in game:\n",
    "                self.D_lose.append(step)\n",
    "            if len(self.D_lose) > self.buffer_capacity:\n",
    "                self.D_lose = self.D_lose[-(self.buffer_capacity - 5000):]\n",
    "            \n",
    "    def get_sample(self):\n",
    "        batch = None\n",
    "        is_mixed = True\n",
    "        if len(self.D_win) > self.win_buffer_start_size:\n",
    "            batch_win = random.sample(self.D_win, self.batch_size // 2)\n",
    "            batch_lose = random.sample(self.D_lose, self.batch_size // 2)\n",
    "            batch = batch_win + batch_lose\n",
    "        elif len(self.D_lose) > self.loss_buffer_start_size:\n",
    "            is_mixed = False\n",
    "            batch = random.sample(self.D_lose, self.batch_size)\n",
    "        return batch, is_mixed\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.D_win), len(self.D_lose)\n",
    "    \n",
    "    def get_item(self, idx, is_win=False):\n",
    "        if is_win:\n",
    "            D = self.D_win\n",
    "        else:\n",
    "            D = self.D_lose\n",
    "        if idx < 0 or idx > len(D) - 1:\n",
    "            return None\n",
    "        return D[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.2100121342750727"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "2 * 0.99 ** 50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Algorithm 1: deep Q-learning with experience replay."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Policy():\n",
    "    def __init__(self, Q, n_action, device, win_gamma=1, loss_gamma=1, epsilon=1, alpha=0.8, learning_rate=0.00003):\n",
    "        self.win_gamma = win_gamma\n",
    "        self.loss_gamma = loss_gamma\n",
    "        self.episode = 1\n",
    "        self.alpha = alpha\n",
    "        self.epsilon = epsilon\n",
    "        self.EPS_DECAY_RATE = 0.989\n",
    "        self.n_action = n_action\n",
    "        self.learning_rate = learning_rate\n",
    "        self.action_mapper = [NOOP, RIGHT, LEFT]\n",
    "        self.final_exploration = 0.05\n",
    "        self.device = device\n",
    "        # Step 1: Initilize Q(State, Action)\n",
    "        self.net = Q.to(device)\n",
    "        self.loss_fn = nn.MSELoss(reduction='sum')\n",
    "        self.model_path = './models/DQN/checkpoint.pth.tar'\n",
    "        self.load_model()\n",
    "        self.min_loss = 1\n",
    "        self.max_iter = 5\n",
    "    def update_epsilon(self,):\n",
    "        self.epsilon *= self.EPS_DECAY_RATE\n",
    "    \n",
    "    def update_min_loss(self,):\n",
    "        self.min_loss *= self.EPS_DECAY_RATE\n",
    "        \n",
    "    def save_model(self,):\n",
    "        torch.save({'state_dict': self.net.state_dict()}, self.model_path)\n",
    "        \n",
    "    def load_model(self,):\n",
    "        if os.path.isfile(self.model_path):\n",
    "            checkpoint = torch.load(self.model_path)\n",
    "            self.net.load_state_dict(checkpoint['state_dict'])\n",
    "        self.optimizer = torch.optim.RMSprop(self.net.parameters(), lr=self.learning_rate)\n",
    "        \n",
    "    def to_tensor(self, frames):\n",
    "        frame = torch.from_numpy(np.stack(frames)).float().to(self.device)\n",
    "        return frame\n",
    "    \n",
    "    def update_state_dict(self, Q):\n",
    "        self.net.load_state_dict(Q.net.state_dict())\n",
    "        \n",
    "    def get_y_hat(self, batch):\n",
    "        batch_state = [sample[0] for sample in batch]\n",
    "        y_Q = self.net(torch.cat(batch_state))\n",
    "\n",
    "        batch_a = [[sample[1]] for sample in batch]\n",
    "        idx = torch.LongTensor(batch_a).to(device)\n",
    "        y_Q = torch.gather(y_Q, 1, idx)\n",
    "        return y_Q\n",
    "    \n",
    "    def update_Q(self, y, batch, verbose=False):\n",
    "        keep_running = True\n",
    "        count = 0\n",
    "        while keep_running:\n",
    "            self.optimizer.zero_grad()\n",
    "            y_hat = self.get_y_hat(batch)\n",
    "            loss = self.loss_fn(y_hat, y)\n",
    "            if verbose:\n",
    "                print(loss)\n",
    "            loss.backward()\n",
    "            self.optimizer.step()\n",
    "            count += 1\n",
    "            keep_running = (loss.item() > max(0.3, self.min_loss)) & (count < self.max_iter)\n",
    "            \n",
    "    def get_target(self, batch, is_mixed=False, verbose=False):\n",
    "        r_t = torch.from_numpy(np.array([sample[2] for sample in batch])).float().to(self.device)\n",
    "        if verbose:\n",
    "            print('r_t', r_t)\n",
    "        s_t_plus_1 = [sample[3] for sample in batch]\n",
    "        \n",
    "        q_t_plus_1 = self.net(torch.cat(s_t_plus_1))\n",
    "        if verbose:\n",
    "            print('q_t_plus_1 for all action: ', q_t_plus_1)\n",
    "        q_t_plus_1 = q_t_plus_1.max(dim=1)[0]\n",
    "        if verbose:\n",
    "            print('q_t_plus_1', q_t_plus_1)\n",
    "        if is_mixed:\n",
    "            q_t_plus_1[0:len(q_t_plus_1)//2] = q_t_plus_1[0:len(q_t_plus_1)//2] * self.win_gamma\n",
    "            q_t_plus_1[len(q_t_plus_1)//2:] = q_t_plus_1[len(q_t_plus_1)//2:] * self.loss_gamma\n",
    "        else:\n",
    "            q_t_plus_1 = q_t_plus_1 * self.loss_gamma\n",
    "        if verbose:\n",
    "            print('q_t_plus_1 with gamma', q_t_plus_1)\n",
    "        idx = (r_t != -1) & (r_t != 1)\n",
    "        q_t_plus_1 = torch.clamp(q_t_plus_1, min=-0.95, max=0.95)\n",
    "        r_t[idx] += q_t_plus_1[idx]\n",
    "        # do net train target value\n",
    "        r_t = r_t.view(-1,1).detach()\n",
    "        # to ensure value in taget in range [-1,1]\n",
    "        return r_t\n",
    "    \n",
    "    def get_next_action(self, state, is_e_greedy=False):\n",
    "        if is_e_greedy and np.random.random() < max(self.epsilon, self.final_exploration):\n",
    "            next_action = np.random.randint(self.n_action)\n",
    "            return next_action, self.action_mapper[next_action]\n",
    "        \n",
    "        output = self.net(state)\n",
    "        _, next_action = torch.max(output, dim=1)\n",
    "        next_action = next_action.item()\n",
    "        return next_action, self.action_mapper[next_action]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.10946072477880486"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "0.989 ** 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor = Preprocessor('pong', 210, 160, device)\n",
    "D = DataLoader(32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon=1\n",
    "Q = Policy(DQN([4,84,84], 3), 3, device, epsilon=epsilon)\n",
    "# Q_hat = Policy(DQN([4,84,84], 3), 3, device, epsilon=epsilon)\n",
    "# Q_hat.update_state_dict(Q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "def play(env, Q, preprocessor, is_e_greedy=False, render=False):\n",
    "    Q.net.train(False)\n",
    "    game = []\n",
    "    for i in range(5):\n",
    "        observation, reward, done, info = env.step(0)\n",
    "        phi_t = preprocessor.phi(observation)\n",
    "    for t in range(10000):\n",
    "        if render:\n",
    "            time.sleep(0.1)\n",
    "            env.render()\n",
    "            time.sleep(0.1)\n",
    "        index_a_t, a_t = Q.get_next_action(phi_t, is_e_greedy=is_e_greedy)\n",
    "        x_t_plus_1, r_t, done, info = env.step(a_t)\n",
    "        phi_t_plus_1 = preprocessor.phi(x_t_plus_1)\n",
    "        \n",
    "        game.append((phi_t, index_a_t, r_t, phi_t_plus_1))\n",
    "        \n",
    "        if (r_t == -1) or (r_t == 1) or done:\n",
    "            return game, env, r_t == 1, done\n",
    "        # repair for next step\n",
    "        phi_t = phi_t_plus_1\n",
    "    Q.net.train(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let play\n",
    "# expected behavior: the bar will move randomly because the Q net is created randomly.\n",
    "env.reset()\n",
    "game, env, win, done = play(env, Q, preprocessor, is_e_greedy=False, render=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAADuCAYAAABRejAmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAABIVJREFUeJzt28GNGksUQNHqrwkB1iYIckJEhMiJIMZryKH/2pLNDDLQXHPOupCexNOdoqSZ5nkeAFX/LT0AwN8QMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSPu45fBqtZo3m82DRnk/5/N5jDHGer2+eu7z83NcLpfpGTPxKzt/X4/Y+Zsittlsxul0uuUjXHE8HscYY+x2u6vnttvtM8bhN+z8fT1i52+KGPf11RcJ/5pH7Lw3MSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMXgR0zQtPUKSiAFpLxkxf5F4R/M8Lz1C0ktGDEoOh8M4HA5Lj/G2RAxIEzEg7e4Ru8fV2tsA8F1uYkCaiAFpIgakiRiQJmJA2sfSA0Ddfr9feoS35iYGpIkYkCZiQNrd38S8DwDP5CYGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkDbN8/z9w9N0HmP8fNw4/MGPeZ7XSw/xjuz8Yr698zdFDODV+DkJpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQNrHLYdXq9W82WweNMr7OZ/PY4wx1uv11XOfn5/jcrlMz5iJX9n5+3rEzt8Usc1mM06n0y0f4Yrj8TjGGGO32109t91unzEOv2Hn7+sRO39TxLivr75I+Nc8Yue9iQFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYvJhpmsY0TUuPkSFiQJqIAWkfSw8A/Gqe56VHSHnpm5h3AeArLx0xKDgcDuNwOCw9xtt66Yi5VgNfeemIAXzl7hFztQaeyU0MSBMxIE3EgDQRA9JEDEjzb0fwl/b7/dIjvDU3MSBNxIC0u/+cdLUGnslNDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSAtGme5+8fnqbzGOPn48bhD37M87xeeoh3ZOcX8+2dvyliAK/Gz0kgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSDtf8WlikpKDMD2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_image_from_frames(game[46][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0593, -0.1079,  0.0416]], device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q.net(game[50][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Counter({0: 37, 1: 29, 2: 15})"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "print(len(game))\n",
    "actions = Counter([t[1]for t in game])\n",
    "# expected uniform distribution on actions because we use randome action in play.\n",
    "actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let play the second round:\n",
    "game, env, win, done = play(env, Q, preprocessor, is_e_greedy=False, render=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Counter({1: 18, 2: 8, 0: 14})"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(game))\n",
    "actions = Counter([t[1]for t in game])\n",
    "# expected uniform distribution on actions because we use randome action in play.\n",
    "actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAADuCAYAAABRejAmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAABGJJREFUeJzt2cGNGksUQNHqrwkB1iYIciIkRE4EgdeQQ3ttacy4/wA913POukBP4umqq5nmeR4AVf+tPQDAZ4gYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakvS05vNls5t1u96RRvp/r9TrGGGO73d49d7lcxu12m14xE7+z84/1jJ1fFLHdbjfO5/OSj3DH6XQaY4xxOBzuntvv968Yh3fY+cd6xs4vihiP9dEPCf+aZ+y8d2JAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkDawyN2PB7H8Xh89Nf+s6ZpWnsESPMkBqSJ2MrmeV57BD7J7WO5R95ARAxIEzHg5R55AxExIE3EgDQRA9JEDEgTMSBtWvIvwTRN1zHGz+eNwx/8mOd5u/YQ35GdX81f7/yiiAF8Na6TQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqS9LTm82Wzm3W73pFG+n+v1OsYYY7vd3j13uVzG7XabXjETv7Pzj/WMnV8Usd1uN87n85KPcMfpdBpjjHE4HO6e2+/3rxiHd9j5x3rGzi+KGI/10Q8JfMw7MSDNkxjwMs+4fXgSA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBOxL2SaprVHgJyHR+x4PI7j8fjorwV4lyexL2Se57VHgFVM0/S/byIiBqSJGHySVyjrelt7AIDPvErxJAakTUsKOE3TdYzx83nj8Ac/5nnerj3Ed2TnV/PXO78oYgBfjeskkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQ9gsRVIVY5KQ6+AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_image_from_frames(game[38][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let play the third round:\n",
    "game, env, win, done = play(env, Q, preprocessor, is_e_greedy=False, render=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(False, False)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "win, done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Counter({1: 17, 2: 16, 0: 7})"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(game))\n",
    "actions = Counter([t[1]for t in game])\n",
    "# expected uniform distribution on actions because we use randome action in play.\n",
    "actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAADuCAYAAABRejAmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAABIpJREFUeJzt3MFt21gUQNHPgUuw1mER7slwRYR6chHOWuqBs84gdsIxZeZG56wp4gF6uPrkQtO6rgOg6p+jBwD4DBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIC0hy0XPz4+rvM832iU+3O5XMYYY5xOpw+ve3t7G9frdfqKmfiRnd/XLXZ+U8TmeR6vr69bPsIHzufzGGOM5+fnD697enr6inH4CTu/r1vs/KaIAXzGr+L1f3gnBqQ5iR3oFr9KcG+cxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSAtN0jtizLWJZl79v+0aZpGtM0HT0G3CUnMSBNxIA0EdvBuq5jXdejx+Ag9/gK5b+OfJ0iYkCaiAFpIgZ82pGvU0QMSBMxIO1h7xu+vLzsfUuAdzmJAWkiBqTt/jgJ98YrlGM5iQFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJA2bflHxmmaLmOM77cbh3d8W9f1dPQQ98jOH+a3d35TxAD+NB4ngTQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEgTMSBNxIA0EQPSRAxIEzEgTcSANBED0kQMSBMxIE3EgDQRA9JEDEh72HLx4+PjOs/zjUa5P5fLZYwxxul0+vC6t7e3cb1ep6+YiR/Z+X3dYuc3RWye5/H6+rrlI3zgfD6PMcZ4fn7+8Lqnp6evGIefsPP7usXOb4oY+/rVFwl/m1vsvHdiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkCZiQJqIAWkiBqSJGJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkLZ7xJZlGcuy7H3bnGmajh4B7oKTGHySH+5jidiNrOt69AhwiK9+ChExIE3EgF199VOIiAFpIgakiRiQJmJA2sPeN3x5edn7lgDvchID0nY/icG98fRxLCcxIE3EgDQRA9JEDEgTMSBNxIA0EQPSpi1/mzFN02WM8f124/COb+u6no4e4h7Z+cP89s5vihjAn8bjJJAmYkCaiAFpIgakiRiQJmJAmogBaSIGpIkYkPYv0XmNR3dXPEgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_image_from_frames(game[30][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# run and generate sample:\n",
    "def generate_game(n_game, D, Q, preprocessor, env, is_e_greedy=True, verbose=True, is_update_epsilon=False):\n",
    "    for _ in range(n_game):\n",
    "        done = False\n",
    "        env.reset()\n",
    "        n_win, n_loss = 0, 0\n",
    "        while done == False:\n",
    "            game, env, win, done = play(env, Q, preprocessor, is_e_greedy=is_e_greedy)\n",
    "            if win:\n",
    "                n_win+=1\n",
    "            else:\n",
    "                n_loss+=1\n",
    "            if win and verbose:\n",
    "                print(\"Yes! We have one game win.\")\n",
    "\n",
    "            if (done or win) & is_update_epsilon:\n",
    "                Q.update_epsilon()\n",
    "\n",
    "            D.add_samples(game, win)\n",
    "        if (n_win > 0):\n",
    "            print(f\"Win {n_win} - {n_loss} Loss \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.reset()\n",
    "generate_game(1, D, Q, preprocessor, env, is_e_greedy=True, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Win 1 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 1 - 21 Loss \n"
     ]
    }
   ],
   "source": [
    "generate_game(30, D, Q, preprocessor, env, True, verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1062"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(D.D_win)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_N_GAMES = 1\n",
    "n_iteration = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "Q_net = DQN([4], 3).to(device)\n",
    "optimizer = torch.optim.RMSprop(Q_net.parameters(), lr=0.00003)\n",
    "MSE = nn.MSELoss(reduction='sum')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0., -1.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.]])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch, is_mixed = D.get_sample()\n",
    "np.array([sample[2] for sample in batch]).reshape(4, 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_gradient_batch(batch, MSE, optimizer, n=4):\n",
    "    batch_state = [sample[0] for sample in batch]\n",
    "    batch_state = torch.cat(batch_state)\n",
    "    y = torch.from_numpy(np.array([sample[2] for sample in batch])).float().to(device).view(-1, 1)\n",
    "    print(y.view(-1, 8))\n",
    "    for i in range(n):\n",
    "        y_Q = Q_net(batch_state)\n",
    "        batch_a = [[sample[1]] for sample in batch]\n",
    "        idx = torch.LongTensor(batch_a).to(device)\n",
    "        y_Q = torch.gather(y_Q, 1, idx)\n",
    "        print(y_Q.view(-1, 8))\n",
    "        loss = MSE(y_Q, y)\n",
    "        print(loss)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        y_Q = Q_net(batch_state)\n",
    "        y_Q = torch.gather(y_Q, 1, idx)\n",
    "    print(y_Q.view(-1, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  0., -1.,  0.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.]], device='cuda:0')\n",
      "tensor([[-0.0720,  0.1031,  0.0685, -0.0389,  0.0808, -0.0054, -0.1607, -0.0227],\n",
      "        [-0.0557, -0.0109,  0.0473,  0.1372, -0.1473, -0.0161, -0.0605, -0.0143],\n",
      "        [-0.0322,  0.0214,  0.0276,  0.1022, -0.0185,  0.0533, -0.0304,  0.0408],\n",
      "        [ 0.0132, -0.0756,  0.1195,  0.0008, -0.0009, -0.0309, -0.0365,  0.0341]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(1.1101, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[ 0.0739, -0.6511, -0.3978, -0.0436, -0.7182,  0.0430,  0.1324, -0.2517],\n",
      "        [ 0.2236, -0.0159, -0.1377, -0.6162,  0.2810,  0.0916,  0.0735, -0.0327],\n",
      "        [ 0.0509, -0.0308, -0.0455, -0.3640, -0.2061, -0.4521,  0.0261, -0.2046],\n",
      "        [-0.2492, -0.0550, -0.7259, -0.0733, -0.1372,  0.1897,  0.4313, -0.1077]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(3.5959, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[ 0.2154,  0.1591,  0.3120, -0.0083,  0.2167, -0.0883,  0.1619,  0.5258],\n",
      "        [ 0.0055,  0.1164,  0.0521,  0.1289,  0.0068, -0.0458,  0.0020,  0.5665],\n",
      "        [ 0.0099, -0.0120,  0.0427,  0.0026, -0.2465,  0.0710,  0.1373,  0.0431],\n",
      "        [ 0.2039,  0.0160,  0.1590,  0.0301, -0.0222,  0.1170, -0.2548, -0.0192]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(1.6255, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-0.1581, -0.1876, -0.2133, -0.0627, -0.1877, -0.0024, -0.1381, -0.1040],\n",
      "        [-0.1051, -0.0698,  0.0273, -0.2245, -0.0792,  0.0370, -0.0705, -0.2269],\n",
      "        [-0.0828,  0.0137,  0.0334, -0.0732, -0.3260,  0.0696, -0.1473,  0.0283],\n",
      "        [-0.1296, -0.0616, -0.0517,  0.0391, -0.0706, -0.4813,  0.0707,  0.0472]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(1.0697, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[ 0.0954,  0.0765,  0.1422,  0.0267,  0.0825,  0.0751,  0.0745,  0.1323],\n",
      "        [ 0.1171,  0.0610, -0.0123,  0.0873,  0.0609, -0.0297,  0.0531,  0.1119],\n",
      "        [ 0.0729, -0.0204,  0.0093,  0.0306, -0.3393, -0.0716,  0.0545, -0.0481],\n",
      "        [ 0.0677,  0.0296,  0.0707, -0.0069,  0.0032,  0.2036, -0.0038, -0.0206]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(0.6226, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-0.0209, -0.0384, -0.0393, -0.0954, -0.0429, -0.1410, -0.0254, -0.0130],\n",
      "        [-0.1485, -0.0106,  0.0150, -0.0438, -0.0948,  0.0354, -0.1067, -0.0521],\n",
      "        [-0.1178,  0.0092,  0.0254, -0.0814, -0.4222,  0.0415, -0.0333,  0.0179],\n",
      "        [-0.0275, -0.0811, -0.1103,  0.0149, -0.0863, -0.0542, -0.0106,  0.0148]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(0.4725, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-0.0020,  0.0034, -0.0045,  0.0441,  0.0011,  0.1302, -0.0024,  0.0167],\n",
      "        [ 0.1384,  0.0083,  0.0057,  0.0037,  0.0908, -0.0482,  0.0792,  0.0115],\n",
      "        [ 0.0899, -0.0133,  0.0106,  0.0544, -0.4198, -0.0167, -0.0084, -0.0067],\n",
      "        [ 0.0046,  0.0460,  0.1272,  0.0059,  0.0276,  0.0038,  0.0005, -0.0099]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n"
     ]
    }
   ],
   "source": [
    "test_gradient_batch(batch, MSE, optimizer, n=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0014,  0.0237, -0.0429],\n",
       "        [ 0.0121,  0.0321, -0.0013]], device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_state = [sample[0] for sample in batch[0:2]]\n",
    "batch_state = torch.cat(batch_state)\n",
    "Q_net(batch_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0094,  0.0179, -0.0462],\n",
       "        [-0.0079,  0.0123,  0.0200],\n",
       "        [-0.1172, -0.0071,  0.0041],\n",
       "        [ 0.0998,  0.0497,  0.0264]], device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_state = [sample[0] for sample in batch[0:4]]\n",
    "batch_state = torch.cat(batch_state)\n",
    "Q_net(batch_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DQN(\n",
       "  (conv): Sequential(\n",
       "    (0): Conv2d(4, 32, kernel_size=(8, 8), stride=(4, 4))\n",
       "    (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "    (3): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2))\n",
       "    (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): ReLU()\n",
       "    (6): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (8): ReLU()\n",
       "  )\n",
       "  (fc): Sequential(\n",
       "    (0): Linear(in_features=3136, out_features=512, bias=True)\n",
       "    (1): ReLU()\n",
       "    (2): Linear(in_features=512, out_features=3, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Problem: Inconsistent results during test using different batch size.\n",
    "# https://discuss.pytorch.org/t/solved-inconsistent-results-during-test-using-different-batch-size/2265/7\n",
    "Q_net.train(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0203,  0.0123, -0.0877],\n",
       "        [ 0.0201,  0.0124, -0.0882]], device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_state = [sample[0] for sample in batch[0:2]]\n",
    "batch_state = torch.cat(batch_state)\n",
    "Q_net(batch_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0203,  0.0123, -0.0877],\n",
       "        [ 0.0201,  0.0124, -0.0882],\n",
       "        [ 0.0207,  0.0119, -0.0879],\n",
       "        [ 0.0204,  0.0122, -0.0883]], device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_state = [sample[0] for sample in batch[0:4]]\n",
    "batch_state = torch.cat(batch_state)\n",
    "Q_net(batch_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DQN(\n",
       "  (conv): Sequential(\n",
       "    (0): Conv2d(4, 32, kernel_size=(8, 8), stride=(4, 4))\n",
       "    (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "    (3): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2))\n",
       "    (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): ReLU()\n",
       "    (6): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (8): ReLU()\n",
       "  )\n",
       "  (fc): Sequential(\n",
       "    (0): Linear(in_features=3136, out_features=512, bias=True)\n",
       "    (1): ReLU()\n",
       "    (2): Linear(in_features=512, out_features=3, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q_net.train(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0242,  0.0177, -0.0190]], device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_state = [sample[0] for sample in batch[0:1]]\n",
    "batch_state = torch.cat(batch_state)\n",
    "Q_net(batch_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0014,  0.0237, -0.0429],\n",
       "        [ 0.0121,  0.0321, -0.0013]], device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_state = [sample[0] for sample in batch[0:2]]\n",
    "batch_state = torch.cat(batch_state)\n",
    "Q_net(batch_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test policy:\n",
    "epsilon=1\n",
    "Q = Policy(DQN([4,84,84], 3), 3, device, epsilon=epsilon)\n",
    "optimizer = torch.optim.RMSprop(Q.net.parameters(), lr=0.00003)\n",
    "MSE = nn.MSELoss(reduction='sum')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_Q_gradient_batch(batch, MSE, optimizer, n=4):\n",
    "    batch_state = [sample[0] for sample in batch]\n",
    "    batch_state = torch.cat(batch_state)\n",
    "    y = torch.from_numpy(np.array([sample[2] for sample in batch])).float().to(device).view(-1, 1)\n",
    "    print(y.view(-1, 8))\n",
    "    for i in range(n):\n",
    "        y_Q = Q.net(batch_state)\n",
    "        batch_a = [[sample[1]] for sample in batch]\n",
    "        idx = torch.LongTensor(batch_a).to(device)\n",
    "        y_Q = torch.gather(y_Q, 1, idx)\n",
    "        print(y_Q.view(-1, 8))\n",
    "        loss = MSE(y_Q, y)\n",
    "        print(loss)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        y_Q = Q.net(batch_state)\n",
    "        y_Q = torch.gather(y_Q, 1, idx)\n",
    "    print(y_Q.view(-1, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  0., -1.,  0.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.]], device='cuda:0')\n",
      "tensor([[ 0.0191, -0.0713, -0.0943, -0.0842,  0.0360,  0.2310, -0.1595, -0.1352],\n",
      "        [ 0.0217, -0.0563, -0.0078, -0.0798, -0.1396, -0.0139,  0.0446, -0.3363],\n",
      "        [ 0.0770,  0.0210,  0.0215, -0.0004,  0.0140,  0.0119,  0.0067,  0.0021],\n",
      "        [-0.0662, -0.0350,  0.0100, -0.0906,  0.0276,  0.0571, -0.0440,  0.0668]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(1.3242, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[ 0.1741,  0.8264,  0.7287, -0.1680,  0.5364, -0.4661,  0.3601,  0.9394],\n",
      "        [-0.4023,  0.1807, -0.0324,  0.5865,  0.1375,  0.0104, -0.4595,  1.1423],\n",
      "        [-0.3393, -0.0095, -0.0080, -0.1200, -0.2264, -0.1098,  0.1799, -0.0657],\n",
      "        [ 0.3802, -0.0274, -0.5195,  0.0108, -0.1818, -0.0300,  0.2466, -0.1604]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(6.1769, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-2.6230e-01, -2.1552e-01, -2.0626e-01,  4.3393e-02, -3.2541e-01,\n",
      "         -1.0222e-01, -1.3734e-01, -1.9490e-01],\n",
      "        [ 2.1302e-01, -1.2769e-01,  4.6203e-02, -1.5599e-01,  1.6363e-01,\n",
      "          8.4341e-02,  1.4019e-02, -1.8461e-01],\n",
      "        [-1.5848e-03,  3.6076e-02,  3.2054e-02,  1.1482e-01, -2.0892e-01,\n",
      "          1.3392e-01, -1.5934e-01,  5.0301e-02],\n",
      "        [-9.3621e-02,  1.0981e-01,  5.2198e-01,  4.6636e-02,  2.2950e-03,\n",
      "         -6.4053e-01, -2.1638e-04,  1.2940e-03]], device='cuda:0',\n",
      "       grad_fn=<ViewBackward>)\n",
      "tensor(1.8821, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[ 0.0280,  0.0969,  0.1372, -0.0569,  0.0064, -0.0562,  0.0722,  0.1359],\n",
      "        [-0.1060,  0.0306, -0.0375,  0.1087, -0.1306, -0.1515, -0.0805,  0.1276],\n",
      "        [-0.0480, -0.0318, -0.0217, -0.1176, -0.3153, -0.1485,  0.0192, -0.0875],\n",
      "        [ 0.0748, -0.0892, -0.1217, -0.0362, -0.0813,  0.2968, -0.1353, -0.0352]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(0.8077, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-0.0400,  0.0014, -0.0200, -0.0233, -0.0450,  0.0017, -0.0160, -0.0044],\n",
      "        [ 0.0290, -0.0195,  0.0483, -0.0040,  0.0187,  0.1266,  0.0014, -0.0172],\n",
      "        [ 0.0008,  0.0418,  0.0391,  0.0168, -0.3504,  0.1339, -0.0325,  0.0801],\n",
      "        [ 0.0051, -0.0020,  0.0113,  0.0544, -0.0245, -0.0760,  0.1541,  0.0807]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n"
     ]
    }
   ],
   "source": [
    "test_Q_gradient_batch(batch, MSE, optimizer, n=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0816, -0.0400, -0.0585],\n",
       "        [-0.0343,  0.0014, -0.2505],\n",
       "        [ 0.0655, -0.0200, -0.1409],\n",
       "        [-0.0636, -0.0151, -0.0233],\n",
       "        [-0.0580, -0.0450, -0.1127],\n",
       "        [ 0.0690, -0.1088,  0.0017],\n",
       "        [-0.0319, -0.0160, -0.0928],\n",
       "        [ 0.0487, -0.0044, -0.2034],\n",
       "        [-0.0101,  0.0350,  0.0290],\n",
       "        [ 0.0165, -0.0195, -0.0867],\n",
       "        [ 0.0483, -0.0359, -0.1145],\n",
       "        [-0.0672, -0.0040, -0.0375],\n",
       "        [-0.0382, -0.1638,  0.0187],\n",
       "        [ 0.1266,  0.0538, -0.0691],\n",
       "        [-0.0659,  0.0424,  0.0014],\n",
       "        [ 0.0812, -0.0172,  0.0654],\n",
       "        [-0.1187, -0.0616,  0.0008],\n",
       "        [ 0.0418, -0.0875, -0.0561],\n",
       "        [ 0.0391, -0.0430, -0.1211],\n",
       "        [ 0.0005,  0.0548,  0.0168],\n",
       "        [ 0.0401, -0.0828, -0.3504],\n",
       "        [ 0.1339, -0.1551, -0.0699],\n",
       "        [ 0.0651, -0.0325, -0.0915],\n",
       "        [ 0.0801, -0.1221, -0.1572],\n",
       "        [-0.0789,  0.0051, -0.0269],\n",
       "        [ 0.0656, -0.1043, -0.0020],\n",
       "        [ 0.2726, -0.2533,  0.0113],\n",
       "        [ 0.0544, -0.0541, -0.1026],\n",
       "        [ 0.0419,  0.0089, -0.0245],\n",
       "        [ 0.0569, -0.0760,  0.1136],\n",
       "        [ 0.1541, -0.0537, -0.1347],\n",
       "        [ 0.0807, -0.0260, -0.1289]], device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_state = [sample[0] for sample in batch]\n",
    "batch_state = torch.cat(batch_state)\n",
    "Q.net(batch_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test policy function:\n",
    "epsilon=1\n",
    "Q = Policy(DQN([4,84,84], 3), 3, device, epsilon=epsilon)\n",
    "optimizer = torch.optim.RMSprop(Q.net.parameters(), lr=0.00003)\n",
    "MSE = nn.MSELoss(reduction='sum')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "Q.min_loss = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  0., -1.,  0.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.]], device='cuda:0')\n",
      "tensor(1.1867, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor(8.5487, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor(1.4312, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor(0.7793, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-0.0022, -0.0168, -0.0263, -0.0339,  0.0104, -0.0202, -0.0124, -0.0567],\n",
      "        [ 0.0401, -0.0200, -0.0096, -0.0026, -0.0066, -0.0278,  0.0081,  0.0041],\n",
      "        [ 0.0342, -0.0224,  0.0021, -0.0135, -0.2814, -0.0176, -0.0175, -0.0130],\n",
      "        [-0.0068, -0.0389, -0.0083, -0.0214, -0.0301, -0.0213, -0.0999, -0.0358]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n"
     ]
    }
   ],
   "source": [
    "y = torch.from_numpy(np.array([sample[2] for sample in batch])).float().to(device).view(-1, 1)\n",
    "print(y.view(-1, 8))\n",
    "\n",
    "Q.update_Q(y, batch, verbose=True)\n",
    "    \n",
    "print(Q.get_y_hat(batch).view(-1, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DQN(\n",
       "  (conv): Sequential(\n",
       "    (0): Conv2d(4, 32, kernel_size=(8, 8), stride=(4, 4))\n",
       "    (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "    (3): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2))\n",
       "    (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): ReLU()\n",
       "    (6): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (8): ReLU()\n",
       "  )\n",
       "  (fc): Sequential(\n",
       "    (0): Linear(in_features=3136, out_features=512, bias=True)\n",
       "    (1): ReLU()\n",
       "    (2): Linear(in_features=512, out_features=3, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q.net.train(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.0008]], device='cuda:0', grad_fn=<GatherBackward>)\n"
     ]
    }
   ],
   "source": [
    "print(Q.get_y_hat(batch[0:1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.0008],\n",
      "        [-0.0007]], device='cuda:0', grad_fn=<GatherBackward>)\n"
     ]
    }
   ],
   "source": [
    "print(Q.get_y_hat(batch[0:2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DQN(\n",
       "  (conv): Sequential(\n",
       "    (0): Conv2d(4, 32, kernel_size=(8, 8), stride=(4, 4))\n",
       "    (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "    (3): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2))\n",
       "    (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): ReLU()\n",
       "    (6): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (8): ReLU()\n",
       "  )\n",
       "  (fc): Sequential(\n",
       "    (0): Linear(in_features=3136, out_features=512, bias=True)\n",
       "    (1): ReLU()\n",
       "    (2): Linear(in_features=512, out_features=3, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q.net.train(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.1021]], device='cuda:0', grad_fn=<GatherBackward>)\n"
     ]
    }
   ],
   "source": [
    "print(Q.get_y_hat(batch[0:1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.0259],\n",
      "        [-0.0304]], device='cuda:0', grad_fn=<GatherBackward>)\n"
     ]
    }
   ],
   "source": [
    "print(Q.get_y_hat(batch[0:2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0862, -0.1021,  0.0036]], device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q.net(batch[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test on real case:\n",
    "# Test policy function:\n",
    "epsilon=1\n",
    "Q = Policy(DQN([4,84,84], 3), 3, device, epsilon=epsilon)\n",
    "Q_hat = Policy(DQN([4,84,84], 3), 3, device, epsilon=epsilon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "Q_hat.update_state_dict(Q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.0074,  0.1195, -0.0486,  0.0025,  0.0565,  0.0503,  0.0558,  0.0411],\n",
      "        [-0.0303,  0.0290,  0.0559,  0.0349,  0.0328,  0.0607,  0.0286,  0.0412],\n",
      "        [ 0.1727, -0.0060,  0.0056,  0.1467, -0.9500,  0.0618,  0.0068,  0.0778],\n",
      "        [ 0.0703,  0.0616,  0.0033,  0.0531,  0.0734,  0.1421,  0.0809,  0.0576]],\n",
      "       device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "y = Q.get_target(batch, is_mixed=is_mixed)\n",
    "print(y.view(-1, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.0074,  0.1195, -0.0486,  0.0025,  0.0565,  0.0503,  0.0558,  0.0411],\n",
      "        [-0.0303,  0.0290,  0.0559,  0.0349,  0.0328,  0.0607,  0.0286,  0.0412],\n",
      "        [ 0.1727, -0.0060,  0.0056,  0.1467, -0.9500,  0.0618,  0.0068,  0.0778],\n",
      "        [ 0.0703,  0.0616,  0.0033,  0.0531,  0.0734,  0.1421,  0.0809,  0.0576]],\n",
      "       device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "y = Q_hat.get_target(batch, is_mixed=is_mixed)\n",
    "print(y.view(-1, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "Q.min_loss = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q.max_iter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(1.5458, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor(5.6665, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor(0.7362, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor(0.5512, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor(0.5625, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[-0.0005,  0.1009, -0.0498, -0.1386,  0.0319, -0.1417,  0.0360,  0.0333],\n",
      "        [-0.2061,  0.0067,  0.0690,  0.0159, -0.1185,  0.0783, -0.1370,  0.0264],\n",
      "        [ 0.0090, -0.0061,  0.0127,  0.0120, -0.4373,  0.0805, -0.0067,  0.0854],\n",
      "        [ 0.0663, -0.0550, -0.2034,  0.0606, -0.0536,  0.1214,  0.1101,  0.0725]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n"
     ]
    }
   ],
   "source": [
    "Q.update_Q(y, batch, verbose=True)\n",
    "    \n",
    "print(Q.get_y_hat(batch).view(-1, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "MAX_N_GAMES = 1\n",
    "n_iteration = 1\n",
    "def train_model(n_iteration, verbose=True):\n",
    "    for i in range(n_iteration):\n",
    "        if verbose:\n",
    "            print(i)\n",
    "        batch, is_mixed = D.get_sample()\n",
    "        rewards = [t[2] for t in batch]\n",
    "        if verbose:\n",
    "            print(np.array(rewards).reshape(4,8), is_mixed)\n",
    "        y = Q_hat.get_target(batch, is_mixed=is_mixed).detach()\n",
    "        if verbose:\n",
    "            print(y.view(4,8))\n",
    "        old_y = Q.get_y_hat(batch)\n",
    "        if verbose:\n",
    "            print(old_y.view(4,8))\n",
    "        Q.update_Q(y, batch, verbose)\n",
    "        new_y = Q.get_y_hat(batch)\n",
    "        if verbose:\n",
    "            print(new_y.view(4,8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "Q.max_iter = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "Q.min_loss = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "[[ 0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 1.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0. -1.  0.  0.  0.]] True\n",
      "tensor([[ 0.0165,  0.0141,  0.1223,  0.0340,  0.0078,  0.0432, -0.0014,  0.0141],\n",
      "        [ 0.9500,  0.1047,  0.0335, -0.0104, -0.0546, -0.0117,  0.0117,  0.0191],\n",
      "        [ 0.0254,  0.1386,  0.0619,  0.0143,  0.2000,  0.0937,  0.0309,  0.0577],\n",
      "        [ 0.0843,  0.0747,  0.0938,  0.0463, -0.9500,  0.0887,  0.1153,  0.0701]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.0758, -0.0074, -0.0049, -0.0122,  0.0387,  0.0133,  0.0323, -0.2301],\n",
      "        [ 0.0279,  0.0072, -0.3359, -0.0533, -0.0028, -0.3018,  0.1840, -0.0007],\n",
      "        [-0.0404, -0.0380, -0.0237, -0.0570,  0.0305, -0.0961, -0.1829, -0.2810],\n",
      "        [-0.1721, -0.0180,  0.0097, -0.1717, -0.3964,  0.0032,  0.1510, -0.0014]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n",
      "tensor(1.9272, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor(1.1417, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor(0.6991, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor(0.4821, device='cuda:0', grad_fn=<MseLossBackward>)\n",
      "tensor([[ 0.0127,  0.0114,  0.1158,  0.0650,  0.0583,  0.0525,  0.0140, -0.0313],\n",
      "        [ 0.7962,  0.1124, -0.1629,  0.0010, -0.0432, -0.0213, -0.0549,  0.0053],\n",
      "        [ 0.0220,  0.1194,  0.0194, -0.0153,  0.1944,  0.0550, -0.0184, -0.1426],\n",
      "        [ 0.0419,  0.0746,  0.0567,  0.0232, -0.4464,  0.0989,  0.0884,  0.0580]],\n",
      "       device='cuda:0', grad_fn=<ViewBackward>)\n"
     ]
    }
   ],
   "source": [
    "train_model(1, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "Win 1 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "5\n",
      "Win 6 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "10\n",
      "Win 3 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "15\n",
      "Win 1 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "20\n",
      "Win 1 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "25\n",
      "Win 3 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "30\n",
      "Win 4 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "35\n",
      "Win 1 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "40\n",
      "Win 2 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "45\n",
      "Win 3 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "50\n",
      "Win 4 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "55\n",
      "Win 6 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "60\n",
      "Win 3 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "65\n",
      "Win 5 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "70\n",
      "Win 2 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 12 - 21 Loss \n",
      "75\n",
      "Win 4 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "80\n",
      "Win 2 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "85\n",
      "Win 2 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "90\n",
      "Win 5 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "95\n",
      "Win 1 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "100\n",
      "Win 5 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "105\n",
      "Win 2 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "110\n",
      "Win 7 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "115\n",
      "Win 7 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "120\n",
      "Win 4 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "125\n",
      "Win 4 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "130\n",
      "Win 5 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "135\n",
      "Win 1 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "140\n",
      "Win 4 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "145\n",
      "Win 9 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "150\n",
      "Win 7 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "155\n",
      "Win 5 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "160\n",
      "Win 6 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 9 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "165\n",
      "Win 8 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "170\n",
      "Win 5 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 11 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "175\n",
      "Win 3 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "180\n",
      "Win 2 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "185\n",
      "Win 5 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 5 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "190\n",
      "Win 8 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 11 - 21 Loss \n",
      "195\n",
      "Win 1 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 2 - 21 Loss \n"
     ]
    }
   ],
   "source": [
    "# train model:\n",
    "# looking for value of reward 1 and -1:\n",
    "MAX_N_GAMES = 200\n",
    "n_iteration = 200\n",
    "Q.min_loss = 1\n",
    "Q.max_iter = 4\n",
    "for episode in range(MAX_N_GAMES):\n",
    "    if episode % 5 == 0:\n",
    "        print(episode)\n",
    "    observation = env.reset()\n",
    "    \n",
    "    generate_game(1, D, Q, preprocessor, env, is_e_greedy=True, verbose=False, is_update_epsilon=False)\n",
    "\n",
    "    train_model(n_iteration, verbose=False)\n",
    "        \n",
    "    Q_hat.update_state_dict(Q)\n",
    "    Q.update_epsilon()\n",
    "    Q.update_min_loss()\n",
    "Q.save_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.03621491574616716, 0.1094607247788048)"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q.epsilon, Q.min_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = play(env, Q, preprocessor, render=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon= Q.epsilon\n",
    "Q = Policy(DQN([4,84,84], 3), 3, device, epsilon=epsilon, learning_rate=1e-5)\n",
    "Q_hat = Policy(DQN([4,84,84], 3), 3, device, epsilon=epsilon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "Q_hat.update_state_dict(Q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "Win 7 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 8 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "5\n",
      "Win 12 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "Win 1 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "10\n",
      "Win 11 - 21 Loss \n",
      "Win 15 - 21 Loss \n",
      "Win 4 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 7 - 21 Loss \n",
      "15\n",
      "Win 10 - 21 Loss \n",
      "Win 2 - 21 Loss \n",
      "Win 6 - 21 Loss \n",
      "Win 3 - 21 Loss \n",
      "Win 8 - 21 Loss \n"
     ]
    }
   ],
   "source": [
    "# train model:\n",
    "# looking for value of reward 1 and -1:\n",
    "MAX_N_GAMES = 20\n",
    "n_iteration = 200\n",
    "Q.min_loss = 0.5\n",
    "Q.max_iter = 4\n",
    "for episode in range(MAX_N_GAMES):\n",
    "    if episode % 5 == 0:\n",
    "        print(episode)\n",
    "    observation = env.reset()\n",
    "    \n",
    "    generate_game(1, D, Q, preprocessor, env, is_e_greedy=True, verbose=False, is_update_epsilon=False)\n",
    "\n",
    "    train_model(n_iteration, verbose=False)\n",
    "        \n",
    "    Q_hat.update_state_dict(Q)\n",
    "    Q.update_epsilon()\n",
    "    Q.update_min_loss()\n",
    "Q.save_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = play(env, Q, preprocessor, render=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}