LSTM, BPTT=8.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "import torch.nn as nn\n",
    "import torch\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import deque\n",
    "from skimage.color import rgb2gray\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Load the game environment </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gridworld import gameEnv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADKZJREFUeJzt3V+MHfV5xvHvUxtCQtqAgVouhtpVEAhVwtBVCiKqWsAtIRH0IkKgqIoqJG7SFppICbRXkXqRSFUSLqpICJKiivInBBpkRaSuQ1RVqhyWP03AhtgQE2wBNimUlEptnby9mHG7cW086z3n7A6/70danTMz52h+4/FzZs7s7PumqpDUll9Y7gFImj2DLzXI4EsNMvhSgwy+1CCDLzXI4EsNWlLwk1yZ5Lkku5PcMqlBSZquHO8NPElWAT8ANgN7gceA66tqx+SGJ2kaVi/hvR8AdlfVCwBJ7gWuAY4a/NNPP702bNiwhFVKejt79uzhtddey7Fet5Tgnwm8tGB6L/Cbb/eGDRs2MD8/v4RVSno7c3Nzg1439Yt7SW5MMp9k/sCBA9NenaQBlhL8fcBZC6bX9/N+TlXdXlVzVTV3xhlnLGF1kiZlKcF/DDgnycYkJwLXAQ9PZliSpum4v+NX1cEkfwR8C1gFfKWqnpnYyCRNzVIu7lFV3wS+OaGxSJoR79yTGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGnTM4Cf5SpL9SZ5eMG9Nkq1JdvWPp053mJImacgR/6+BKw+bdwuwrarOAbb105JG4pjBr6p/BP71sNnXAHf1z+8Cfn/C45I0Rcf7HX9tVb3cP38FWDuh8UiagSVf3Kuu6+ZRO2/aSUdaeY43+K8mWQfQP+4/2gvtpCOtPMcb/IeBj/fPPw58YzLDkTQLQ36ddw/wz8C5SfYmuQH4HLA5yS7gin5a0kgcs5NOVV1/lEWXT3gskmbEO/ekBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBg0pvXVWkkeT7EjyTJKb+vl205FGasgR/yDwqao6H7gY+ESS87GbjjRaQzrpvFxVT/TPfwLsBM7EbjrSaC3qO36SDcCFwHYGdtOxoYa08gwOfpL3Al8Hbq6qNxcue7tuOjbUkFaeQcFPcgJd6O+uqgf72YO76UhaWYZc1Q9wJ7Czqr6wYJHddKSROmZDDeBS4A+A7yd5qp/3Z3Tdc+7vO+u8CFw7nSFKmrQhnXT+CchRFttNRxoh79yTGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYN+Xt8zVCO9gfQ6vkPNAke8aUGGXypQUNq7p2U5LtJ/qXvpPPZfv7GJNuT7E5yX5ITpz9cSZMw5Ij/n8BlVXUBsAm4MsnFwOeBL1bV+4HXgRumN0xJkzSkk05V1b/3kyf0PwVcBjzQz7eTjjQiQ+vqr+or7O4HtgLPA29U1cH+JXvp2mod6b120pFWmEHBr6qfVtUmYD3wAeC8oSuwk4608izqqn5VvQE8ClwCnJLk0H0A64F9Ex6bpCkZclX/jCSn9M/fDWym65j7KPDR/mV20pFGZMide+uAu5KsovuguL+qtiTZAdyb5C+AJ+nabEkagSGddL5H1xr78Pkv0H3flzQy3rknNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNWhw8PsS208m2dJP20lHGqnFHPFvoiuyeYiddKSRGtpQYz3wYeCOfjrYSUcaraFH/C8BnwZ+1k+fhp10pNEaUlf/I8D+qnr8eFZgJx1p5RlSV/9S4OokVwEnAb8E3EbfSac/6ttJRxqRId1yb62q9VW1AbgO+HZVfQw76UijtZTf438G+GSS3XTf+e2kI43EkFP9/1VV3wG+0z+3k440Ut65JzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1KBBhTiS7AF+AvwUOFhVc0nWAPcBG4A9wLVV9fp0hilpkhZzxP+dqtpUVXP99C3Atqo6B9jWT0sagaWc6l9D10gDbKghjcrQ4Bfw90keT3JjP29tVb3cP38FWDvx0UmaiqHFNj9YVfuS/DKwNcmzCxdWVSWpI72x/6C4EeDss89e0mAlTcagI35V7esf9wMP0VXXfTXJOoD+cf9R3msnHWmFGdJC6+Qkv3joOfC7wNPAw3SNNMCGGtKoDDnVXws81DXIZTXwt1X1SJLHgPuT3AC8CFw7vWFKmqRjBr9vnHHBEeb/GLh8GoOSNF3euSc1yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81aOhf52lmstwDUAM84ksNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1KBBwU9ySpIHkjybZGeSS5KsSbI1ya7+8dRpD1bSZAw94t8GPFJV59GV4dqJnXSk0RpSZfd9wG8BdwJU1X9V1RvYSUcarSFH/I3AAeCrSZ5MckdfZttOOtJIDQn+auAi4MtVdSHwFoed1ldV0bXZ+n+S3JhkPsn8gQMHljpeSRMwJPh7gb1Vtb2ffoDug8BOOtJIHTP4VfUK8FKSc/tZlwM7sJOONFpD/yz3j4G7k5wIvAD8Id2Hhp10pBEaFPyqegqYO8IiO+lII+Sde1KDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDhtTVPzfJUwt+3kxys510pPEaUmzzuaraVFWbgN8A/gN4CDvpSKO12FP9y4Hnq+pF7KQjjdZig38dcE//3E460kgNDn5fWvtq4GuHL7OTjjQuiznifwh4oqpe7aftpCON1GKCfz3/d5oPdtKRRmtQ8PvuuJuBBxfM/hywOcku4Ip+WtIIDO2k8xZw2mHzfoyddKRR8s49qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUFDS2/9aZJnkjyd5J4kJyXZmGR7kt1J7uur8EoagSEttM4E/gSYq6pfB1bR1df/PPDFqno/8DpwwzQHKmlyhp7qrwbenWQ18B7gZeAy4IF+uZ10pBEZ0jtvH/CXwI/oAv9vwOPAG1V1sH/ZXuDMaQ1S0mQNOdU/la5P3kbgV4CTgSuHrsBOOtLKM+RU/wrgh1V1oKr+m662/qXAKf2pP8B6YN+R3mwnHWnlGRL8HwEXJ3lPktDV0t8BPAp8tH+NnXSkERnyHX873UW8J4Dv9++5HfgM8Mkku+mabdw5xXFKmqB0jW5nY25urubn52e2vjHqTqqk41dVx/xP5J17UoMMvtQggy81yOBLDZrpxb0kB4C3gNdmttLpOx23Z6V6J20LDNueX62qY94wM9PgAySZr6q5ma50ityeleudtC0w2e3xVF9qkMGXGrQcwb99GdY5TW7PyvVO2haY4PbM/Du+pOXnqb7UoJkGP8mVSZ7r6/TdMst1L1WSs5I8mmRHX3/wpn7+miRbk+zqH09d7rEuRpJVSZ5MsqWfHm0txSSnJHkgybNJdia5ZMz7Z5q1LmcW/CSrgL8CPgScD1yf5PxZrX8CDgKfqqrzgYuBT/TjvwXYVlXnANv66TG5Cdi5YHrMtRRvAx6pqvOAC+i2a5T7Z+q1LqtqJj/AJcC3FkzfCtw6q/VPYXu+AWwGngPW9fPWAc8t99gWsQ3r6cJwGbAFCN0NIquPtM9W8g/wPuCH9NetFswf5f6hK2X3ErCGrublFuD3JrV/Znmqf2hDDhltnb4kG4ALge3A2qp6uV/0CrB2mYZ1PL4EfBr4WT99GuOtpbgROAB8tf/qckeSkxnp/qkp17r04t4iJXkv8HXg5qp6c+Gy6j6GR/FrkiQfAfZX1ePLPZYJWQ1cBHy5qi6kuzX8507rR7Z/llTr8lhmGfx9wFkLpo9ap2+lSnICXejvrqoH+9mvJlnXL18H7F+u8S3SpcDVSfYA99Kd7t/GwFqKK9BeYG91FaOgqxp1EePdP0uqdXksswz+Y8A5/VXJE+kuVDw8w/UvSV9v8E5gZ1V9YcGih+lqDsKIag9W1a1Vtb6qNtDti29X1ccYaS3FqnoFeCnJuf2sQ7UhR7l/mHatyxlfsLgK+AHwPPDny30BZZFj/yDdaeL3gKf6n6vovhdvA3YB/wCsWe6xHse2/TawpX/+a8B3gd3A14B3Lff4FrEdm4D5fh/9HXDqmPcP8FngWeBp4G+Ad01q/3jnntQgL+5JDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy816H8AYafimUS73ewAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "env = gameEnv(partial=True,size=9)\n",
    "prev_state = env.reset()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Set Device </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 align='center'> Training Deep Recurrent Q Network (LSTM) </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Hyper-parameters </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_IMAGE_DIM = 84\n",
    "OUT_SIZE = 4\n",
    "BATCH_SIZE = 32\n",
    "TIME_STEP = 8\n",
    "GAMMA = 0.99\n",
    "INITIAL_EPSILON = 1.0\n",
    "FINAL_EPSILON = 0.1\n",
    "TOTAL_EPSIODES = 20000\n",
    "MAX_STEPS = 50\n",
    "MEMORY_SIZE = 3000\n",
    "UPDATE_FREQ = 5\n",
    "PERFORMANCE_SAVE_INTERVAL = 500\n",
    "TARGET_UPDATE_FREQ = 20000 #steps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Build Model </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Network(\n",
      "  (conv_layer1): Conv2d(1, 32, kernel_size=(8, 8), stride=(4, 4))\n",
      "  (conv_layer2): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2))\n",
      "  (conv_layer3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1))\n",
      "  (conv_layer4): Conv2d(64, 512, kernel_size=(7, 7), stride=(1, 1))\n",
      "  (lstm_layer): LSTM(512, 512, batch_first=True)\n",
      "  (adv): Linear(in_features=512, out_features=4, bias=True)\n",
      "  (val): Linear(in_features=512, out_features=1, bias=True)\n",
      "  (relu): ReLU()\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "class Network(nn.Module):\n",
    "    \n",
    "    def __init__(self,input_size,out_size):\n",
    "        super(Network,self).__init__()\n",
    "        self.input_size = input_size\n",
    "        self.out_size = out_size\n",
    "        \n",
    "        self.conv_layer1 = nn.Conv2d(in_channels=1,out_channels=32,kernel_size=8,stride=4) # potential check - in_channels\n",
    "        self.conv_layer2 = nn.Conv2d(in_channels=32,out_channels=64,kernel_size=4,stride=2)\n",
    "        self.conv_layer3 = nn.Conv2d(in_channels=64,out_channels=64,kernel_size=3,stride=1)\n",
    "        self.conv_layer4 = nn.Conv2d(in_channels=64,out_channels=512,kernel_size=7,stride=1)\n",
    "        self.lstm_layer = nn.LSTM(input_size=512,hidden_size=512,num_layers=1,batch_first=True)\n",
    "        self.adv = nn.Linear(in_features=512,out_features=self.out_size)\n",
    "        self.val = nn.Linear(in_features=512,out_features=1)\n",
    "        self.relu = nn.ReLU()\n",
    "        \n",
    "    def forward(self,x,bsize,time_step,hidden_state,cell_state):\n",
    "        x = x.view(bsize*time_step,1,self.input_size,self.input_size)\n",
    "        \n",
    "        conv_out = self.conv_layer1(x)\n",
    "        conv_out = self.relu(conv_out)\n",
    "        conv_out = self.conv_layer2(conv_out)\n",
    "        conv_out = self.relu(conv_out)\n",
    "        conv_out = self.conv_layer3(conv_out)\n",
    "        conv_out = self.relu(conv_out)\n",
    "        conv_out = self.conv_layer4(conv_out)\n",
    "        conv_out = self.relu(conv_out)\n",
    "        \n",
    "        conv_out = conv_out.view(bsize,time_step,512)\n",
    "        \n",
    "        lstm_out = self.lstm_layer(conv_out,(hidden_state,cell_state))\n",
    "        out = lstm_out[0][:,time_step-1,:]\n",
    "        h_n = lstm_out[1][0]\n",
    "        c_n = lstm_out[1][1]\n",
    "        \n",
    "        adv_out = self.adv(out)\n",
    "        val_out = self.val(out)\n",
    "        \n",
    "        qout = val_out.expand(bsize,self.out_size) + (adv_out - adv_out.mean(dim=1).unsqueeze(dim=1).expand(bsize,self.out_size))\n",
    "        \n",
    "        return qout, (h_n,c_n)\n",
    "    \n",
    "    def init_hidden_states(self,bsize):\n",
    "        h = torch.zeros(1,bsize,512).float().to(device)\n",
    "        c = torch.zeros(1,bsize,512).float().to(device)\n",
    "        \n",
    "        return h,c\n",
    "    \n",
    "main_model = Network(input_size=INPUT_IMAGE_DIM,out_size=OUT_SIZE).to(device)\n",
    "print(main_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Experience Replay </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Memory():\n",
    "    \n",
    "    def __init__(self,memsize):\n",
    "        self.memsize = memsize\n",
    "        self.memory = deque(maxlen=self.memsize)\n",
    "    \n",
    "    def add_episode(self,epsiode):\n",
    "        self.memory.append(epsiode)\n",
    "    \n",
    "    def get_batch(self,bsize,time_step):\n",
    "        sampled_epsiodes = random.sample(self.memory,bsize)\n",
    "        batch = []\n",
    "        for episode in sampled_epsiodes:\n",
    "            point = np.random.randint(0,len(episode)+1-time_step)\n",
    "            batch.append(episode[point:point+time_step])\n",
    "        return batch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Preprocess Image </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_image(image):\n",
    "    return rgb2gray(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fd367714080>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADLhJREFUeJzt3V2MXPV5x/Hvr14ICUljm7SWi0kxigVCVTGRlYLggpLSEhpBLqIUlEhpldY3qUraSsG0Fy2VIiVSlYSLKpIFSVGV8hKHJhYXSV2HpL1ysDFtwcbBJBBs+YUKyNsFqsPTizluF7p4zu7O7O7h//1Iq5lz5uX8j45+c15m9nlSVUhqyy8s9wAkLT2DLzXI4EsNMvhSgwy+1CCDLzXI4EsNWlTwk1yf5FCSw0m2TWpQkqYrC/0BT5JVwPeA64AjwCPALVV1YHLDkzQNM4t47XuAw1X1fYAk9wE3Aa8b/CT+TFCasqrKuOcs5lD/fOC5WdNHunmSVrjF7PF7SbIV2Drt5UjqbzHBPwpcMGt6QzfvVapqO7AdPNSXVorFHOo/AmxKsjHJ2cDNwM7JDEvSNC14j19Vp5L8MfBNYBXwxap6YmIjkzQ1C/46b0EL81BfmrppX9WXNFAGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUFjg5/ki0lOJnl81ry1SXYleaq7XTPdYUqapD57/L8Hrn/NvG3A7qraBOzupiUNxNjgV9W/Ai+8ZvZNwD3d/XuAD0x4XJKmaKHn+Ouq6lh3/ziwbkLjkbQEFt1Jp6rqTNVz7aQjrTwL3eOfSLIeoLs9+XpPrKrtVbWlqrYscFmSJmyhwd8JfLS7/1Hg65MZjqSlMLahRpJ7gWuAdwAngL8CvgY8ALwTeBb4UFW99gLgXO9lQw1pyvo01LCTjvQGYycdSXMy+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw3q00nngiQPJzmQ5Ikkt3bz7aYjDVSfmnvrgfVV9WiStwH7GDXQ+H3ghar6dJJtwJqqum3Me1l6S5qyiZTeqqpjVfVod/8nwEHgfOymIw3WvBpqJLkQuBzYQ89uOjbUkFae3lV2k7wV+A7wqap6MMlLVbV61uMvVtUZz/M91Jemb2JVdpOcBXwV+HJVPdjN7t1NR9LK0ueqfoC7gYNV9dlZD9lNRxqoPlf1rwb+DfhP4JVu9l8wOs+fVzcdD/Wl6bOTjtQgO+lImpPBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxo0r5p7mr6l/DfpIRrVhdFiuceXGmTwpQb1qbl3TpLvJvn3rpPOHd38jUn2JDmc5P4kZ09/uJImoc8e/2Xg2qq6DNgMXJ/kCuAzwOeq6l3Ai8DHpjdMSZPUp5NOVdVPu8mzur8CrgV2dPPtpCMNSN+6+quSPMaodv4u4Gngpao61T3lCKO2WnO9dmuSvUn2TmLAkhavV/Cr6udVtRnYALwHuKTvAqpqe1VtqaotCxyjpAmb11X9qnoJeBi4Elid5PTvADYARyc8NklT0ueq/i8lWd3dfzNwHaOOuQ8DH+yeZicdaUD6dNL5dUYX71Yx+qB4oKr+JslFwH3AWmA/8JGqennMe/mztDH85d6Z+cu98eykM0AG/8wM/nh20pE0J4MvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UoN7B70ps70/yUDdtJx1poOazx7+VUZHN0+ykIw1U34YaG4DfBe7qpoOddKTB6rvH/zzwSeCVbvo87KQjDVafuvrvB05W1b6FLMBOOtLKMzP+KVwF3JjkBuAc4BeBO+k66XR7fTvpSAPSp1vu7VW1oaouBG4GvlVVH8ZOOtJgLeZ7/NuAP0tymNE5/92TGZKkabOTzgpjJ50zs5POeHbSkTQngy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoP61NwjyTPAT4CfA6eqakuStcD9wIXAM8CHqurF6QxT0iTNZ4//m1W1eVa13G3A7qraBOzupiUNwGIO9W9i1EgDbKghDUrf4Bfwz0n2JdnazVtXVce6+8eBdRMfnaSp6HWOD1xdVUeT/DKwK8mTsx+sqnq9QprdB8XWuR6TtDzmXWU3yV8DPwX+CLimqo4lWQ98u6ouHvNaS8iOYZXdM7PK7ngTqbKb5Nwkbzt9H/ht4HFgJ6NGGmBDDWlQxu7xk1wE/FM3OQP8Y1V9Ksl5wAPAO4FnGX2d98KY93J3NoZ7/DNzjz9enz2+DTVWGIN/ZgZ/PBtqSJqTwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2pQ3//O0xLxl2laCu7xpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qUK/gJ1mdZEeSJ5McTHJlkrVJdiV5qrtdM+3BSpqMvnv8O4FvVNUlwGXAQeykIw1Wn2KbbwceAy6qWU9OcgjLa0srzqRq7m0Enge+lGR/kru6Mtt20pEGqk/wZ4B3A1+oqsuBn/Gaw/ruSOB1O+kk2Ztk72IHK2ky+gT/CHCkqvZ00zsYfRCc6A7x6W5PzvXiqtpeVVtmddmVtMzGBr+qjgPPJTl9/v5e4AB20pEGq1dDjSSbgbuAs4HvA3/A6EPDTjrSCmMnHalBdtKRNCeDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1KCxwU9ycZLHZv39OMkn7KQjDde8Sm8lWQUcBX4D+DjwQlV9Osk2YE1V3Tbm9ZbekqZsGqW33gs8XVXPAjcB93Tz7wE+MM/3krRM5hv8m4F7u/t20pEGqnfwk5wN3Ah85bWP2UlHGpb57PHfBzxaVSe6aTvpSAM1n+Dfwv8d5oOddKTB6ttJ51zgh4xaZf+om3cedtKRVhw76UgNspOOpDkZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQb1Cn6SP03yRJLHk9yb5JwkG5PsSXI4yf1dFV5JA9Cnhdb5wJ8AW6rq14BVjOrrfwb4XFW9C3gR+Ng0Byppcvoe6s8Ab04yA7wFOAZcC+zoHreTjjQgY4NfVUeBv2VUZfcY8CNgH/BSVZ3qnnYEOH9ag5Q0WX0O9dcw6pO3EfgV4Fzg+r4LsJOOtPLM9HjObwE/qKrnAZI8CFwFrE4y0+31NzDqovv/VNV2YHv3WstrSytAn3P8HwJXJHlLkjDqmHsAeBj4YPccO+lIA9K3k84dwO8Bp4D9wB8yOqe/D1jbzftIVb085n3c40tTZicdqUF20pE0J4MvNcjgSw0y+FKD+nyPP0n/Bfysu32jeAeuz0r1RloX6Lc+v9rnjZb0qj5Akr1VtWVJFzpFrs/K9UZaF5js+nioLzXI4EsNWo7gb1+GZU6T67NyvZHWBSa4Pkt+ji9p+XmoLzVoSYOf5Pokh7o6fduWctmLleSCJA8nOdDVH7y1m782ya4kT3W3a5Z7rPORZFWS/Uke6qYHW0sxyeokO5I8meRgkiuHvH2mWetyyYKfZBXwd8D7gEuBW5JculTLn4BTwJ9X1aXAFcDHu/FvA3ZX1SZgdzc9JLcCB2dND7mW4p3AN6rqEuAyRus1yO0z9VqXVbUkf8CVwDdnTd8O3L5Uy5/C+nwduA44BKzv5q0HDi332OaxDhsYheFa4CEgjH4gMjPXNlvJf8DbgR/QXbeaNX+Q24fRv70/x+jf3me67fM7k9o+S3mof3pFThtsnb4kFwKXA3uAdVV1rHvoOLBumYa1EJ8HPgm80k2fx3BrKW4Enge+1J263JXkXAa6fWrKtS69uDdPSd4KfBX4RFX9ePZjNfoYHsTXJEneD5ysqn3LPZYJmQHeDXyhqi5n9NPwVx3WD2z7LKrW5ThLGfyjwAWzpl+3Tt9KleQsRqH/clU92M0+kWR99/h64ORyjW+ergJuTPIMo0pK1zI6R17dlVGHYW2jI8CRqtrTTe9g9EEw1O3zv7Uuq+q/gVfVuuyes+Dts5TBfwTY1F2VPJvRhYqdS7j8RenqDd4NHKyqz856aCejmoMwoNqDVXV7VW2oqgsZbYtvVdWHGWgtxao6DjyX5OJu1unakIPcPky71uUSX7C4Afge8DTwl8t9AWWeY7+a0WHifwCPdX83MDov3g08BfwLsHa5x7qAdbsGeKi7fxHwXeAw8BXgTcs9vnmsx2Zgb7eNvgasGfL2Ae4AngQeB/4BeNOkto+/3JMa5MU9qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBv0PANIhuBFMr1IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(preprocess_image(prev_state),cmap='gray')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Save Dictionary Function </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def save_obj(obj, name ):\n",
    "    with open('data/'+ name + '.pkl', 'wb') as f:\n",
    "        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> DQN With LSTM </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populated with 3000 Episodes\n"
     ]
    }
   ],
   "source": [
    "mem = Memory(memsize=MEMORY_SIZE)\n",
    "\n",
    "main_model = Network(input_size=INPUT_IMAGE_DIM,out_size=OUT_SIZE).float().to(device)\n",
    "target_model = Network(input_size=INPUT_IMAGE_DIM,out_size=OUT_SIZE).float().to(device)\n",
    "\n",
    "target_model.load_state_dict(main_model.state_dict())\n",
    "criterion = nn.SmoothL1Loss()\n",
    "optimizer = torch.optim.Adam(main_model.parameters(),lr=0.00025)\n",
    "\n",
    "\n",
    "# Fill memory\n",
    "for i in range(0,MEMORY_SIZE):\n",
    "    \n",
    "    prev_state = env.reset()\n",
    "    processed_prev_state = preprocess_image(prev_state)\n",
    "    step_count = 0\n",
    "    local_memory = []\n",
    "    \n",
    "    while step_count < MAX_STEPS:\n",
    "        \n",
    "        step_count +=1\n",
    "        action = np.random.randint(0,4)\n",
    "        next_state,reward,done = env.step(action)\n",
    "        processed_next_state = preprocess_image(next_state)\n",
    "        \n",
    "        local_memory.append((processed_prev_state,action,reward,processed_next_state))\n",
    "        \n",
    "        prev_state = next_state\n",
    "        processed_prev_state = processed_next_state\n",
    "    \n",
    "    mem.add_episode(local_memory)\n",
    "        \n",
    "print('Populated with %d Episodes'%(len(mem.memory)))\n",
    "\n",
    "# Start Algorithm\n",
    "epsilon = INITIAL_EPSILON\n",
    "loss_stat = []\n",
    "reward_stat = []\n",
    "total_steps = 0\n",
    "\n",
    "for episode in range(0,TOTAL_EPSIODES):\n",
    "    \n",
    "    prev_state = env.reset()\n",
    "    processed_prev_state = preprocess_image(prev_state)\n",
    "    total_reward = 0\n",
    "    step_count = 0\n",
    "    local_memory = []\n",
    "\n",
    "    hidden_state, cell_state = main_model.init_hidden_states(bsize=1)\n",
    "    \n",
    "    while step_count < MAX_STEPS:\n",
    "        \n",
    "        step_count +=1\n",
    "        total_steps +=1\n",
    "        \n",
    "        if np.random.rand(1) < epsilon:\n",
    "            torch_x = torch.from_numpy(processed_prev_state).float().to(device)\n",
    "            model_out = main_model.forward(torch_x,bsize=1,time_step=1,hidden_state=hidden_state,cell_state=cell_state)\n",
    "            action = np.random.randint(0,4)\n",
    "            hidden_state = model_out[1][0]\n",
    "            cell_state = model_out[1][1]\n",
    "            \n",
    "        else:\n",
    "            torch_x = torch.from_numpy(processed_prev_state).float().to(device)\n",
    "            model_out = main_model.forward(torch_x,bsize=1,time_step=1,hidden_state=hidden_state,cell_state=cell_state)\n",
    "            out = model_out[0]\n",
    "            action = int(torch.argmax(out[0]))\n",
    "            hidden_state = model_out[1][0]\n",
    "            cell_state = model_out[1][1]\n",
    "        \n",
    "        next_state,reward,done = env.step(action)\n",
    "        total_reward += reward\n",
    "        processed_next_state = preprocess_image(next_state)\n",
    "        \n",
    "        local_memory.append((processed_prev_state,action,reward,processed_next_state))\n",
    "        \n",
    "        prev_state = next_state\n",
    "        processed_prev_state = processed_next_state\n",
    "        \n",
    "        \n",
    "        if (total_steps % TARGET_UPDATE_FREQ) == 0:\n",
    "            target_model.load_state_dict(main_model.state_dict())\n",
    "       \n",
    "        if (total_steps % UPDATE_FREQ) == 0:\n",
    "            \n",
    "            hidden_batch, cell_batch = main_model.init_hidden_states(bsize=BATCH_SIZE)\n",
    "            \n",
    "            batch = mem.get_batch(bsize=BATCH_SIZE,time_step=TIME_STEP)\n",
    "            \n",
    "            current_states = []\n",
    "            acts = []\n",
    "            rewards = []\n",
    "            next_states = []\n",
    "            \n",
    "            for b in batch:\n",
    "                cs,ac,rw,ns = [],[],[],[]\n",
    "                for element in b:\n",
    "                    cs.append(element[0])\n",
    "                    ac.append(element[1])\n",
    "                    rw.append(element[2])\n",
    "                    ns.append(element[3])\n",
    "                current_states.append(cs)\n",
    "                acts.append(ac)\n",
    "                rewards.append(rw)\n",
    "                next_states.append(ns)\n",
    "            \n",
    "            current_states = np.array(current_states)\n",
    "            acts = np.array(acts)\n",
    "            rewards = np.array(rewards)\n",
    "            next_states = np.array(next_states)\n",
    "            \n",
    "            torch_current_states = torch.from_numpy(current_states).float().to(device)\n",
    "            torch_acts = torch.from_numpy(acts).long().to(device)\n",
    "            torch_rewards = torch.from_numpy(rewards).float().to(device)\n",
    "            torch_next_states = torch.from_numpy(next_states).float().to(device)\n",
    "            \n",
    "            \n",
    "            Q_next,_ = target_model.forward(torch_next_states,bsize=BATCH_SIZE,time_step=TIME_STEP,hidden_state=hidden_batch,cell_state=cell_batch)\n",
    "            Q_next_max,__ = Q_next.detach().max(dim=1)\n",
    "            target_values = torch_rewards[:,TIME_STEP-1] + (GAMMA * Q_next_max)\n",
    "            \n",
    "            Q_s, _ = main_model.forward(torch_current_states,bsize=BATCH_SIZE,time_step=TIME_STEP,hidden_state=hidden_batch,cell_state=cell_batch)\n",
    "            Q_s_a = Q_s.gather(dim=1,index=torch_acts[:,TIME_STEP-1].unsqueeze(dim=1)).squeeze(dim=1)\n",
    "            \n",
    "            \n",
    "            loss = criterion(Q_s_a,target_values)\n",
    "            \n",
    "            #  save performance measure\n",
    "            loss_stat.append(loss.item())\n",
    "            \n",
    "            # make previous grad zero\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            # backward\n",
    "            loss.backward()\n",
    "            \n",
    "            # update params\n",
    "            optimizer.step()\n",
    "\n",
    "    # save performance measure\n",
    "    reward_stat.append(total_reward)\n",
    "    \n",
    "    mem.add_episode(local_memory)\n",
    "\n",
    "    if epsilon > FINAL_EPSILON:\n",
    "        epsilon -= (INITIAL_EPSILON - FINAL_EPSILON)/TOTAL_EPSIODES\n",
    "    \n",
    "    if (episode + 1)% PERFORMANCE_SAVE_INTERVAL == 0:\n",
    "        perf = {}\n",
    "        perf['loss'] = loss_stat\n",
    "        perf['total_reward'] = reward_stat\n",
    "        save_obj(name='LSTM_POMDP_V4',obj=perf)\n",
    "    \n",
    "    #print('Episode : ',episode+1,'Epsilon : ',epsilon,'Reward : ',total_reward,)\n",
    "            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Save Primary Network Weights </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "torch.save(main_model.state_dict(),'data/LSTM_POMDP_V4_WEIGHTS.torch')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 align='center'> Testing Policy </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Load Primary Network Weights </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = torch.load('data/LSTM_POMDP_V4_WEIGHTS.torch', map_location='cpu')\n",
    "main_model.load_state_dict(weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Start Testing\n",
    "epsilon = INITIAL_EPSILON\n",
    "FINAL_EPSILON = 0.01\n",
    "TOTAL_EPSIODES = 10000\n",
    "loss_stat = []\n",
    "reward_stat = []\n",
    "total_steps = 0\n",
    "\n",
    "for episode in range(0,TOTAL_EPSIODES):\n",
    "    \n",
    "    prev_state = env.reset()\n",
    "    processed_prev_state = preprocess_image(prev_state)\n",
    "    total_reward = 0\n",
    "    step_count = 0\n",
    "    \n",
    "    hidden_state, cell_state = main_model.init_hidden_states(bsize=1)\n",
    "    \n",
    "    while step_count < MAX_STEPS:\n",
    "        \n",
    "        step_count +=1\n",
    "        total_steps +=1\n",
    "        \n",
    "        if np.random.rand(1) < epsilon:\n",
    "            torch_x = torch.from_numpy(processed_prev_state).float().to(device)\n",
    "            model_out = main_model.forward(torch_x,bsize=1,time_step=1,hidden_state=hidden_state,cell_state=cell_state)\n",
    "            action = np.random.randint(0,4)\n",
    "            hidden_state = model_out[1][0]\n",
    "            cell_state = model_out[1][1]\n",
    "            \n",
    "        else:\n",
    "            torch_x = torch.from_numpy(processed_prev_state).float().to(device)\n",
    "            model_out = main_model.forward(torch_x,bsize=1,time_step=1,hidden_state=hidden_state,cell_state=cell_state)\n",
    "            out = model_out[0]\n",
    "            action = int(torch.argmax(out[0]))\n",
    "            hidden_state = model_out[1][0]\n",
    "            cell_state = model_out[1][1]\n",
    "        \n",
    "        next_state,reward,done = env.step(action)\n",
    "        total_reward += reward\n",
    "        processed_next_state = preprocess_image(next_state)\n",
    "        \n",
    "        prev_state = next_state\n",
    "        processed_prev_state = processed_next_state\n",
    "            \n",
    "    # save performance measure\n",
    "    reward_stat.append(total_reward)\n",
    "    \n",
    "    if epsilon > FINAL_EPSILON:\n",
    "        epsilon -= (INITIAL_EPSILON - FINAL_EPSILON)/TOTAL_EPSIODES\n",
    "    \n",
    "    if (episode + 1)% PERFORMANCE_SAVE_INTERVAL == 0:\n",
    "        perf = {}\n",
    "        perf['loss'] = loss_stat\n",
    "        perf['total_reward'] = reward_stat\n",
    "        save_obj(name='LSTM_POMDP_V4_TEST',obj=perf)\n",
    "    \n",
    "    print('Episode : ',episode+1,'Epsilon : ',epsilon,'Reward : ',total_reward)\n",
    "            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 align='center'> Create Policy GIF </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Collect Frames Of an Episode Using Trained Network </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADOBJREFUeJzt3X/oXfV9x/Hna4nW1m7VqAuZ0X0zKooMjC44xTI2NZu1RfdHEaWMMgT/6TZdC61uf5TC/mhhtPWPURBtJ8P5o1bXEIqdSy1jMFLjj7WaaBNtrAlqYqezc7At7Xt/nBP2bZqY88333vv9nnyeD7jce869N+dzOHndc+655/t+p6qQ1JZfWuoBSJo9gy81yOBLDTL4UoMMvtQggy81yOBLDVpU8JNcleT5JLuS3DqpQUmarhzrBTxJVgA/ADYCe4DHgRuqavvkhidpGlYu4r0XA7uq6kWAJPcB1wJHDP7pp59ec3Nzi1ikpHeye/duXn/99RztdYsJ/pnAy/Om9wC//U5vmJubY9u2bYtYpKR3smHDhkGvm/rJvSQ3JdmWZNv+/funvThJAywm+HuBs+ZNr+3n/ZyquqOqNlTVhjPOOGMRi5M0KYsJ/uPAOUnWJTkRuB7YNJlhSZqmY/6OX1UHkvwJ8C1gBfCVqnp2YiOTNDWLOblHVX0T+OaExiJpRrxyT2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcatKg/y10OkqPWFZSWraVqU+8eX2qQwZcadNTgJ/lKkn1Jnpk3b1WSR5Ps7O9Pne4wJU3SkD3+3wJXHTLvVmBLVZ0DbOmnJY3EUYNfVf8M/Pshs68F7u4f3w384YTHJWmKjvU7/uqqeqV//CqwekLjkTQDiz65V93vEUf8TcJOOtLyc6zBfy3JGoD+ft+RXmgnHWn5OdbgbwI+1j/+GPCNyQxH0iwM+TnvXuBfgXOT7ElyI/A5YGOSncCV/bSkkTjqJbtVdcMRnrpiwmORNCNeuSc1yOBLDTL4UoMMvtQggy81yOBLDRp9BZ53uFp4gazkc9yaRpGbkf93cY8vNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0aUnrrrCSPJdme5NkkN/fz7aYjjdSQPf4B4JNVdT5wCfDxJOdjNx1ptIZ00nmlqp7sH/8E2AGcid10pNFa0Hf8JHPAhcBWBnbTsaGGtPwMDn6S9wJfB26pqrfmP/dO3XRsqCEtP4OCn+QEutDfU1UP9bMHd9ORtLwMOasf4C5gR1V9Yd5TdtORRmpIBZ7LgD8Cvp/k6X7eX9B1z3mg76zzEnDddIYoadKGdNL5F45caMhuOtIIeeWe1KDjoNjmyKseSkvAPb7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0aUnPvpCTfTfJvfSedz/bz1yXZmmRXkvuTnDj94UqahCF7/P8GLq+qC4D1wFVJLgE+D3yxqt4PvAHcOL1hSpqkIZ10qqr+s588ob8VcDnwYD/fTjrSiAytq7+ir7C7D3gUeAF4s6oO9C/ZQ9dW63DvtZOOtMwMCn5V/bSq1gNrgYuB84YuYDSddGoKNy0PmcJt5BZ0Vr+q3gQeAy4FTklysFjnWmDvhMcmaUqGnNU/I8kp/eN3AxvpOuY+Bnykf5mddKQRGVJeew1wd5IVdB8UD1TV5iTbgfuS/BXwFF2bLUkjMKSTzvfoWmMfOv9Fuu/7kkbGK/ekBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUFD/h6/DcdBOSVpKPf4UoMMvtSgwcHvS2w/lWRzP20nHWmkFrLHv5muyOZBdtKRRmpoQ421wIeAO/vpYCcdabSG7vG/BHwK+Fk/fRp20pFGa0hd/Q8D+6rqiWNZwGg66UgNGfI7/mXANUmuBk4CfgW4nb6TTr/Xt5OONCJDuuXeVlVrq2oOuB74dlV9FDvpSKO1mN/xPw18Iskuuu/8dtKRRmJBl+xW1XeA7/SP7aQjjZRX7kkNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXImntadmrC/57lFH+Re3ypQQZfapDBlxpk8KUGGXypQQZfapDBlxo06Hf8JLuBnwA/BQ5U1YYkq4D7gTlgN3BdVb0xnWFKmqSF7PF/r6rWV9WGfvpWYEtVnQNs6acljcBiDvWvpWukATbUkEZlaPAL+MckTyS5qZ+3uqpe6R+/Cqye+OgkTcXQa/U/UFV7k/wq8GiS5+Y/WVWV5LCXWPcfFDcBnH322YsarKTJGLTHr6q9/f0+4GG66rqvJVkD0N/vO8J77aQjLTNDWmidnOSXDz4Gfh94BthE10gDbKghjcqQQ/3VwMNdg1xWAn9fVY8keRx4IMmNwEvAddMbpqRJOmrw+8YZFxxm/o+BK6YxKEnT5ZV7UoMMvtQgS29p2bFU1vS5x5caZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBg0KfpJTkjyY5LkkO5JcmmRVkkeT7OzvT532YCVNxtA9/u3AI1V1Hl0Zrh3YSUcarSFVdt8H/A5wF0BV/U9VvYmddKTRGrLHXwfsB76a5Kkkd/Zltu2kI43UkOCvBC4CvlxVFwJvc8hhfVUVXZutX5DkpiTbkmzbv3//YscraQKGBH8PsKeqtvbTD9J9ENhJZ8RqwjeNy1GDX1WvAi8nObefdQWwHTvpSKM1tMrunwL3JDkReBH4Y7oPDTvpSCM0KPhV9TSw4TBP2UlHGiGv3JMaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaNKSu/rlJnp53eyvJLXbSGbfUZG8alyHFNp+vqvVVtR74LeC/gIexk440Wgs91L8CeKGqXsJOOtJoLTT41wP39o/tpCON1ODg96W1rwG+duhzdtKRxmUhe/wPAk9W1Wv9tJ10pJFaSPBv4P8P88FOOtJoDQp+3x13I/DQvNmfAzYm2Qlc2U9LGoGhnXTeBk47ZN6PsZOONEpeuSc1yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81aNCVe8tZ94eBWnJuhlFxjy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoOGlt768yTPJnkmyb1JTkqyLsnWJLuS3N9X4ZU0AkNaaJ0J/Bmwoap+E1hBV1//88AXq+r9wBvAjdMcqKTJGXqovxJ4d5KVwHuAV4DLgQf75+2kI43IkN55e4G/Bn5EF/j/AJ4A3qyqA/3L9gBnTmuQkiZryKH+qXR98tYBvwacDFw1dAF20pGWnyGH+lcCP6yq/VX1v3S19S8DTukP/QHWAnsP92Y76UjLz5Dg/wi4JMl7koSulv524DHgI/1r7KQjjciQ7/hb6U7iPQl8v3/PHcCngU8k2UXXbOOuKY5T0gQN7aTzGeAzh8x+Ebh44iOSNHVeuSc1yOBLDTL4UoMMvtSgzLJYZZL9wNvA6zNb6PSdjuuzXB1P6wLD1ufXq+qoF8zMNPgASbZV1YaZLnSKXJ/l63haF5js+nioLzXI4EsNWorg37EEy5wm12f5Op7WBSa4PjP/ji9p6XmoLzVopsFPclWS5/s6fbfOctmLleSsJI8l2d7XH7y5n78qyaNJdvb3py71WBciyYokTyXZ3E+PtpZiklOSPJjkuSQ7klw65u0zzVqXMwt+khXA3wAfBM4Hbkhy/qyWPwEHgE9W1fnAJcDH+/HfCmypqnOALf30mNwM7Jg3PeZaircDj1TVecAFdOs1yu0z9VqXVTWTG3Ap8K1507cBt81q+VNYn28AG4HngTX9vDXA80s9tgWsw1q6MFwObAZCd4HIysNts+V8A94H/JD+vNW8+aPcPnSl7F4GVtH9Fe1m4A8mtX1meah/cEUOGm2dviRzwIXAVmB1Vb3SP/UqsHqJhnUsvgR8CvhZP30a462luA7YD3y1/+pyZ5KTGen2qSnXuvTk3gIleS/wdeCWqnpr/nPVfQyP4meSJB8G9lXVE0s9lglZCVwEfLmqLqS7NPznDutHtn0WVevyaGYZ/L3AWfOmj1inb7lKcgJd6O+pqof62a8lWdM/vwbYt1TjW6DLgGuS7Abuozvcv52BtRSXoT3AnuoqRkFXNeoixrt9FlXr8mhmGfzHgXP6s5In0p2o2DTD5S9KX2/wLmBHVX1h3lOb6GoOwohqD1bVbVW1tqrm6LbFt6vqo4y0lmJVvQq8nOTcftbB2pCj3D5Mu9bljE9YXA38AHgB+MulPoGywLF/gO4w8XvA0/3tarrvxVuAncA/AauWeqzHsG6/C2zuH/8G8F1gF/A14F1LPb4FrMd6YFu/jf4BOHXM2wf4LPAc8Azwd8C7JrV9vHJPapAn96QGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxr0f7d4JK6fu0G/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "full_env = gameEnv(partial=False,size=9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Reward : 3\n"
     ]
    }
   ],
   "source": [
    "frames = []\n",
    "agent_locs = []\n",
    "local_frames = []\n",
    "random.seed(110)\n",
    "np.random.seed(110)\n",
    "\n",
    "for episode in range(0,1):\n",
    "    \n",
    "    prev_state = env.reset()\n",
    "    random.seed(110)\n",
    "    np.random.seed(110)\n",
    "    full_env_prev = full_env.reset()\n",
    "    \n",
    "    frames.append(full_env_prev)\n",
    "    agent_locs.append((full_env.objects[0].x, full_env.objects[0].y))\n",
    "    local_frames.append(prev_state)\n",
    "    processed_prev_state = preprocess_image(prev_state)\n",
    "    total_reward = 0\n",
    "    step_count = 0\n",
    "    \n",
    "    hidden_state, cell_state = main_model.init_hidden_states(bsize=1)\n",
    "    \n",
    "    while step_count < MAX_STEPS:\n",
    "        \n",
    "        step_count +=1\n",
    "        \n",
    "        torch_x = torch.from_numpy(processed_prev_state).float().to(device)\n",
    "        model_out = main_model.forward(torch_x,bsize=1,time_step=1,hidden_state=hidden_state,cell_state=cell_state)\n",
    "        out = model_out[0]\n",
    "        action = int(torch.argmax(out[0]))\n",
    "        hidden_state = model_out[1][0]\n",
    "        cell_state = model_out[1][1]\n",
    "        \n",
    "        next_state, reward, d = env.step(action)\n",
    "        full_env_next,r,g = full_env.step(action)\n",
    "        agent_locs.append((full_env.objects[0].x, full_env.objects[0].y))\n",
    "        #print(full_env.objects[0].x, full_env.objects[0].y)\n",
    "        total_reward += reward\n",
    "        frames.append(full_env_next)\n",
    "        local_frames.append(next_state)\n",
    "        processed_next_state = preprocess_image(next_state)\n",
    "        \n",
    "        \n",
    "        prev_state = next_state\n",
    "        processed_prev_state = processed_next_state\n",
    "\n",
    "print('Total Reward : %d'%(total_reward))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image, ImageDraw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "locs = {}\n",
    "locs[(2,1)] = [(15, 8), (37, 30)]\n",
    "locs[(2,2)] = [(15, 15), (37, 37)]\n",
    "locs[(2,0)] = [(15, 0), (37, 22)]\n",
    "locs[(3,0)] = [(22, 0), (44, 22)]\n",
    "locs[(3,2)] = [(22, 15), (44, 37)]\n",
    "locs[(3,3)] = [(22, 22), (45, 45)]\n",
    "locs[(4,3)] = [(29, 22), (51, 45)]\n",
    "locs[(4,2)] = [(29, 15), (51, 37)]\n",
    "locs[(5,2)] = [(38, 15), (60, 37)]\n",
    "locs[(5,3)] = [(38, 23), (60, 45)]\n",
    "locs[(5,4)] = [(38, 30), (60, 52)]\n",
    "locs[(5,5)] = [(38, 38), (60, 60)]\n",
    "locs[(4,5)] = [(31, 38), (53, 60)]\n",
    "locs[(4,6)] = [(31, 46), (53, 68)]\n",
    "locs[(4,7)] = [(31, 53), (53, 75)]\n",
    "locs[(4,8)] = [(31, 61), (53, 83)]\n",
    "locs[(5,8)] = [(38, 61), (60, 83)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(2, 0),\n",
       " (2, 1),\n",
       " (2, 2),\n",
       " (3, 0),\n",
       " (3, 2),\n",
       " (3, 3),\n",
       " (4, 2),\n",
       " (4, 3),\n",
       " (4, 5),\n",
       " (4, 6),\n",
       " (4, 7),\n",
       " (4, 8),\n",
       " (5, 2),\n",
       " (5, 3),\n",
       " (5, 4),\n",
       " (5, 5),\n",
       " (5, 8)}"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(agent_locs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx, img in enumerate(frames):\n",
    "    image = Image.fromarray(img)\n",
    "    drawer = ImageDraw.Draw(image)\n",
    "    drawer.rectangle(locs[agent_locs[idx]], outline=(255, 255, 0))\n",
    "    frames[idx] = np.array(image)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Frames to GIF </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mayank/miniconda3/envs/rdqn/lib/python3.5/site-packages/ipykernel_launcher.py:7: DeprecationWarning: `imresize` is deprecated!\n",
      "`imresize` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n",
      "Use ``skimage.transform.resize`` instead.\n",
      "  import sys\n",
      "/home/mayank/miniconda3/envs/rdqn/lib/python3.5/site-packages/ipykernel_launcher.py:8: DeprecationWarning: `imresize` is deprecated!\n",
      "`imresize` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n",
      "Use ``skimage.transform.resize`` instead.\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "import imageio\n",
    "from scipy.misc import imresize\n",
    "resized_frames = []\n",
    "resized_local_frames = []\n",
    "\n",
    "for i in range(0,len(frames)):\n",
    "    resized_frames.append(imresize(frames[i],(256,256)))\n",
    "    resized_local_frames.append(imresize(local_frames[i],(256,256)))\n",
    "\n",
    "imageio.mimsave('data/GIFs/LSTM_SIZE_9_frames.gif',resized_frames,fps=3)\n",
    "imageio.mimsave('data/GIFs/LSTM_SIZE_9_local.gif',resized_local_frames,fps=3)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}