编写强化学习环境进阶（Predator Game）-代码.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: opencv-python in /Users/liangsy/opt/anaconda3/lib/python3.8/site-packages (4.5.2.52)\r\n",
      "Requirement already satisfied: numpy>=1.17.3 in /Users/liangsy/opt/anaconda3/lib/python3.8/site-packages (from opencv-python) (1.19.5)\r\n"
     ]
    }
   ],
   "source": [
    "!pip install opencv-python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pillow in /Users/liangsy/opt/anaconda3/lib/python3.8/site-packages (7.2.0)\r\n"
     ]
    }
   ],
   "source": [
    "!pip install pillow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import cv2\n",
    "from PIL import Image\n",
    "import time\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import style\n",
    "style.use('ggplot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPISODES = 30000\n",
    "SHOW_EVERY = 3000\n",
    "\n",
    "epsilon = 0.6\n",
    "EPS_DECAY = 0.9998\n",
    "DISCOUNT = 0.95\n",
    "LEARNING_RATE = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Cube:\n",
    "    def __init__(self,size):\n",
    "        self.size = size\n",
    "        self.x = np.random.randint(0, self.size)\n",
    "        self.y = np.random.randint(0, self.size)\n",
    "        \n",
    "    def __str__(self):\n",
    "        return f'{self.x},{self.y}'\n",
    "    \n",
    "    def __sub__(self,other):\n",
    "        return (self.x-other.x,self.y-other.y)\n",
    "    \n",
    "    def __eq__(self,other):\n",
    "        return self.x == other.x and self.y == other.y\n",
    "    \n",
    "    def action(self,choise):\n",
    "        if choise == 0 :\n",
    "            self.move(x=1, y=1)\n",
    "        elif choise == 1 :\n",
    "            self.move(x=-1, y=1)\n",
    "        elif choise == 2 :\n",
    "            self.move(x=1, y=-1)\n",
    "        elif choise == 3 :\n",
    "            self.move(x=-1, y=-1)\n",
    "        elif choise == 4 :\n",
    "            self.move(x=0, y=1)        \n",
    "        elif choise == 5 :\n",
    "            self.move(x=0, y=-1) \n",
    "        elif choise == 6 :\n",
    "            self.move(x=1, y=0) \n",
    "        elif choise == 7 :\n",
    "            self.move(x=-1, y=0)             \n",
    "        elif choise == 8 :\n",
    "            self.move(x=0, y=0)             \n",
    "            \n",
    "    def move(self,x=False,y=False):\n",
    "        if not x:\n",
    "            self.x += np.random.randint(-1,2)\n",
    "        else:\n",
    "            self.x += x\n",
    "            \n",
    "        if not y:\n",
    "            self.y += np.random.randint(-1,2)\n",
    "        else:\n",
    "            self.y += y   \n",
    "            \n",
    "        if self.x < 0 :\n",
    "            self.x = 0\n",
    "        elif self.x >= self.size :\n",
    "            self.x = self.size -1\n",
    "\n",
    "        if self.y < 0 :\n",
    "            self.y = 0\n",
    "        elif self.y >= self.size :\n",
    "            self.y = self.size -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class envCube:\n",
    "    SIZE = 10\n",
    "    OBSERVATION_SPACE_VALUES = (SIZE,SIZE,3)\n",
    "    ACTION_SPACE_VALUES = 9\n",
    "    RETURN_IMAGE = False\n",
    "    \n",
    "    FOOD_REWARD = 25\n",
    "    ENEMY_PENALITY = -300\n",
    "    MOVE_PENALITY = -1    \n",
    "    \n",
    "    d = {1:(255,0,0), #blue\n",
    "         2:(0,255,0), #green\n",
    "         3:(0,0,255)} #red\n",
    "\n",
    "    PLAYER_N = 1\n",
    "    FOOD_N =2\n",
    "    ENEMY_N =3    \n",
    "    \n",
    "    def reset(self):\n",
    "        self.player = Cube(self.SIZE)\n",
    "        self.food = Cube(self.SIZE)\n",
    "        while self.food == self.player:\n",
    "            self.food = Cube(self.SIZE)\n",
    "        \n",
    "        self.enemy = Cube(self.SIZE)\n",
    "        while self.enemy == self.player or self.enemy == self.food:\n",
    "            self.enemy = Cube(self.SIZE)\n",
    "        \n",
    "        if self.RETURN_IMAGE:\n",
    "            observation = np.array(self.get_image())\n",
    "        else:\n",
    "            observation = (self.player - self.food)+(self.player - self.enemy) \n",
    "        \n",
    "        self.episode_step = 0\n",
    "        \n",
    "        return observation\n",
    "    \n",
    "    def step(self,action):\n",
    "        self.episode_step += 1\n",
    "        self.player.action(action)\n",
    "        self.food.move()\n",
    "        self.enemy.move()\n",
    "\n",
    "        if self.RETURN_IMAGE:\n",
    "            new_observation = np.array(self.get_image())\n",
    "        else:\n",
    "            new_observation = (self.player - self.food)+(self.player - self.enemy)\n",
    "\n",
    "        if self.player == self.food :\n",
    "            reward = self.FOOD_REWARD\n",
    "        elif self.player == self.enemy :\n",
    "            reward = self.ENEMY_PENALITY\n",
    "        else:\n",
    "            reward = self.MOVE_PENALITY\n",
    "\n",
    "        done = False\n",
    "        if self.player == self.food or self.player == self.enemy or self.episode_step>=200:\n",
    "            done = True\n",
    "        \n",
    "        return new_observation,reward,done\n",
    "    \n",
    "    def render(self):\n",
    "        img = self.get_image()       \n",
    "        img = img.resize((800,800))\n",
    "        cv2.imshow('Predator',np.array(img))\n",
    "        cv2.waitKey(1)\n",
    "    \n",
    "    def get_image(self):\n",
    "        env = np.zeros((self.SIZE,self.SIZE,3), dtype=np.uint8)\n",
    "        env[self.food.x][self.food.y] = self.d[self.FOOD_N]\n",
    "        env[self.player.x][self.player.y] = self.d[self.PLAYER_N]\n",
    "        env[self.enemy.x][self.enemy.y] = self.d[self.ENEMY_N]\n",
    "        img = Image.fromarray(env,'RGB')\n",
    "        return img\n",
    "\n",
    "    def get_qtable(self,qtable_name=None):\n",
    "        if qtable_name is None:\n",
    "            q_table = {}\n",
    "            for x1 in range(-self.SIZE+1, self.SIZE):\n",
    "                for y1 in range(-self.SIZE+1, self.SIZE):\n",
    "                    for x2 in range(-self.SIZE+1, self.SIZE):\n",
    "                        for y2 in range(-self.SIZE+1, self.SIZE):\n",
    "                            q_table[(x1,y1,x2,y2)] = [np.random.uniform(-5,0) for i in range(self.ACTION_SPACE_VALUES)]\n",
    "        else:\n",
    "            with open(qtable_name,'rb') as f:\n",
    "                q_table=pickle.load(f)\n",
    "        return q_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode #0, epsilon:0.6\n",
      "mean reward:nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/liangsy/opt/anaconda3/lib/python3.8/site-packages/numpy/core/fromnumeric.py:3372: RuntimeWarning: Mean of empty slice.\n",
      "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
      "/Users/liangsy/opt/anaconda3/lib/python3.8/site-packages/numpy/core/_methods.py:170: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode #3000, epsilon:0.32926722239570905\n",
      "mean reward:-177.114\n",
      "episode #6000, epsilon:0.18069483957364205\n",
      "mean reward:-137.52466666666666\n",
      "episode #9000, epsilon:0.09916147987941909\n",
      "mean reward:-99.12666666666667\n",
      "episode #12000, epsilon:0.054417708414240505\n",
      "mean reward:-76.72733333333333\n",
      "episode #15000, epsilon:0.029863279497827713\n",
      "mean reward:-62.33233333333333\n",
      "episode #18000, epsilon:0.016388331819794114\n",
      "mean reward:-53.786\n",
      "episode #21000, epsilon:0.00899356749667138\n",
      "mean reward:-45.93933333333333\n",
      "episode #24000, epsilon:0.00493547831509555\n",
      "mean reward:-42.602333333333334\n",
      "episode #27000, epsilon:0.0027084853933429503\n",
      "mean reward:-35.649\n",
      "27001\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEJCAYAAAC61nFHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA0oUlEQVR4nO3de1xUZf7A8c85DAqsaAIiGSh5W7U0KC11y2xl92fmZqWtdnGVtMi0MrRs12xrSaWtNBVN85J22VK3LNtcK3LLWiwxcyutNbysF0CFAUEFgTnP7w/WkVkuDsPMnJnh+369enXOc84Zvt/G/HKe55zn0ZRSCiGEEKKRdLMDEEII4Z+kgAghhHCJFBAhhBAukQIihBDCJVJAhBBCuEQKiBBCCJdYzA7A23Jzc126LioqioKCAjdH4zsCPT8I/BwlP//nqzl26NChzna5AxFCCOESKSBCCCFcIgVECCGES6SACCGEcIkUECGEEC6RAiKEEMIlUkCEEEK4RAqIEEJ4ifHJ+9jSHsF2782ovMMAqMpKVHGhw3mq8ET1v202r8fYGM3uRUIhhPAEVVqCFt4adeYUavc3aJddCSEhgIaRckut840nJ9dqO1bPZ+v3Pw4WC1x2JZrFgjIM1Cfvow0ehhYc7M40GkUKiBBCuEgpBSXFGNPH1T7mxp9jLE2v++evW1m9kdgf/d7paMEtqmMqO4MW9jN7jJqmuTGa86SACCGEC9S/v8N4fqbT52sjx6EPHVl9bXkZxoOjAdBffAMsLTCm3G4/T7tyIES1hzOnMNKmgvUC05t88yXGA6MaPEWf/zpaq9ZOx+sMKSBCCNFI6lRJreKhTUhF690XLMFoLVtWn5d/BGPOo2g3/Rb9/249f25IKEHLNzpcH7R8Y+25sFq1JujZVSjDBjYbFByD0DDIPQzdL4fcQxiL0uB/xlDq4u7iAVJAhBDCKep0KcaCp+HAXod2/cU30H4WXuc1WkwsQQvfbPLP1vQg0IPg4rjqhosiq//dsTNBz70CgG1pOnydhf7iXzCm3ll93ZDfoP3qFrTIdk2OoS5+UUA2btzI66+/zooVK2jdurqKbtiwgS1btqDrOsnJySQkJJgbpBAiIBlZW1Cffwg5P9Q6pj/8x3qLh7cF3f/4+e3/ubvxFJ8vIAUFBXz33XdERUXZ244cOUJWVhbz5s2jqKiItLQ0FixYgK7LU8lCCPdQSmHcN6LOY9o9j6APuMHLEfken/8bd82aNdx1110OTxFkZ2czcOBAgoODiY6OJiYmhpycHBOjFEIEClVxFlV4os7iEbR8I0HLN0rx+C+fvgPZsWMHERERxMfHO7RbrVa6detm34+IiMBqtdb5GZmZmWRmZgKQnp7ucCfTGBaLxeVr/UGg5weBn6Pk1zTqbDmFU8di5B91aI9+6x/2QXFP87fv0PQCkpaWRnFxca32MWPGsGHDBp544olax5Ry/gnrpKQkkpKS7PuurvblqyuFuUug5weBn6Pk5zylFFRVQeFxjFmT6j1PT19BYWkplJa65edeiK9+h/WtSGh6AZk1a1ad7YcOHeL48eM8+uijABQWFjJjxgzmzp1LZGQkhYXnH1uzWq1ERER4JV4hhLmUYat+Ue5/Bq9t6Y/Bvh8bvFZ/Yj7GM49c8GfoSzegBQU1Kc7mwPQCUp+OHTuyYsUK+/7kyZOZO3curVu3pm/fvixcuJDhw4dTVFREXl4eXbt2NTFaIYQnqYqzKMPAdu/NtY7pc17GWPXiBYsHUG/x0B96EnolStFoJJ8tIA2Ji4tjwIABpKamous6EyZMkCewhPAzKmcPtGgJUTFoYT9DHc+r/gu+7Azabb9D638DxmPJ9vOP1/M5xh/us29rg2+En4WjPliHnr4S/vMTxkvp6NNnO7z4p93zCNqVA8BWhRbWykMZBj5NNWZAIQDk5ua6dJ2v9k26S6DnB4Gfo6/lp/IOQ8lJjOf/4NbP1SfPxHhlAfysFZzIt7drSTejj55YfzxKQXkZWmiYW+NxJ1/7Ds/x2TEQIURgUUpBwbE6Z5ttirDfjKb817eihYQRtOAv539eeRm0DLnghIGaplVPAyLcRgqIEMItVFUVxqTbah9oF4N23f+h3lkDVI83qB+/g7YR6EkjUOVnoKICbDa0ttVTdKjC49A2Cior7Y/QhkdFcbaO3861kFDPJSUaJAVECOEShze1Q8Pq/O1ef/m983cGN460t2u9+57fDgmDEMdrtcjo6g0vvX8hXCMFRAgBVBcElbUFLeEatJ+dH1hWO7fB5VdivPhHCG+DduVA1IoXHC8uO1P9D6DPXgpo1XceHlqHQvgGKSBCCAD73cSFnqpRO7fVe0wb9yBadN0DriLwSAERoplRZ05hrFkEDRQCZ+jPLEVr36F60Pxf26H7ZfJIbDMjBUSIZsD421qIao9aOa/B87TbxqG2bYHw1rB3d3Vbv+ugXQzq0H70u+5Hi2rveI2mQcI1Hotd+C4pIEIEONufHobDB+o8pj/3CpSXwdlytE7/nc2hxmC3EA2RAiJEADM+3VSreGi3J6P/+tZ6rhDCeVJAhAhQxtoVqMzqlem0wTei31X/rLNCuEImkBIiAKnTp+zFg9hLpXgIj5A7ECH8nFIKjhyEqChsj9wNp0ocjgf9cYE5gYmAJwVECD+k9v0IxVaMpen2tmN1nKcv2+C9oESzIwVECD+hvvsaLktEvb0a9dG7FzxfG5WMpsv6FsJzpIAI4cNUVRXs3omR8Uz9J7VoiZ6xjpD3/0JZUDD6Tb/1XoCiWZMCIoQPUsdzMV5dDP/+ru4T4i4l6EnHsY3wex6uc7ZaITxFCogQPkKVnsRYlIYW3w31jw9qHdeGj66ehyqsFfqjs02IUAhHUkCE8BFG6lgA1IG9Du36krfBsKG1DIERd5kRmhB18vkC8ve//53NmzcTFBTElVdeyd133w3Ahg0b2LJlC7quk5ycTEJCgrmBCtEExofvOOzrM9KhS88a06EHez8oIS7ApwvI999/z44dO3j++ecJDg7m5MmTABw5coSsrCzmzZtHUVERaWlpLFiwAF2X9yKFfzA+fAf119W12rWkm9Ful6enhH/w6QLy0UcfMWLECIKDq3/7atOmDQDZ2dkMHDiQ4OBgoqOjiYmJIScnh+7du5sZrhAXpI4ewnhqSr3H9dETvRiNEE3j0wUkLy+PH3/8kbfeeovg4GDGjh1L165dsVqtdOvWzX5eREQEVqvVxEiFqGZseA1tyG/QWl/k2F5jXqpaIqPRBvwSfcSdng9QCDcyvYCkpaVRXFxcq33MmDEYhsGpU6eYPXs2+/btY/78+WRkZFRP3eCkzMxMMjMzAUhPTycqKsqlOC0Wi8vX+oNAzw88m6Nx+hQn7v41AGrTeiJffI3K/XuxxHfh1JsrsGV/4RhLx85EvPiaW5d8DfTvMNDzA//L0fQCMmvWrHqPffTRR1xzzTVomkbXrl3RdZ3S0lIiIyMpLCy0n2e1WomIiKjzM5KSkkhKSrLvF7j4nHxUVJTL1/qDQM8PPJOjMgyMGROguNChvXDq2DrP14aORB85DgUOf4bdIdC/w0DPD3w3xw4d6l6m2KdHnfv168f3338PQG5uLlVVVYSHh9O3b1+ysrKorKzk+PHj5OXl0bVrV5OjFc2JOn0KtXc3RsotjsUjMrruC664mqDlG9FHjvNKfEJ4g+l3IA355S9/yZIlS5g2bRoWi4XJkyejaRpxcXEMGDCA1NRUdF1nwoQJ8gSW8BilFMZ9I+z7esY6jKmO4xX6tGegay8ICnLoljK++Bj1r+0ETZ7ptXiF8BZNNWZAIQDk5ua6dJ2v3lq6S6DnB67lqM6cxnj4jgbP0ee/jtaqdVNCc4tA/w4DPT/w3Rzr68Ly6TsQIcykTpdiTK3/zW/9pXfQLPK/kGi+pN9HiP9hbPsHtntvdiwe8d0IWr4R7dwYRpceUjxEsyf/BwhRg/HBOtS7rzu0aROnoV9zPQD60JEwdKQZoQnhc+otIOeefrqQyy+/3G3BCOFN6kQ+6m9r0W6+Aw7sxVj251rn6C+/59Z3NYQIJPUWkJdeeslh32q1omka4eHhlJaWopQiMjKSjIwMjwcphLupqiqMP9xXvZ31Sa3jUjiEuLB6C8jixYvt2++88w6nTp1i9OjRtGzZkrNnz7J27VrCw8O9EqQQ7nLuoUNjyZy6T+h+OUGP1nNMCOHAqTGQDz74gGXLlmH576Bhy5YtufPOO0lJSeHWW2/1aIBCuFPN9zkA9NQ0uLQ7WkioSREJ4b+cegorJCSEnJwch7Z9+/bRsmVLjwQlhCcUzXmsVpvW8wopHkK4yKk7kNGjRzNnzhyuuuoq+zxUO3fuZMKECZ6OTwi3UGfPUlFjQkN94VtooWEmRiSE/3OqgFx77bV07tyZL7/8kqKiIi655BJGjhxJbGysp+MToslsL6XDzqzqnSsHEjTpcXMDEiJAXLCAGIbB2LFjWb16NaNGjfJGTEK4jaqsOF88QIqHEG50wTEQXdfp0KEDpaWl3ohHCLdSO7fZt6P/utXESIQIPE53YT377LPceOONREZGOjwfLy8SCl+k9v2IkX5+0Fxf8Be0IJl4QQh3cur/qI8++giA9evXO7RrmiYvEgqfoyorHIoHgBbWyqRohAhcThWQmi8VCuFrbPfeXL3RLgZO5Nc6rt01ycsRCdE8yD298Gu2/05HAtQqHvqMdLSuvbwckRDNh1MF5MyZM6xfv549e/bY58E653/nzBLC01T5GdT61aitm+s9R5+7HC2qvRejEqL5caqArFixAqvVyqhRo1i0aBEPPvggGzdu5JprrvF0fELUYkz7HVRU2Pe1/jeg/WIIhIahdepqYmRCNC9OFZBvv/2W+fPnEx4ejq7r9OvXjy5duvDss88yfPhwT8cohJ0yDIfiQdso9AmPmBeQEM2YUwVEKUVYWPW0DyEhIZw+fZqLLrqI/PzaA5budPDgQZYvX05FRQVBQUFMnDiRrl2rf8PcsGEDW7ZsQdd1kpOTSUhI8GgswlzKZkP9bS3qb2/Z2/RFa2UeKyFM5FQB6dSpE3v27KF379706NGDlStXEhISwsUXX+zR4F5//XVGjRpFYmIiO3fu5PXXX+epp57iyJEjZGVlMW/ePIqKikhLS2PBggXouqzQG4iUUhj3O876rM9eKsVDCJM59TduSkoK7dq1A+Cee+6hRYsWnD59milTpng0OE3TKCsrA6oH8tu2bQtAdnY2AwcOJDg4mOjoaGJiYmrNFiwCgzJstaZg14aPRovuYFJEQohzNFXzkSofc+TIEWbPng1Uz8n1zDPP0K5dO1auXEm3bt0YNGgQUP0kWGJiIv3796/1GZmZmWRmZgKQnp5ORc3+80awWCxUVVW5mInv88X8jNISTvxuqH2/zWNzCBkw2OXP88Uc3Uny83++mmOLFi3qbHeqC+uxxx6jV69e9n9atXLfW71paWkUFxfXah8zZgzfffcd48aNo3///mRlZbF06VJmzZpFY2peUlISSUlJ9v2CggKX4oyKinL5Wn/gi/nZXxAEtKEjOdXtck41IUZfzNGdJD//56s5duhQ9x2/UwVk7Nix/PDDD2zatImFCxcSExNjLyZ1/dbfGLNmzar3WEZGBsnJyQAMGDCAZcuWAdjXJDnHarUSERHRpDiE71BKodatsu9rg4aijxxnYkRCiLo4VUB69+5N7969ASgtLeVvf/sbmzdv5sMPP2Tt2rUeCy4iIoI9e/Zw2WWX8f333xMTEwNA3759WbhwIcOHD6eoqIi8vDz701nCv6iTRRjT/1scWobC2TKH49o9j6APuMGEyIQQF+JUAdm1axd79uxhz549FBYW0q1bN+6880569fLsNBEpKSm88sorGIZBcHAwKSkpAMTFxTFgwABSU1PRdZ0JEybIE1h+yl48oFbxANCuud6L0QghGsOpQfTRo0fTvn17brnlFq6//nqCgoK8EZtH5ObmunSdr/ZNuosZ+TmsFFiHoOUb3frz5Dv0b4GeH/hujk0aA3n66af54Ycf+PLLL1m7di1xcXH06tWLnj170rNnT7cGKpqHmgPk9Eog6JE/2XeVzYbmx7+kCNFcOFVAevToQY8ePbj11ls5efIkmzZt4r333mPt2rUeHQMRgUkVnnDYr1k8ACkeQvgJpwrI9u3b2b17N3v27CEvL4/OnTszdOhQj4+BiMCiqipRO75ArZwPgDbgl+j3TDU3KCGEy5wqIJs2baJXr16MGzeO7t271/tSiRB1UXlHAIXx5GSHdm3cg+YEJIRwC6cKyFNPPeXhMESgMl5bUve6HT/vLV1VQvg5pwpIZWUlf/3rX/nnP/9JaWkpa9as4V//+hd5eXkMHTr0wh8gmh3j/bdQu76CQ/sc2vXJf0BLaNrLp0II3+DUyxOrV6/m8OHDPPTQQ2iaBlS/i/HRRx95NDjhn1RJMWrjXxyKhzbst2j3PSbFQ4gA4tQdSHZ2NgsXLiQkJMReQCIiIrBarR4NTvgfpVT1ioE16KlpaD2vMCkiIYSnOFVALBYLhmE4tJWUlBAeHu6RoIR/Ugd+wpgzzb6vp6+sXmY27GcmRiWE8BSnurD69+9PRkYGx48fB6CoqIiVK1cycOBAjwYn/IdSyrF4zJqPFtlOiocQAcypAnLnnXcSHR3NtGnTOHPmDA899BBt27bl9ttv93R8wl98/U/7pjZ2MlrHLiYGI4Twhgt2YRmGwdtvv81dd93F+PHj7V1X58ZChFBnTmMs+zMAespjaH2vNTkiIYQ3XPAORNd1PvzwQ/sEiq1bt5biIRwYD99h35biIUTz4VQX1vXXX8/HH3/s6ViEH1LW8zOH6nNeNjESIYS3OfUUVk5ODps3b2bjxo1ERkY63IE8/fTTHgtO+D5jcfWa9drNd6K1izE5GiGENzlVQIYMGcKQIUM8HYvwM0op+8uC2vDRJkcjhPA2pwrI4MGDPRyG8Dfq7FnUP893a8q4mBDNj1MFRIialM2GMeX8I9z6rBfNC0YIYRpZSFw0Ws3iAaB17GxSJEIIM5l+B7Jt2zbWr1/P0aNHmTNnDl26nH8BbcOGDWzZsgVd10lOTiYhIQGA/fv3s3jxYioqKkhMTCQ5OVm6ULypqqr6370S0UdPMDcWIYRpTL8DiYuLY/r06bXWVj9y5AhZWVnMmzePmTNnsnLlSvt8XMuXLyclJYWFCxeSn5/Prl27TIi8+VF5R86vZR4URNAjT6N16GhuUEII01zwDuTIkSNs3bqVI0eOUFZWRmhoKLGxsQwaNIjY2NgmB1DfZ2RnZzNw4ECCg4OJjo4mJiaGnJwc2rVrR1lZGd27dwdg0KBBZGdnk5iY2ORYRP2MLX9DvXn+PQ99znIToxFC+IIGC8gXX3zBihUr6Nu3Lz179iQsLIyysjIOHjzIrFmzuPfeez02oaLVaqVbt272/XPTxwcFBREZGWlvj4yMbHBa+czMTDIzMwFIT08nKirKpXgsFovL1/qDhvJTNhvHaxSPi/74Ii279/BWaG7TnL/DQBDo+YH/5dhgAXnzzTd5/PHH6dGj9l8WP/74I4sWLXKqgKSlpVFcXFyrfcyYMfTr16/Oa5RSjWqvT1JSEklJSfb9goKCBs6uX1RUlMvX+oOG8rPVWMtcS36Y0tjOlPrhf4vm/B0GgkDPD3w3xw4dOtTZ3mABKSkpoXPnup+wufTSSykpKXHqh8+aNcup82qKjIyksLDQvm+1WomIiKjVXlhYSERERKM/XzhHnT0LeYcB0P/wPNql3U2OSAjhKxocRO/Tpw9LliwhPz/foT0/P59ly5bRp08fjwXWt29fsrKyqKys5Pjx4+Tl5dG1a1fatm1LaGgoe/fuRSnF1q1b6du3r8fiaO7UpvXVGy1aSPEQQjho8A5k0qRJrFixgtTUVIKCgggLC+PMmTMYhsHVV1/NpEmTmhzA9u3bWbVqFSUlJaSnpxMfH8/MmTOJi4tjwIABpKamous6EyZMQNer693EiRNZsmQJFRUVJCQkyAC6h6izZ1Gb1gGgL3jT5GiEEL5GU04MKpw9e5a8vDzKy8sJCQnh4osvpmXLlt6Iz+1yc3Ndus5X+ybd5Vx+qrIC9flHaINvxEi51X48aPlG84Jzk+byHQaqQM8PfDdHl8ZAzmnZsiVRUVH2AuKvxUM0TOUdxvjvgLnDI7sPP2VSREIIX9ZgAamqqmLdunV8+umnnDx50t5+0UUXMXjwYG6//XYsFtNfZhduYtR42uoc/Y8L0WLjvR+MEMLnNfi3/4oVKzh27BgPPfQQnTp1cngP5J133mHFihXcf//93opVeFDVf/bZt7VRyZB3GO2OFDS52xRC1KPBAvLVV1+xePFiwsLC7G2tWrXi8ssvJz4+nilTpkgBCRCFU8cCoD+5AC3uUpOjEUL4gwYf4w0ODqaoqKjOY8XFxQQHB3skKOFdNZ+jkOIhhHBWg3cgN998M08//TS//OUva3Vh/eMf/+CWW27xUpjCk9Tba8wOQQjhhxosIMOHDyc2NpatW7fy9ddf25/CiouLY9KkSfbp1YX/UoYN9eE7AOjTnjE5GiGEP7ngI1QJCQlSKAKYkTHbvq318NzMAkKIwHPBAmKz2fj+++85fPiw/Q6kY8eOXHbZZQQFBXkjRuFJ3+0AoNXd91NmcihCCP/SYAE5ePAgzz33HEopOnbsaB8D+fvf/w7AY489RqdOnbwSqHA/9d8FugB+NvJ3lPngG7BCCN/VYAFZtmwZw4cP58Ybb6x1bPPmzbz00kukp6d7LDjhWeqDdWaHIITwYw0+xnvkyBF+9atf1XksKSmJo0ePeiQo4SXnpmmXiRKFEC5osIBccsklfPTRR3Ue+/jjj7nkkks8EpTwPFVZgcr+HC5LRAv7mdnhCCH8UINdWPfffz/PPfcc77//vsMYyH/+8x90XefRRx/1VpzCzYwHRgGgdep2gTOFEKJuDRaQ+Ph4FixYwO7duzly5Ij9Kaxhw4bRq1cvmUjRT9nuvdm+rd0wzMRIhBD+7IIVwGKxcMUVV3DFFVd4Ix7hTe1i0C6S5YCFEK65YAHZvn07+fn5XHvttYSFhbFu3TqOHTtG7969GTp0qDdiFG5km5lSvdGtF0GPyRN0QgjXNVhANmzYwBdffIGmaXz88ccMGjSI1q1b06pVKzZs2EB5ebnMh+UnVGWFfdwDQOt/g4nRCCECQYMFJDMzk2eeeQalFJMmTaJfv37Ex8cD0KdPHzIyMqSA+Am1eqHDvj7o/0yKRAgRKBosIKdOnaJt27ZA9bK254oHQNeuXeud6r0xtm3bxvr16zl69Chz5syhS5cuAHz77be88cYbVFVVYbFYGDt2LJdffjkA+/fvZ/HixVRUVJCYmEhycjKapjU5lkCmtm8FQJ/0ONqVA02ORggRCBp8DyQsLIyKigoAbrvtNodjp0+fdstTWHFxcUyfPp2ePXs6tIeHhzNjxgxeeOEFJk+ezKJFi+zHli9fTkpKCgsXLiQ/P59du3Y1OY5Apior7dtSPIQQ7tJgAfnFL35BYWEhQK2uqm3bttnvFpoiNjaWDh061Gq/9NJLiYiofkIoLi6OyspKKisrKSoqoqysjO7du6NpGoMGDSI7O7vJcQQy44GRAGi/SDI5EiFEIGnwFuLuu++u99iQIUMYMmSI2wOqy1dffcWll15KcHAwVquVyMhI+7HIyEisVmu912ZmZpKZmQlAeno6UVFRLsVgsVhcvtZMyjA4/t/t8KsGEFpPDv6aX2MEeo6Sn//ztxxd7oNqzJhDWloaxcXFtdrHjBlDv379Grz28OHDvPHGG8ycORNwXH7VGUlJSSQlnf/Nu8DFGWejoqJcvtZMtunj7dune/fjdD05+Gt+jRHoOUp+/s9Xc6yrlwiaUEAaY9asWS5dV1hYyPPPP8/kyZOJiYkBqu84znWrnTvnXFeXcKQMG5ysvjvTkh82ORohRKBpcAzETKdPnyY9PZ077riDHj162Nvbtm1LaGgoe/fuRSnF1q1b6du3r4mR+i4j5Vb7tj7QO92NQojmw/TJrLZv386qVasoKSkhPT2d+Ph4Zs6cyebNm8nPz+ftt9/m7bffBuCJJ56gTZs2TJw4kSVLllBRUUFCQgKJiYkmZ+F7as53pT+VYV4gQoiApSknBxXOnDlDbm4u5eXlDu3n3s3wF7m5uS5d56t9k3UxvvwUtXJe9U5oGEEL37rgNf6Un6sCPUfJz//5ao5NGgP59NNPWblyJSEhIbRo0cLermkaGRny262vsRePHn0ImvaMucEIIQKWUwXkzTffJDU1VbqK/IA6XWrfluIhhPAkpwbRDcOQ6dz9gLKewJh6l9lhCCGaCacKyIgRI3j77bcxDMPT8YgmMGZMsG/rM2SqdiGEZznVhfXBBx9QXFzMxo0badWqlcOxl156ySOBicYxPn7Pvq0vexdN99kntIUQAcKpAvLggw96Og7RRGrdSgC08Q9J8RBCeIVTBaRXr16ejkM0gfHPT+zbukyYKITwEqdfJDx48CA//PADpaWlDvNRjR492iOBCeep1QuqN2IvNTcQIUSz4lQByczMZM2aNfTp04ddu3aRkJDAt99+K1OI+ICab5wH/XGBiZEIIZobpzrL33vvPf7whz/w6KOP0qJFCx599FFSU1MJCgrydHzCSdpvJ1z4JCGEcCOnCkhJSYl9xUBN0zAMg8TERL7++muPBicapmo8Vq3/aoSJkQghmiOnurAiIiI4fvw40dHRXHzxxezYsYPw8HC3LGkrXGNseA21ab3ZYQghmjGnKsCIESM4evQo0dHRjBo1innz5lFVVUVycrKn4xN1MFa8gPrqM/u+/sR8E6MRQjRXThWQwYMH27cTExN55ZVXqKqqIiQkxFNxiQbULB4AWqemr00vhBCN5XQfVGlpKd988w1FRUWMGDGCkpISTp8+7bA+ufA8dfqUfVvPWA81ZkcWQghvcmoQfc+ePUydOpXPP//cvrhTfn4+y5cv92hwojZj6p0AaDcMQ2vZslFr0wshhDs5VUBWr17N1KlTmTlzpv3R3a5du7Jv3z6PBicc2R65276t3TbOxEiEEMLJAnLixAl69+7t0GaxWLDZbB4JStSmDBucKgFAG/sAWkioyREJIZo7pwpIbGwsu3btcmj77rvv6NixY5MD2LZtG6mpqYwePbrOO5qCggLGjh3Lxo0b7W379+9n2rRpPPjgg6xatQonV+X1a2rTX+3b+qChJkYihBDVnCogY8eOZdGiRWRkZFBRUcHLL7/MkiVLuPvuuy988QXExcUxffp0+4uK/2v16tW1VkJcvnw5KSkpLFy4kPz8/FrFLRCp994AQH/8zyZHIoQQ1Zx6Cqt79+4899xzfP7554SEhBAVFcWcOXPc8gRWbGxsvce2b99O+/btadmypb2tqKiIsrIyunfvDsCgQYPIzs4O6OV2Vf4R+7bWpYeJkQghxHlOP8YbERHBiBHemy6jvLyc9957j1mzZjl0X1mtVofCFRkZidVqrfdzMjMzyczMBCA9PZ2oqCiX4rFYLC5f21THakyY6KkYzMzPWwI9R8nP//lbjk4VkDNnzrBp0yYOHjxIeXm5w7EnnnjigtenpaVRXFxcq33MmDH069evzmvWrVvHTTfdVOtlxcaOdyQlJZGUdH6NjIKCgkZdf05UVJTL1zaFsXalfVtf9q7HYjArP28K9BwlP//nqzl26NChznanCsi8efMwDIOrr76aFi68uDZr1qxGX5OTk8NXX33FG2+8wenTp9E0jRYtWnDNNddQWFhoP6+wsJCIiIhGf76/UJnnl6qVlQaFEL7EqQLy008/sXLlSq9OnvinP/3Jvr1u3TpCQkIYOrT66aPQ0FD27t1Lt27d2Lp1q709kKiSIoxp59/1CFq+sYGzhRDC+5yqCD169ODo0aN06tTJ7QFs376dVatWUVJSQnp6OvHx8cycObPBayZOnMiSJUuoqKggISEh4AbQVVWVQ/HQZKp2IYQP0pQTgwonT55k7ty5dO3alYsuusjh2KhRozwVm0fk5ua6dJ03+ybV7m8wXvyjfd8bdx++2vfqToGeo+Tn/3w1xyaNgbz55psUFhbSrl07ysrK7O0yD5P7qTOn7MVDf/gptMuvNDkiIYSom1MFJCsriwULFtC2bVtPx9PsqXdetW9L8RBC+DKnHutp3769rH/uJeqzzYC8cS6E8H1O3YFcd911/PnPf2bo0KG1xkAuv/xyT8TVLKmzZ+3b8sa5EMLXOVVAPvzwQ6B6LKQmTdPIyMhwf1TNkKqsxJhye/WOvO8hhPADThWQxYsXezqOZs94YKR9W1/ytomRCCGEc+RXXZMZWzdjqzHXlb5oLZqMNwkh/IAUEBMZWVtQry2x72v3PCILRQkh/IYUEBOpV160b2vDbkcfcIN5wQghRCN5b3IrYadKT6I+/8i+L/NcCSH8kRQQExipY80OQQghmky6sLxMFTsufiV3H0IIfyUFxMuMR8cDoA35DfrL7zV8shBC+DApIF6kjp2fCVj77QSZjFII4dekgHiR8cT9AOiTZ8rqgkIIvyd/i3mJOnPq/M4VV5sXiBBCuIkUEG/54dvqf18cJ11XQoiAIAXES4yl6QDoU54wORIhhHAPKSBeoM6cPr/TLsa8QIQQwo1Mf5Fw27ZtrF+/nqNHjzJnzhy6dOliP/af//yHl19+mbKyMjRNY+7cubRo0YL9+/ezePFiKioqSExMJDk52ae7hYxV8wHQho706TiFEKIxTC8gcXFxTJ8+nZdfftmh3WazsWjRIqZMmUJ8fDylpaVYLNXhLl++nJSUFLp168bcuXPZtWsXiYmJZoR/QaqyEv61HQBtxJ0mRyOEEO5jehdWbGwsHTp0qNX+r3/9i44dOxIfHw9AeHg4uq5TVFREWVkZ3bt3R9M0Bg0aRHZ2tpejdl7NdT40S7CJkQghhHuZfgdSn7y8PDRNY/bs2ZSUlDBw4EBGjBiB1WolMjLSfl5kZCRWq7Xez8nMzCQzMxOA9PR0oqKiXIrHYrE0+lrj9ClO/He7zeNzCXHxZ3uDK/n5m0DPUfLzf/6Wo1cKSFpaGsXFxbXax4wZQ79+/eq8xmaz8eOPPzJ37lxatmzJn/70Jzp37kxoaOPWy0hKSiIpKcm+X1BQ0Kjrz4mKimrUtaqkGGPa7wDQbvsdp7pcxikXf7Y3NDY/fxToOUp+/s9Xc6yrlwi8VEBmzZrV6GsiIyPp1asXrVu3BiAxMZEDBw5w3XXXUVhYaD+vsLCQiIgIt8XqLueKB4D2qxEmRiKEEJ5h+hhIfa644goOHTrE2bNnsdls/PDDD8TGxtK2bVtCQ0PZu3cvSim2bt1K3759zQ7XgfGx4ySJMvYhhAhEpo+BbN++nVWrVlFSUkJ6ejrx8fHMnDmTVq1acdNNN/H73/8eTdNITEzkyiuvBGDixIksWbKEiooKEhISfOYJLGPTetSG1843dOmBPvUp0+IRQghP0pRSyuwgvCk3N/fCJ9XhQn2TqqoSY9JIhzZ/WuvDV/te3SnQc5T8/J+v5ljfGIjPdmH5G/XuGw77+ot/MSkSIYTwDtO7sAKF+vAdALTkh9EHDjE5GiGE8Dy5A3EzKR5CiOZCCogb2DKeMTsEIYTwOikgTaSqqs7PdTVqvLnBCCGEF0kBaSL1ygL7tv5/t5kYiRBCeJcUkCZQBcdQ2z8DQJ/32gXOFkKIwCIFpAmM399r39bC25gYiRBCeJ88xusC2703O+xroyeYFIkQQphH7kAaSVVVOexro5LRk2SyRCFE8yMFpJHU5rcd9rWkm+s5UwghApt0YTWSeq96yhI9fQVaZLTJ0QghhHnkDqQRlM1m35biIYRo7qSANMa/vzM7AiGE8BlSQBrBmP8kAPpTi8wNRAghfIAUECcdG3nt+Z2YWPMCEUIIHyEFxAnGFx+DYQCgP/IntKAgkyMSQgjzSQFxglpzvstK65VgXiBCCOFDpIA4IzQMAP3hp8yNQwghfIjp74Fs27aN9evXc/ToUebMmUOXLl0AqKqqYunSpRw4cADDMBg0aBC33norAPv372fx4sVUVFSQmJhIcnIymqZ5LMaghW/57FrFQghhFtPvQOLi4pg+fTo9e/Z0aP/yyy+pqqrihRdeID09nczMTI4fPw7A8uXLSUlJYeHCheTn57Nr1y4TIhdCiObN9AISGxtLhw4d6jxWXl6OzWajoqICi8VCWFgYRUVFlJWV0b17dzRNY9CgQWRnZ3s5aiGEEKZ3YdWnf//+7Nixg/vuu4+KigrGjRtHq1at2LdvH5GRkfbzIiMjsVqt9X5OZmYmmZmZAKSnpxMVFeVSPBaLxeVr/UGg5weBn6Pk5//8LUevFJC0tDSKi4trtY8ZM4Z+/frVeU1OTg66rrNs2TJOnz7Nk08+Se/evVFKNepnJyUlkZSUZN93dRwj0MdAAj0/CPwcJT//56s51tdL5JUCMmvWrEZf88UXX5CQkIDFYqFNmzb8/Oc/Z9++ffTs2ZPCwkL7eYWFhURERLgzXCGEEE4wfQykPlFRUXz//fcopSgvL+enn37ikksuoW3btoSGhrJ3716UUmzdupW+ffuaHa4QQjQ7po+BbN++nVWrVlFSUkJ6ejrx8fHMnDmToUOHsmTJEqZNm4ZSihtuuIFOnToBMHHiRJYsWUJFRQUJCQkkJiaanIUQQjQ/mmrsoIKfy83Ndek6X+2bdJdAzw8CP0fJz//5ao71jYE0uwIihBDCPXx2DMTXPP7442aH4FGBnh8Efo6Sn//ztxylgAghhHCJFBAhhBAukQLipJovIwaiQM8PAj9Hyc//+VuOMoguhBDCJXIHIoQQwiVSQIQQQrjE9DfRfd2uXbt45ZVXMAyDIUOGcMstt5gdktMmT55MSEgIuq4TFBREeno6p06dYv78+Zw4cYJ27drxyCOP0KpVKwA2bNjAli1b0HWd5ORkEhISAO8v4NWQJUuWsHPnTtq0acMLL7wA4NacKisrycjIYP/+/YSHhzN16lSio6NNzW/dunV88skntG7dGoA77riDK6+80i/zKygoYPHixRQXF6NpGklJSQwbNiygvsP6cgyk79FOiXrZbDY1ZcoUlZ+fryorK9X06dPV4cOHzQ7LaQ888IA6efKkQ9trr72mNmzYoJRSasOGDeq1115TSil1+PBhNX36dFVRUaGOHTumpkyZomw2m1JKqccff1z9+9//VoZhqNmzZ6udO3d6NY+adu/erfbt26dSU1Ptbe7MafPmzWrZsmVKKaW++OILNW/ePC9mV3d+a9euVe+9916tc/0xP6vVqvbt26eUUurMmTPqoYceUocPHw6o77C+HAPpezxHurAakJOTQ0xMDO3bt8disTBw4EC/X7wqOzub66+/HoDrr7/enk92djYDBw4kODiY6OhoYmJiyMnJ8bkFvHr16mX/zfQcd+a0Y8cOBg8eDFSvSXNuQk8z86uPP+bXtm1bOnfuDEBoaCiXXHIJVqs1oL7D+nKsjz/meI4UkAZYrdZGLV7li2bPns2MGTPsi2qdPHmStm3bAtV/0EtKSoDauUZERGC1Wv3iv4E7c6p5LCgoiLCwMEpLS72VSr0+/PBDpk+fzpIlSzh16hTg//kdP36cAwcO0LVr14D9DmvmCIH3PcoYSAPqquhm9f27Ii0tjYiICE6ePMkzzzxT74RoUHeuDbX7A1dy8sXv/Ne//jWjRo0CYO3atbz66qs88MADfp1feXk5L7zwAuPHjycsLKze8wIpx0D8HuUOpAGRkZG1Fq8691uSPzi30FabNm3o168fOTk5tGnThqKiIgCKiorsA3r/m6vVaiUiIqLO/wa+toCXO3Oqecxms3HmzBmnu5Q85aKLLkLXdXRdZ8iQIezbtw/w3/yqqqp44YUXuO6667jmmmuAwPsO68ox0L5HkALSoC5dupCXl8fx48epqqoiKyvLbxavKi8vp6yszL797bff0rFjR/r27ctnn30GwGeffWZfUrhv375kZWVRWVnJ8ePHycvLo2vXrn6xgJc7c7rqqqv49NNPAfjyyy+57LLLTL8DOfcXK1SvnxMXFwf4Z35KKZYuXcoll1zC8OHD7e2B9B3Wl2MgfY/nyJvoF7Bz507WrFmDYRjccMMN3HbbbWaH5JRjx47x/PPPA9W/oVx77bXcdtttlJaWMn/+fAoKCoiKiiI1NdX+m8s777zDP/7xD3RdZ/z48faFuvbt2+ewgNc999xj2l+qL774Inv27KG0tJQ2bdrw29/+ln79+rktp4qKCjIyMjhw4ACtWrVi6tSptG/f3tT8du/ezcGDB9E0jXbt2nHffffZ74T9Lb8ff/yRJ598ko4dO9r/DN1xxx1069YtYL7D+nL85z//GTDf4zlSQIQQQrhEurCEEEK4RAqIEEIIl0gBEUII4RIpIEIIIVwiBUQIIYRLpIAI4Uapqans3r3brZ+5ePFi3nrrLbd+phDuIAVECDeaN28el112mdlh1CklJYWKigq+//57+ztCQjSFFBAhmoGCggLCw8Np0aIF+/fv59JLLzU7JBEAZDJFIf6H1Wpl1apV/PDDD4SEhHDTTTcxbNgwoHpxp8OHD6PrOt988w0XX3wxkyZNIj4+HqhexCslJYU+ffqQk5PDihUryMvLo0WLFlx77bWMGzcOqJ6O+y9/+QtWq5X4+HgmTpxIbGwsAAcOHGDp0qXk5eWRmJhY663/r7/+mrfeeosTJ04QGxvLvffeS6dOnRrMaf/+/fYpxvft22efOl2IppACIkQNhmHw7LPP0q9fP6ZOnUphYSFpaWl06NDBvkrcjh07ePjhh3nwwQfZtGkTzz33HAsWLMBicfzf6ZVXXmHYsGEMGjSI8vJyDh06BEBubi4LFizg0UcfpVevXnzwwQc8++yzzJ8/H4DnnnuOYcOGMXToUHbs2MGCBQsYMWIEUF0IXnrpJWbMmEGXLl3YunUrf/7zn3nxxRcJDg6ulc/69ev54IMPqKysRNM0tm/fTllZGbt27ULTNFatWoWuS0eEcI38yRGihn379lFSUsKoUaOwWCy0b9+eIUOGkJWVZT+nc+fO9O/fH4vFwvDhw6msrOSnn36q9VkWi4X8/HxKSkoICQmhe/fuAGRlZZGYmEifPn2wWCz85je/oaKign//+9/s3bsXm83GTTfdhMVioX///nTp0sX+mZ988glJSUl069YNXdcZPHgwFoulzp8PcPvtt7Ny5Uqio6PJyMjg97//PQkJCaxZs4bVq1dL8RBNIncgQtRw4sQJioqKGD9+vL3NMAx69uxp36+5yI+u60RGRjrMtHrO/fffz9q1a3nkkUeIjo5m1KhRXHXVVRQVFdGuXTuHz4iKisJqtaLrOhEREQ7dVlFRUfbtgoICPvvsMzZv3mxvq6qqqnORr4MHD/LUU09hGAaVlZVMnTqViooKgoKCGD9+PJMmTbJPNS6EK6SACFFDVFQU0dHRLFy4sN5zaq7RYBhGvevEXHzxxUydOhXDMNi+fTvz5s1j5cqVtG3b1t6dBdXTfxcUFNgLh9VqRSllLyKFhYXExMQA1cXrtttuc2pW6Pj4eFavXs27776LUopbb72Vxx57jNTUVPvnCdEUcv8qRA1du3YlNDSUd999l4qKCgzD4NChQ+Tk5NjP2b9/P1999RU2m41NmzYRHBxMt27dan3W1q1bKSkpQdd1+6p7uq4zcOBAvvnmG7777juqqqp4//33CQ4O5uc//zndu3dH13X+/ve/Y7PZ+Oqrrxx+9pAhQ/j444/56aefUEpRXl7Ozp077Wu/1OXcU1fn7lTMmPZbBCa5AxGiBl3XmTFjBq+++iqTJ0+mqqqKDh06MHr0aPs55xYAWrx4MTExMUybNq3WADrArl27ePXVVzl79izt2rXj4YcfpkWLFnTo0IEHH3yQVatW2Z/CmjFjhv0zpk+fzrJly3jrrbdITEzk6quvtn9mly5dSElJYdWqVfanu3r06OHQxfa/9u/fz8SJEzl06JDDGhVCNJWsByJEI6xbt478/Hweeughs0MRwnTShSWEEMIlUkCEEEK4RLqwhBBCuETuQIQQQrhECogQQgiXSAERQgjhEikgQgghXCIFRAghhEv+H0ZNNWAXi8biAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "env = envCube()\n",
    "q_table = env.get_qtable()\n",
    "\n",
    "episode_rewards = []\n",
    "for episode in range(EPISODES):\n",
    "    obs = env.reset()\n",
    "    done = False\n",
    "    \n",
    "    if episode%SHOW_EVERY == 0 :\n",
    "        print(f'episode #{episode}, epsilon:{epsilon}')\n",
    "        print(f'mean reward:{np.mean(episode_rewards[-SHOW_EVERY:])}')\n",
    "        show = True\n",
    "    else:\n",
    "        show = False\n",
    "    \n",
    "    episode_reward = 0\n",
    "    while not done:\n",
    "        \n",
    "        if np.random.random() > epsilon :\n",
    "            action = np.argmax(q_table[obs])\n",
    "        else:\n",
    "            action = np.random.randint(0,env.ACTION_SPACE_VALUES)\n",
    "        \n",
    "        new_obs,reward,done =  env.step(action)\n",
    "        \n",
    "        #Update the Q_table\n",
    "        current_q = q_table[obs][action]\n",
    "        max_future_q = np.max(q_table[new_obs])                \n",
    "        if reward == env.FOOD_REWARD:\n",
    "            new_q = env.FOOD_REWARD\n",
    "        else:\n",
    "            new_q = (1 - LEARNING_RATE)*current_q + LEARNING_RATE*(reward+DISCOUNT*max_future_q)        \n",
    "        q_table[obs][action] = new_q\n",
    "        obs = new_obs\n",
    "        \n",
    "        if show:\n",
    "            env.render()\n",
    "                   \n",
    "        episode_reward += reward\n",
    "\n",
    "    \n",
    "    episode_rewards.append(episode_reward)\n",
    "    epsilon *= EPS_DECAY\n",
    "\n",
    "moving_avg = np.convolve(episode_rewards,np.ones((SHOW_EVERY,))/SHOW_EVERY,mode='valid')\n",
    "print(len(moving_avg))\n",
    "plt.plot([i for i in range(len(moving_avg))],moving_avg)\n",
    "plt.xlabel('episode #')\n",
    "plt.ylabel(f'mean {SHOW_EVERY} reward')\n",
    "plt.show()\n",
    "\n",
    "with open(f'qtable_{int(time.time())}.pickle','wb') as f:\n",
    "    pickle.dump(q_table,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(-2, -8, -8, -5)\n",
      "(-1, -9, -7, -5)\n",
      "-1\n",
      "False\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[-2.1932051595425923,\n",
       " -4.688459827321189,\n",
       " -4.667169373304091,\n",
       " -2.6067662017395,\n",
       " -4.14886543538474,\n",
       " -3.0245484307537507,\n",
       " -3.0331143214077096,\n",
       " -3.5303108268635826,\n",
       " -1.1933610124636345]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = envCube()\n",
    "print(env.reset())\n",
    "new_observation,reward,done = env.step(3)\n",
    "print(new_observation)\n",
    "print(reward)\n",
    "print(done)\n",
    "env.render()\n",
    "q_table1 = env.get_qtable()\n",
    "q_table1[new_observation]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f'qtable_{int(time.time())}.pickle','wb') as f:\n",
    "    pickle.dump(q_table,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27001\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEKCAYAAAA8QgPpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAzmklEQVR4nO3de1wWZf7/8dcMNwdNVA4iEnjikJoabGrqlrUru+uam0Wa1uYvTYvMNENNy9ytSMVMDY951sottSTdb2ZGVm5hSblWHsrQzBNocIOggBxmfn+wjt5x8Ba57+G++Twfj+/jO3PNzM3n8m55M6frUnRd1xFCCCGukmp2AUIIIVyTBIgQQohakQARQghRKxIgQgghakUCRAghRK1IgAghhKgVi9kF1OSNN97gm2++wWKx0LJlSx5//HGuu+46AFJSUtixYweqqjJixAiio6PNLVYIIRoYpT6/B/Ltt9/SuXNnPDw8ePPNNwF48MEHOXHiBMnJycyYMYPc3FwSExNJTk5GVa98QnXq1Kla1RIYGEh2dnatjnUF7t4/cP8+Sv9cX33tY0hISJXt9foS1k033YSHhwcAUVFRWK1WANLT0+nduzeenp4EBQURHBxMRkaGmaUKIUSDU68D5HI7duwwLlNZrVYCAgKMbf7+/ka4CCGEcA7T74EkJiaSl5dXqX3o0KF0794dgE2bNuHh4cFtt90GwNVcdUtNTSU1NRWApKQkAgMDa1WnxWKp9bGuwN37B+7fR+mf63O1PpoeINOmTatx+6effso333zDP/7xDxRFASAgIICcnBxjH6vVir+/f5XHx8bGEhsba6zX9vpifb02WVfcvX/g/n2U/rm++tpHl7wHsnfvXjZv3szkyZPx9vY22rt160ZaWhqlpaWcOXOGzMxMIiIiTKxUCCEaHtPPQGqycuVKysrKSExMBCAyMpJHH32UsLAwevXqRUJCAqqqMnLkSLuewBJCCFF36nWALFiwoNptcXFxxMXFObEaIYQQl5M/24UQwk3ouk75Y/egfboV/XwB2ppk9LJSh/28en0GIoQQomb6hQuQnwuqB9qUkRVt615DX/daxfIXHwPgsXxLnf9sCRAhhDCRXlqK4ulp25afC42boFg80b5IRV8z32a7umwz+vsb0Devs//nlFxA8fK+8o5XQQJECCFMoOs62vQJ8MulUTRO23ms9ujAarep419AuTHm0s85/AP60QxwwINGEiBCCOFk2qa16B+8a/8BFk/Uf7yKtjoZfj5ks0mdsxalqV+VZzIASngHlPAO11py1WU55FOFEEIAoBeeh6LzUFoCnl7om15H373T2K5Ofw2OH0XbngJHfrzUvmQT7N8DN3RB8WkEgMezr1R8Zn4enPwFpeNNxv5VhYejSYAIIUQd0vNzAQV+OYw2/4Vq91PuegD1b0MrVoJC8Li5d+U30W/qUfWxTZtD0+Z1VnNtSYAIIUQt6IXn4af9aAtfutTo3QguFF3xWGXUBNRbbndgdc4hASKEEHbSS0vRP/8ICs+hv/dm5R1+Gx4BQahT56L4NnVOgU4mASKEEHbQiwrRxg2tcps6cTraK1Mrlp96EaVTtBMrM48EiBCiQdOLCqHwHBQXg5cX+p409HfWoE5fihLUCm3rRvSUNyodpwwajvqXS8MpOeJFvfpOAkQI0eDoBWfR31uHvnNbtftoU+OrbFeXvIticf4TT/WRBIgQwq3pFy6gPTH42j4ktB34BaCOnWbMSyQkQIQQbkq/UIz2xH1X3E99ZjZK+xvQiwsBBby8UFQP9H3fgE8jCG1nvIchbEmACCHcjq7rlcJDfXUdHDsCAS2gIB/yclBu/r2xXfFpbLO/0vlmp9TqyiRAhBBuRdfK0eLvMdbV2atRmgdUrFx8czuo6ilaxdWRABFCuDS95AL6N2koN/VAe/J+m23qS69dCg9R5yRAhBAuo/yRuwBQJ81Am/2szTb9N/uqT72A0lLONBxJAkQIUa/opSVQcBZt5TzIOIDyp7tR4oZx+p7exj6/DY/LqROnQ1RneVrKCSRAhBCm04uL4NhhtO3vwbe7bbd9uAn9w001Hq8ufgfF08uBFYqqSIAIIUylXyhGGzvErn3V2WugabOKm+Q398bjsSmOLU7UyCUCZMuWLbz55pusWLGCpk0rBiVLSUlhx44dqKrKiBEjiI6ONrdIIcRV0zauRt+eYtOm9OmHMvSRSvNbXD7UeUMcNqQ+qvcBkp2dzffff09gYKDRduLECdLS0pg7dy65ubkkJiaSnJyM6oApG4UQdUfXdSgvQxt9b6VtEgqup97/xl27di1///vfbW6Ipaen07t3bzw9PQkKCiI4OJiMjIwaPkUIYSb9pwOUP3IX2qMDK4WHMvIpCQ8XVa/PQL7++mv8/f1p27atTbvVaiUyMtJY9/f3x2q1Ork6IcRv6QVnwacx+ppkm2lbq6Iu2YRiqde/gsQVmP7tJSYmkpeXV6l96NChpKSk8Nxzz1Xapuu/feK7eqmpqaSmpgKQlJRkcynsalgsllof6wrcvX/g/n00o396WRn6hSKKP/uQguVz7T7uuvsepklw8FX9LHf//sD1+qjoV/Pb2ImOHTvGiy++iLe3NwA5OTn4+fkxc+ZMPvnkEwDuuadiuILp06czePBgoqKirvi5p06dqlU9leYqdjPu3j9w/z46u3+6pqHF313zTr7NoOAsysNPofb6wzX9PHf//qD+9jEkpOoXMk0/A6lO69atWbFihbE+ZswYZs6cSdOmTenWrRvz589nwIAB5ObmkpmZSUREhInVCtGw6Id/QEt6ulK7unAjeHigv7MaZdBwmTfDzdXbAKlJWFgYvXr1IiEhAVVVGTlypDyBJYQTaP/Zjv76Qps29cl/QtPmKK3DjTZl6CPOLk2YwGUCZNGiRTbrcXFxxMXFVbO3EKKu6QX5lcNDZudr0FwmQIQQznWlmfzkKSoh374QwoZuzUab/HC129Wl76HIJWOBBIgQAtCLCtHGDa164+96Q8kF1IEPQJsIGeVWGCRAhGjgdE2rMjzUybNQIjqaUJFwFRIgQjQAelEhaBo0agyKAkWFaFNGQlGh7Y7tolAiO6EOrv4SlhAXSYAI4aa01C1g8YSsE+gf//uK+6uvrkO5ztcJlQl3IQEihAvT0v+DckNnlKZ+FU9NrZmPcud9aM8+av+HhLRGfX6B3NsQV00CRIh6SC8uRPFpjK7raI8OrHnf//3/MxfXv0itvFO7KNRHJqItfwU14UUoLUXxbVanNYuGRwJECCfTSy5Azhn07e+hf/6Rw36O8uDjqLf3s2nzePaVigUfh/1Y0YBIgAjhRNWNIXUlysAHoEkzOHYYPT/v0rzhHhYoL4ObetAi4XlysrLQP3gH5a4HUHyb1mntQvyWBIgQTqJr5ZXDI7wDHP4BAPW5eWjTE1DjJ6OfPome8kZFew0v7um6DlknUFqFoTZtjlJShvL3xxzaDyEukgARwsH0slK0lxLg5C9GW3Uz8Hks2wyAAtC/+mFELlIUBVqF1UWZQlw1CRAhHOy3U7iqr64zqRIh6pYEiBAOov1racWLe5dR570p71oIt1FtgOzbt8+uD+jcuXOdFSOEq9Hzc+HQfujSHX78Dm7oguLtQ/kjd9nsp/z5bnm7W7idagNkyZIlNutWqxVFUfD19aWgoABd1wkICGDhwoXVfIIQ7k374mP0Ncl27asMGuHgaoRwvmoD5PIJnDZt2sS5c+cYMmQI3t7eXLhwgfXr1+PrK6fiouHRNQ1++O6K4aH+MxkltJ2TqhLC+ey6B/L++++zdOlSLP+bPMbb25sHHniA+Ph47rnnHocWKITZ9OIi+Gk/2vwXq9yuvpYCgOLhgbZ2AfrnH6FOTpLwEG7PrgDx8fEhIyODDh06GG2HDx/G29vbYYUJYQb9x31orzxr386Nr8Mj+S2bJvWhsfDQWAdUJkT9Y1eADBkyhBkzZnDzzTcTEBBATk4Oe/bsYeTIkY6uTwin0S9cuGJ4KHc/CGetqA/Iy3pC2BUgt956K+3bt+fLL78kNzeX66+/nnvvvZfQ0FBH1yeE09Q4//drKSgeHk6sRoj674oBomkaw4YNY82aNQwaNMgZNdn44IMP2LZtGx4eHvzud7/jwQcfBCAlJYUdO3agqiojRowgOjra6bUJ96Dn56FN+H/GurrgbRSfxjUcIYQAOwJEVVVCQkIoKCjA39/fGTUZ9u3bx9dff80rr7yCp6cnZ8+eBeDEiROkpaUxd+5ccnNzSUxMJDk5GbWa8YKEqIquaWjxd9u0qaOnSHgIYSe7L2HNmjWLv/71rwQEBNhMPOPIFwm3b9/OwIED8fT0BKBZs4r5C9LT0+nduzeenp4EBQURHBxMRkYGUVFRDqtFuBZdK0f/8jOUm3+P4u2NXnxp6lb9QjHoGtpY23nA1Skvo4R3+O1HCSGqYVeAbN++HYCNGzfatCuK4tAXCTMzM/nhhx94++238fT0ZNiwYURERGC1WomMjDT28/f3x2q1VvkZqamppKZWTLCTlJREYGBgrWqxWCy1PtYVuEv/yrNPc3beC5Qe2AuAvvpVY9tpANUDtHKbY4Le+hjFp5HTanQUd/kOq+Pu/QPX66NdAXL5S4V1LTExkby8vErtQ4cORdM0zp07x/Tp0zl8+DDz5s1j4cKFFUNY2yk2NpbY2FhjPTs7u1Z1BgYG1vpYV+AO/dOLCyudVVTym/BQl2wi59x5OHfegZU5hzt8hzVx9/5B/e1jSEhIle2mD6Y4bdq0ardt376dW265BUVRiIiIQFVVCgoKjEeJL7JarU6/PyPqDz0/F23CQ5Xa1Sf/iX7wW/Tt70G7KPj5UEX7xOkoN3RxcpVCuB+7AqSwsJCNGzdy4MABYxysi347ZlZd6t69O/v27ePGG2/k1KlTlJWV4evrS7du3Zg/fz4DBgwgNzeXzMxMIiIiHFaHqN9+Gx7qUy+gdIoBQOl8M/xvEMP6+tedEK7KrgBZsWIFVquVQYMGsWDBAsaOHcuWLVu45ZZbHFrcH//4RxYvXsyECROwWCyMGTMGRVEICwujV69eJCQkoKoqI0eOlCewGijtndXGsjx+K4Rz2RUg3333HfPmzcPX1xdVVenevTvh4eHMmjWLAQMGOK44i4Vx48ZVuS0uLo64uDiH/WxR/+n5eegfVoxDRfQtEh5COJldf7bruk7jxhX/4/Tx8eH8+fM0b96crKwshxYnRHX08wU2L/95jJlqYjVCNEx2nYG0adOGAwcO0KVLFzp06MDKlSvx8fGhVatWjq5PiEr0nDNoU0YZ6+r/5hEXQjiXXQESHx9v3Dh/+OGH+de//sX58+d54oknHFqcEADlS2bCnl1VblMTEm1ebBVCOI9dAdKyZUtjuWnTpjz2mIxEKhxP18rR4qufb0Z9bi5KG3n6Tgiz2BUgTz/9NJ06dTL+r0mTJo6uSzRwv51T3ODljTr/bSgtcYu3x4VwZXYFyLBhwzh48CBbt25l/vz5BAcHG2HSs2dPR9coGhj94Le2DZGd8Hg6ybbNQ8JDCLPZFSBdunShS5eKN3cLCgr4v//7P7Zt28aHH37I+vXrHVqgaHi0uZdGJ1D+0F8mbxKinrIrQPbu3cuBAwc4cOAAOTk5REZG8sADD9CpUydH1ycaEL3wHPq6pca6x/ItJlYjhLgSuwJk5syZtGzZkrvvvpvbb78dD5mZTTiA9uQDxrI65WUTKxFC2MOuAHnhhRc4ePAgX375JevXrycsLIxOnTrRsWNHOnbs6OgaRQOgnz93aaWJr8zLIYQLsCtAOnToQIcOHbjnnns4e/YsW7duZfPmzaxfv17ugYhrdvkTV0qvP6I+PN68YoQQdrMrQHbv3s3+/fs5cOAAmZmZtG/fnn79+sk9EFEruqahr1sCIa2hqZ/NNgkPIVyHXQGydetWOnXqxEMPPURUVBReXl6Orku4Me35sZB53KZNuftB1DvvM6kiIURt2BUgzz//vIPLEA2FXnC2UngAKP0Hm1CNEOJa2BUgpaWlvPPOO3zxxRcUFBSwdu1avv32WzIzM+nXr5+jaxRuonzWZMg4aKzLY7pCuDa7hnNfs2YNx48fZ9y4ccbAdWFhYWzfvt2hxQn3oX31mU14qAvk4QshXJ1dZyDp6enMnz8fHx8fI0D8/f2xWq0OLU64B5txrZo2x2PO6+YVI4SoM3adgVgsFjRNs2nLz8/H19fXIUUJ96DreqVBESU8hHAfdgVIz549WbhwIWfOnAEgNzeXlStX0rt3b4cWJ1yXrutojw401pV7hsk9DyHcjF0B8sADDxAUFMSECRMoLCxk3Lhx+Pn5MXiwPDkjqmYTHj1uR5WnrIRwO1e8B6JpGu+++y5///vfGT58uHHpyhmzwB09epTly5dTUlKCh4cHo0aNIiKiYgKhlJQUduzYgaqqjBgxgujoaIfXI+xzcfZKAGXEeNTefzSxGiGEo1zxDERVVT788ENjAMWmTZs6bQrRN998k0GDBjF79mzuu+8+3nzzTQBOnDhBWloac+fOZerUqaxcubLSPRphHuPso12UhIcQbsyuS1i33347H330kaNrqURRFIqKigAoLCzEz69i2Iv09HR69+6Np6cnQUFBBAcHk5GR4fT6hC3tg3dtbpqrE6abWI0QwtHseow3IyODbdu2sWXLFgICAmzOQF544QWHFffQQw8xffp03njjDTRN46WXXgLAarUSGRlp7CePFJtH1zQ4tA9tznM27eroKSje3iZVJYRwBrsCpG/fvvTt29chBSQmJpKXl1epfejQoXz//fc89NBD9OzZk7S0NF577TWmTZtmc439SlJTU0lNTQUgKSmJwMDAWtVpsVhqfawrqG3/zgz7C/q5Apu2pk89T6M+f66r0uqMfIeuzd37B67XR7sC5I477nBYAdOmTat228KFCxkxYgQAvXr1YunSitnqAgICyMnJMfazWq34+/tX+RmxsbHExsYa69nZ2bWqMzAwsNbHuoKr7d9v3++4SF24kfPe3pyvh/9W8h26NnfvH9TfPoaEhFTZbtc9ELP4+/tz4MABAPbt20dwcDAA3bp1Iy0tjdLSUs6cOUNmZqbxdJZwLP1cfpXhoS5Yj8fyLXLZSogGxK4zELPEx8ezevVqNE3D09OT+Ph4oGIcrl69epGQkICqqowcORJVrddZ6BbKZ0yEnw8Z68pf4tBPn0J9/BmnPZknhKg/6nWAdOjQgVmzZlW5LS4ujri4OCdX1HDpOWdswkOdOAPlhs4mViSEMJv82S6uSC8tQZsyylhX+t0r4SGEuPIZyIkTJ9i5cycnTpygqKiIRo0aERoaSp8+fQgNDXVGjcJk2uODjGV12Wa5XCWEAK5wBvL555/z3HPPYbVa6dixI7feeiudOnXCarUybdo00tLSnFWnMIl+2Rv+6uw1Eh5CCEONZyBvvfUWU6ZMoUOHDpW2/fDDDyxYsEBG5HVz2nOPVSwEh6I0r/pRaSFEw1TjGUh+fj7t27evclu7du3Iz893SFGiHvk1CwB12OMmFyKEqG9qDJCuXbuyePFisrKybNqzsrJYunQpXbt2dWhxwjz6kR8vve/RqDFKlNw0F0LYqvES1ujRo1mxYgUJCQl4eHjQuHFjCgsL0TSNHj16MHr0aGfVKZxILy5CmznJWFefX2heMUKIeqvGAGnSpAnjx4/nwoULZGZmUlxcjI+PD61atcJb3jh2W9rYIcay0ucvKP6uMzaPEMJ57HqR0Nvbm8DAQCNAJDwaBnlkVwhRkxoDpKysjA0bNvDpp59y9uxZo7158+bccccdDB48GIulXr/MLmqgH/0JWrdHUT3QL1xAP5MJuZcGcpPwEELUpMbf/itWrOD06dOMGzeONm3a0LhxY4qKijh69CibNm1ixYoVPPbYY86qVdQh7e3l6B//21g/85vtyj3DnFuQEMLl1PgU1ldffcWkSZPo3Lkzvr6+eHh40KRJEzp37kxCQgJffvmls+oUdezy8KiK8hcZZ0wIUbMaz0A8PT3Jzc2lcePGlbbl5eXh6enpsMKEE1g8oazUWFXuGwnlZSg39UDx8DCxMCGEK6gxQO666y5eeOEF/vjHP1a6hPXJJ59w9913O6lMUZe09zdULPg2w+PlVejfpcPrCyH2LrnvIYSwW40BMmDAAEJDQ9m5cyfffPON8RRWWFgYo0ePJjo62kllirqkv/cmAOpjkwFQunYncM379XImNCFE/XXFR6iio6MlKNyInnXCWFba32BiJUIIV3fFACkvL2ffvn0cP37cOANp3bo1N954Ix5yndxl6LqO9uhAY1352/0mViOEcAc1BsjRo0eZPXs2uq7TunVr4x7IBx98AMDTTz9NmzZtnFKouDaXhweA0vdvJlUihHAXNQbI0qVLGTBgAH/9618rbdu2bRtLliwhKSnJYcWJumEMingZ5bomJlQihHAnNQbIiRMn+NOf/lTlttjYWNatW+eQokTd0IsK0be9a6wrdz+Ieud9JlYkhHAnNb5IeP3117N9+/Yqt3300Udcf/31DilKXDtt1ydo44aib91otEl4CCHqUo1nII899hizZ8/m3//+t809kF9++QVVVZk0aVJNh9tl165dbNy4kZMnTzJjxgzCw8ONbSkpKezYsQNVVRkxYoTxNNiRI0dYtGgRJSUlxMTEMGLECHl/4TLaqlfRd+2waVOXbTapGiGEu6oxQNq2bUtycjL79+/nxIkTxlNY/fv3p1OnTnUykGJYWBgTJ05k2bJlNu0nTpwgLS2NuXPnkpubS2JiIsnJyaiqyvLly4mPjycyMpKZM2eyd+9eYmJirrkWV6dfuID2xGCbNo/lW0yqRgjh7q6YABaLhZtuuombbrrJIQWEhoZW2Z6enk7v3r3x9PQkKCiI4OBgMjIyaNGiBUVFRURFRQHQp08f0tPTG3yAaJteR//gHZs2del75hQjhGgQrhggu3fvJisri1tvvZXGjRuzYcMGTp8+TZcuXejXr5/DCrNarURGRhrr/v7+WK1WPDw8CAgIMNoDAgKwWq0Oq8MV6NZfbcJDGfh31AFDajhCCCGuXY0BkpKSwueff46iKHz00Uf06dOHpk2b0qRJE1JSUiguLrZrPKzExETy8vIqtQ8dOpTu3btXeYyu61fVXp3U1FRSU1MBSEpKIjCwdrPrWSyWWh/rSOXWbLInjzTWmz+fjPdNVf+b1qS+9q8uuXsfpX+uz9X6WGOApKam8tJLL6HrOqNHj6Z79+60bdsWgK5du7Jw4UK7AmTatGlXXVhAQAA5OTnGutVqxd/fv1J7Tk4O/v7+1X5ObGwssbGxxnptx3sKDAysl2NFXf6Ohzr3DQp8m1FQizrra//qkrv3Ufrn+uprH0NCQqpsr/Ex3nPnzuHn54e/vz/e3t5GeABERESQm5tbp0Verlu3bqSlpVFaWsqZM2fIzMwkIiICPz8/GjVqxKFDh9B1nZ07d9KtWzeH1VGf6dmnL610ikbxbWZeMUKIBqfGM5DGjRtTUlKCl5cXcXG2EwydP3++Tp7C2r17N6tWrSI/P5+kpCTatm3L1KlTCQsLo1evXiQkJKCqKiNHjkRVK/Ju1KhRLF68mJKSEqKjoxvsDXTtmUcAUAYNR5UJoIQQTlZjAvz+978nJyeHVq1aVbpUtWvXLpt3NmqrR48e9OjRo8ptcXFxlYILIDw8nDlz5lzzz3YXyp/vMbsEIUQDVGOAPPjgg9Vu69u3L3379q3zgsSVaV99hr7ifwHq3UheohRCmKLW16Dkl5Z5jPAA1EcnmliJEKIhu/abGMJp9LJStNH3GuvqvDdRmjQ1sSIhRENW41NYon6xCY9X1kp4CCFMJQHiImze90hcjNLMz8RqhBDiKi5hFRYWcurUKYqLi23aO3fuXOdFCVv6/v/arCvBVY8fJoQQzmRXgHz66aesXLkSHx8fvLy8jHZFUVi4cKHDihMVk0Jpr/7TWFdnLjexGiGEuMSuAHnrrbdISEhosC/smUmbGm8sq8s2y9NvQoh6w657IJqmOWw4d1E9/cIFKDgLgDpnrYSHEKJesStABg4cyLvvvoumaY6uR/yPXlZ6aXIobx+UpnLTXAhRv9h1Cev9998nLy+PLVu20KRJE5ttS5YscUhhDZ22aLqxrL68ysRKhBCianYFyNixYx1dh7iMruuwbw8AysgElMZNrnCEEEI4n10B0qlTJ0fXIS6jvTbLWFZ73mFeIUIIUQO73wM5evQoBw8epKCgwGZWwCFDZOrUOrcnDah4YVAIIeoruwIkNTWVtWvX0rVrV/bu3Ut0dDTfffddg53IyZH0vEuzLcoLg0KI+syup7A2b97Ms88+y6RJk/Dy8mLSpEkkJCTg4eHh6PoaDP1CMfqeXWiTRgCgPPi4yRUJIUTN7DoDyc/Pp2PHjkDF2+eaphETE8P8+fMdWlxDoj1xn826ctufTKpECCHsY1eA+Pv7c+bMGYKCgmjVqhVff/01vr6+dTKlrQD9u3SbdfUfySiqnN0JIeo3uxJg4MCBnDx5kqCgIAYNGsTcuXMpKytjxIgRjq6vQdAWJAKgDH8SpecdKHJpUAjhAuwKkDvuuMNYjomJYfXq1ZSVleHj4+OouhqMy88+1N/LFMFCCNdh93wgBQUF7Ny5k82bN2OxWCgsLCQnJ+fKB4oaGWcfPW43uRIhhLg6dp2BHDhwgDlz5tC+fXt+/PFHBg4cSFZWFlu2bGHKlCnXVMCuXbvYuHEjJ0+eZMaMGYSHhwPw3XffsW7dOsrKyrBYLAwbNsyYe+TIkSMsWrSIkpISYmJiGDFihEsONKifOWUsK/9vjImVCCHE1bPrDGTNmjWMHz+eqVOnGo/uRkREcPjw4WsuICwsjIkTJxpPeV3k6+vL5MmTmTNnDmPGjGHBggXGtuXLlxMfH8/8+fPJyspi796911yHGbTZzxrLirdcDhRCuBa7zkB+/fVXunTpYnugxUJ5efk1FxAaWvXLcu3atTOWw8LCKC0tpbS0lHPnzlFUVERUVBQAffr0IT093TXnKsmzAqBOX2pyIUIIcfXsCpDQ0FDjDfSLvv/+e1q3bu2oumx89dVXtGvXDk9PT6xWKwEBAca2gIAArFZrtcempqaSmpoKQFJSEoGBgbWqwWKx1PrYqmgF+fz6v+UWnbrUuK8z1HX/6iN376P0z/W5Wh/tCpBhw4Yxa9YsYmJiKCkpYdmyZXzzzTdMmjTJrh+SmJhIXl5epfahQ4fSvXv3Go89fvw469atY+rUqQA243DZIzY2ltjYWGM9Ozv7qo6/KDAwsNbHVkVL/xwApU+/Ov3c2qrr/tVH7t5H6Z/rq699DAkJqbLdrgCJiopi9uzZ/Oc//8HHx4fAwEBmzJhhcyZQk2nTptlf6WVycnJ45ZVXGDNmDMHBwUDFGcflT3/l5OTg7+9fq883k77sZQCU+x8xuRIhhKgdu18l9/f3Z+DAgY6sxcb58+dJSkri/vvvp0OHDka7n58fjRo14tChQ0RGRrJz50769evntLrqmmLxNLsEIYSoFbsCpLCwkK1bt3L06FGKi4tttj333HPXVMDu3btZtWoV+fn5JCUl0bZtW6ZOncq2bdvIysri3Xff5d133zV+VrNmzRg1ahSLFy+mpKSE6Ohol7uBrr2zumKh8XXmFiKEENfArgCZO3cumqbRo0cPvLy86rSAHj160KNHj0rt9957L/fee2+Vx4SHhzNnzpw6rcOZ9A9TAFBu+7PJlQghRO3ZFSA//fQTK1eulMET64CeccBYVgfJWGJCCNdlVyJ06NCBkydP0qZNG0fX49bKZ0yEnw9VrPi7zqN6QghRFbsC5PHHH2fmzJlERETQvHlzm22DBg1yRF1uR1s2+1J4AOrMFSZWI4QQ186uAHnrrbfIycmhRYsWFBUVGe2uOP6UWfT0/xjL6uw1KKrd41gKIUS9ZFeApKWlkZycjJ+fn6PrcUv64R+MZY/lW0ysRAgh6o5dfwa3bNlS5j+/Bhcf21XuesDkSoQQou7YdQZy22238fLLL9OvX79K90AuDrEuqqYf/QkyDgKgDBhicjVCCFF37AqQDz/8EKi4F3I5RVFYuHBh3VflRrTpE4xluWckhHAndgXIokWLHF2H25N7H0IIdyOPAjmQbv3fqJo3utZQK0IIYQ8JEAfSJj8MyBvnQgj3JAHiINr/rb+0cr28wS+EcD8SIA6if7sbAOWv98rNcyGEW5IAcZSjPwGgxj1kciFCCOEYEiAOoJ/8xewShBDC4SRAHEB7fiwAygOPmVyJEEI4jgRIHdN13VhW/9DfxEqEEMKxJEDqmL5rh9klCCGEU0iA1DF9dTIA6rOvmFyJEEI4lgRIHdILzhrLSrsoEysRQgjHMz1Adu3aRUJCAkOGDOHw4cOVtmdnZzNs2DC2bLk0ltSRI0eYMGECY8eOZdWqVTb3Hcyk//B9xYJvM3MLEUIIJzA9QMLCwpg4cSIdO3ascvuaNWuIibEdS2r58uXEx8czf/58srKy2Lt3rxMqvTJ92csAqEkyXa0Qwv2ZHiChoaGEhIRUuW337t20bNmS0NBQoy03N5eioiKioqJQFIU+ffqQnp7urHKrpX36gbGseHmbWIkQQjiH6QFSneLiYjZv3szgwYNt2q1WKwEBAcZ6QEAAVqvV2eVVov9nOwDKqAlX2FMIIdyDXfOBXKvExETy8vIqtQ8dOpTu3btXecyGDRu488478fHxsWm/2vsdqamppKamApCUlERgYOBVHX+RxWKp9lgtP49fj1Xcvwm6895afb7Zauqfu3D3Pkr/XJ+r9dEpATJt2rSrPiYjI4OvvvqKdevWcf78eRRFwcvLi1tuuYWcnBxjv5ycHPz9/av9nNjYWGJjY4317Ozsq64FIDAwsNpjtY//75o/32w19c9duHsfpX+ur772sbrbDE4JkNp48cUXjeUNGzbg4+NDv379AGjUqBGHDh0iMjKSnTt3Gu1m0d9eBoC65F1T6xBCCGcyPUB2797NqlWryM/PJykpibZt2zJ16tQajxk1ahSLFy+mpKSE6OjoSk9pOZNeWmIsKxZP0+oQQghnMz1AevToQY8ePWrc57777rNZDw8PZ86cOY4sy35HfgRAfeI5kwsRQgjnqrdPYbkCXdfRXvnf2VJkJ3OLEUIIJ5MAuQb6ZdPWKo2bmFiJEEI4nwTINdB3f1axENrO3EKEEMIEEiDXIuskAB7/TDa5ECGEcD4JkFrS83PNLkEIIUwlAVILuqahTXgIAGXgAyZXI4QQ5pAAqQUt/m5jWel/X/U7CiGEG5MAuUqXj8WlDH0URZV/QiFEwyS//a6SnvZxxUKbCNS+A8wtRgghTCQBcrVOHQNAffwZkwsRQghzSYBcJX37ewAo/i3MLUQIIUwmAXIV6svc60IIUR9IgFyNXzMBUPoPvsKOQgjh/iRAroK+9ysAlNbhJlcihBDmkwC5CvrG1RULnX9nbiFCCFEPSIDYqSzzhLGsePvUsKcQQjQMEiB2Kjv+MwDKnwaaXIkQQtQPEiB2OjtzMgDKnUNMrkQIIeoHCRA76GWlxrJynUwcJYQQIAFiF230vRULv+ttbiFCCFGPSIBcBfWhJ8wuQQgh6g2L2QXs2rWLjRs3cvLkSWbMmEF4+KV3LH755ReWLVtGUVERiqIwc+ZMvLy8OHLkCIsWLaKkpISYmBhGjBiBoigOq1GJHch1oa0pknnPhRDCYHqAhIWFMXHiRJYtW2bTXl5ezoIFC3jiiSdo27YtBQUFWCwV5S5fvpz4+HgiIyOZOXMme/fuJSYmxmE1qkNGcl1gIEXZ2Q77GUII4WpMv4QVGhpKSEhIpfZvv/2W1q1b07ZtWwB8fX1RVZXc3FyKioqIiopCURT69OlDenq6k6sWQghh+hlIdTIzM1EUhenTp5Ofn0/v3r0ZOHAgVquVgIAAY7+AgACsVquJlQohRMPklABJTEwkLy+vUvvQoUPp3r17lceUl5fzww8/MHPmTLy9vXnxxRdp3749jRo1uqqfnZqaSmpqKgBJSUkEBgZedf0AFoul1se6AnfvH7h/H6V/rs/V+uiUAJk2bdpVHxMQEECnTp1o2rQpADExMfz888/cdttt5OTkGPvl5OTg7+9f7efExsYSGxtrrGfX8j5GYGBgrY91Be7eP3D/Pkr/XF997WNVtxmgHtwDqc5NN93EsWPHuHDhAuXl5Rw8eJDQ0FD8/Pxo1KgRhw4dQtd1du7cSbdu3cwuVwghGhzT74Hs3r2bVatWkZ+fT1JSEm3btmXq1Kk0adKEO++8k2eeeQZFUYiJieF3v6sYBXfUqFEsXryYkpISoqOjHfoElhBCiKopegObZu/UqVO1Oq6+nlrWFXfvH7h/H6V/rq++9tHlLmEJIYSo3xrcGYgQQoi6IWcgdpoyZYrZJTiUu/cP3L+P0j/X52p9lAARQghRKxIgQgghakUCxE6Xv4zojty9f+D+fZT+uT5X66PcRBdCCFErcgYihBCiVkx/E72+27t3L6tXr0bTNPr27cvdd99tdkl2GzNmDD4+PqiqioeHB0lJSZw7d4558+bx66+/0qJFC5566imaNKmYKCslJYUdO3agqiojRowgOjoawOkTeNVk8eLF7Nmzh2bNmjFnzhyAOu1TaWkpCxcu5MiRI/j6+jJ+/HiCgoJM7d+GDRv4+OOPjXHh7r//fmNUBlfrX3Z2NosWLSIvLw9FUYiNjaV///5u9R1W10d3+h4NuqhWeXm5/sQTT+hZWVl6aWmpPnHiRP348eNml2W3xx9/XD979qxN2xtvvKGnpKTouq7rKSkp+htvvKHruq4fP35cnzhxol5SUqKfPn1af+KJJ/Ty8nJd13V9ypQp+o8//qhrmqZPnz5d37Nnj1P7cbn9+/frhw8f1hMSEoy2uuzTtm3b9KVLl+q6ruuff/65PnfuXCf2rur+rV+/Xt+8eXOlfV2xf1arVT98+LCu67peWFiojxs3Tj9+/LhbfYfV9dGdvseL5BJWDTIyMggODqZly5ZYLBZ69+7t8pNXpaenc/vttwNw++23G/1JT0+nd+/eeHp6EhQURHBwMBkZGfVuAq9OnToZf5leVJd9+vrrr7njjjsA6NmzJ/v27UN34m3CqvpXHVfsn5+fH+3btwegUaNGXH/99VitVrf6DqvrY3VcsY8XSYDUwB0mr5o+fTqTJ0825kQ5e/Ysfn5+QMV/6Pn5+UDlvvr7+2O1Wl3i36Au+3T5Ng8PDxo3bkxBQYGzulKtDz/8kIkTJ7J48WLOnTsHuH7/zpw5w88//0xERITbfoeX9xHc73uUeyA1qCrRzbr2XxuJiYn4+/tz9uxZXnrppWoHRIOq+1pTuyuoTZ/q43f+5z//mUGDBgGwfv16Xn/9dR5//HGX7l9xcTFz5sxh+PDhNG7cuNr93KmP7vg9yhlIDQICAipNXnXxryRXcHGirWbNmtG9e3cyMjJo1qwZubm5AOTm5ho39H7bV6vVir+/f5X/BjVN4GWGuuzT5dvKy8spLCy0+5KSozRv3hxVVVFVlb59+3L48GHAdftXVlbGnDlzuO2227jlllsA9/sOq+qju32PIAFSo/DwcDIzMzlz5gxlZWWkpaW5zORVxcXFFBUVGcvfffcdrVu3plu3bnz22WcAfPbZZ8aUwt26dSMtLY3S0lLOnDlDZmYmERERLjGBV1326eabb+bTTz8F4Msvv+TGG280/Qzk4i9WqJg/JywsDHDN/um6zmuvvcb111/PgAEDjHZ3+g6r66M7fY8XyYuEV7Bnzx7Wrl2Lpmn84Q9/IC4uzuyS7HL69GleeeUVoOIvlFtvvZW4uDgKCgqYN28e2dnZBAYGkpCQYPzlsmnTJj755BNUVWX48OHGRF2HDx+2mcDr4YcfNu2X6quvvsqBAwcoKCigWbNm3HfffXTv3r3O+lRSUsLChQv5+eefadKkCePHj6dly5am9m///v0cPXoURVFo0aIFjz76qHEm7Gr9++GHH/jHP/5B69atjf+G7r//fiIjI93mO6yuj1988YXbfI8XSYAIIYSoFbmEJYQQolYkQIQQQtSKBIgQQohakQARQghRKxIgQgghakUCRIg6lJCQwP79++v0MxctWsTbb79dp58pRF2QABGiDs2dO5cbb7zR7DKqFB8fT0lJCfv27TPeERLiWkiACNEAZGdn4+vri5eXF0eOHKFdu3ZmlyTcgAymKMRvWK1WVq1axcGDB/Hx8eHOO++kf//+QMXkTsePH0dVVf773//SqlUrRo8eTdu2bYGKSbzi4+Pp2rUrGRkZrFixgszMTLy8vLj11lt56KGHgIrhuP/1r39htVpp27Yto0aNIjQ0FICff/6Z1157jczMTGJiYiq99f/NN9/w9ttv8+uvvxIaGsojjzxCmzZtauzTkSNHjCHGDx8+bAydLsS1kAAR4jKapjFr1iy6d+/O+PHjycnJITExkZCQEGOWuK+//ponn3ySsWPHsnXrVmbPnk1ycjIWi+3/nFavXk3//v3p06cPxcXFHDt2DIBTp06RnJzMpEmT6NSpE++//z6zZs1i3rx5AMyePZv+/fvTr18/vv76a5KTkxk4cCBQEQRLlixh8uTJhIeHs3PnTl5++WVeffVVPD09K/Vn48aNvP/++5SWlqIoCrt376aoqIi9e/eiKAqrVq1CVeVChKgd+S9HiMscPnyY/Px8Bg0ahMVioWXLlvTt25e0tDRjn/bt29OzZ08sFgsDBgygtLSUn376qdJnWSwWsrKyyM/Px8fHh6ioKADS0tKIiYmha9euWCwW/va3v1FSUsKPP/7IoUOHKC8v584778RisdCzZ0/Cw8ONz/z444+JjY0lMjISVVW54447sFgsVf58gMGDB7Ny5UqCgoJYuHAhzzzzDNHR0axdu5Y1a9ZIeIhrImcgQlzm119/JTc3l+HDhxttmqbRsWNHY/3ySX5UVSUgIMBmpNWLHnvsMdavX89TTz1FUFAQgwYN4uabbyY3N5cWLVrYfEZgYCBWqxVVVfH397e5bBUYGGgsZ2dn89lnn7Ft2zajraysrMpJvo4ePcrzzz+PpmmUlpYyfvx4SkpK8PDwYPjw4YwePdoYalyI2pAAEeIygYGBBAUFMX/+/Gr3uXyOBk3Tqp0nplWrVowfPx5N09i9ezdz585l5cqV+Pn5GZezoGL47+zsbCM4rFYruq4bIZKTk0NwcDBQEV5xcXF2jQrdtm1b1qxZw3vvvYeu69xzzz08/fTTJCQkGJ8nxLWQ81chLhMREUGjRo147733KCkpQdM0jh07RkZGhrHPkSNH+OqrrygvL2fr1q14enoSGRlZ6bN27txJfn4+qqoas+6pqkrv3r3573//y/fff09ZWRn//ve/8fT05IYbbiAqKgpVVfnggw8oLy/nq6++svnZffv25aOPPuKnn35C13WKi4vZs2ePMfdLVS4+dXXxTMWMYb+Fe5IzECEuo6oqkydP5vXXX2fMmDGUlZUREhLCkCFDjH0uTgC0aNEigoODmTBhQqUb6AB79+7l9ddf58KFC7Ro0YInn3wSLy8vQkJCGDt2LKtWrTKewpo8ebLxGRMnTmTp0qW8/fbbxMTE0KNHD+Mzw8PDiY+PZ9WqVcbTXR06dLC5xPZbR44cYdSoURw7dsxmjgohrpXMByLEVdiwYQNZWVmMGzfO7FKEMJ1cwhJCCFErEiBCCCFqRS5hCSGEqBU5AxFCCFErEiBCCCFqRQJECCFErUiACCGEqBUJECGEELUiASKEEKJW/j/MSh/TkSTquwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "moving_avg = np.convolve(episode_rewards,np.ones((SHOW_EVERY,))/SHOW_EVERY,mode='valid')\n",
    "print(len(moving_avg))\n",
    "plt.plot([i for i in range(len(moving_avg))],moving_avg)\n",
    "plt.xlabel('episode #')\n",
    "plt.ylabel(f'mean {SHOW_EVERY} reward')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 7.          9.         10.         14.33333333 26.         40.66666667\n",
      " 52.         53.33333333]\n"
     ]
    }
   ],
   "source": [
    "conv = np.convolve([3,8,10,9,11,23,44,55,57,48],np.ones((3,))/3,mode='valid')\n",
    "print(conv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10.0"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean([10,9,11])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "food = Cube()\n",
    "player = Cube()\n",
    "enemy = Cube()\n",
    "\n",
    "env = np.zeros((SIZE,SIZE,3), dtype=np.uint8)\n",
    "# env[food.x][food.y] = d[FOOD_N]\n",
    "env[player.x][player.y] = d[PLAYER_N]\n",
    "# env[enemy.x][enemy.y] = d[ENEMY_N]\n",
    "\n",
    "img = Image.fromarray(env,'RGB')\n",
    "img = img.resize((800,800))\n",
    "\n",
    "cv2.imshow('',np.array(img))\n",
    "\n",
    "if cv2.waitKey(500) & 0xFF == ord('q') :\n",
    "    pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "130321\n"
     ]
    }
   ],
   "source": [
    "print(len(q_table))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "130321"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "19**4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-2.7773554618819225, -3.563705005054187, -2.2330000919778783, -2.4506302861658162]\n"
     ]
    }
   ],
   "source": [
    "print(q_table[((1,3),(-2,-4))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7,4\n",
      "8,5\n"
     ]
    }
   ],
   "source": [
    "player = Cube()\n",
    "print(player)\n",
    "player.action(0)\n",
    "print(player)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 255, 0)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d[FOOD_N]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}