Navigation_Pixels.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Not complete\n",
    "---\n",
    "\n",
    "\n",
    "\n",
    "### 1. Start the Environment\n",
    "\n",
    "We begin by importing some necessary packages.  If the code cell below returns an error, please revisit the project instructions to double-check that you have installed [Unity ML-Agents](https://github.com/Unity-Technologies/ml-agents/blob/master/docs/Installation.md) and [NumPy](http://www.numpy.org/)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from unityagents import UnityEnvironment\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already up-to-date: imbalanced-learn in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (0.4.2)\n",
      "Requirement already satisfied, skipping upgrade: scikit-learn>=0.20 in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from imbalanced-learn) (0.20.0)\n",
      "Requirement already satisfied, skipping upgrade: numpy>=1.8.2 in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from imbalanced-learn) (1.14.5)\n",
      "Requirement already satisfied, skipping upgrade: scipy>=0.13.3 in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from imbalanced-learn) (1.1.0)\n",
      "Requirement already satisfied: recordclass in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (0.5)\n",
      "Requirement already satisfied: scikit-image in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (0.14.1)\n",
      "Requirement already satisfied: matplotlib>=2.0.0 in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from scikit-image) (3.0.0)\n",
      "Requirement already satisfied: PyWavelets>=0.4.0 in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from scikit-image) (1.0.1)\n",
      "Requirement already satisfied: six>=1.10.0 in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from scikit-image) (1.11.0)\n",
      "Requirement already satisfied: pillow>=4.3.0 in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from scikit-image) (5.3.0)\n",
      "Requirement already satisfied: scipy>=0.17.0 in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from scikit-image) (1.1.0)\n",
      "Requirement already satisfied: networkx>=1.8 in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from scikit-image) (2.2)\n",
      "Requirement already satisfied: cloudpickle>=0.2.1 in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from scikit-image) (0.6.1)\n",
      "Requirement already satisfied: dask[array]>=0.9.0 in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from scikit-image) (0.19.4)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from matplotlib>=2.0.0->scikit-image) (1.0.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from matplotlib>=2.0.0->scikit-image) (0.10.0)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from matplotlib>=2.0.0->scikit-image) (2.7.3)\n",
      "Requirement already satisfied: numpy>=1.10.0 in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from matplotlib>=2.0.0->scikit-image) (1.14.5)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from matplotlib>=2.0.0->scikit-image) (2.2.2)\n",
      "Requirement already satisfied: decorator>=4.3.0 in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from networkx>=1.8->scikit-image) (4.3.0)\n",
      "Requirement already satisfied: toolz>=0.7.3; extra == \"array\" in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from dask[array]>=0.9.0->scikit-image) (0.9.0)\n",
      "Requirement already satisfied: setuptools in /home/gerti/miniconda3/envs/rl/lib/python3.6/site-packages (from kiwisolver>=1.0.1->matplotlib>=2.0.0->scikit-image) (40.4.3)\n"
     ]
    }
   ],
   "source": [
    "# Install a pip package in the current Jupyter kernel\n",
    "import sys\n",
    "!{sys.executable} -m pip install -U imbalanced-learn\n",
    "!{sys.executable} -m pip install recordclass\n",
    "!{sys.executable} -m pip install scikit-image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we will start the environment!  **_Before running the code cell below_**, change the `file_name` parameter to match the location of the Unity environment that you downloaded.\n",
    "\n",
    "- **Mac**: `\"path/to/VisualBanana.app\"`\n",
    "- **Windows** (x86): `\"path/to/VisualBanana_Windows_x86/Banana.exe\"`\n",
    "- **Windows** (x86_64): `\"path/to/VisualBanana_Windows_x86_64/Banana.exe\"`\n",
    "- **Linux** (x86): `\"path/to/VisualBanana_Linux/Banana.x86\"`\n",
    "- **Linux** (x86_64): `\"path/to/VisualBanana_Linux/Banana.x86_64\"`\n",
    "- **Linux** (x86, headless): `\"path/to/VisualBanana_Linux_NoVis/Banana.x86\"`\n",
    "- **Linux** (x86_64, headless): `\"path/to/VisualBanana_Linux_NoVis/Banana.x86_64\"`\n",
    "\n",
    "For instance, if you are using a Mac, then you downloaded `VisualBanana.app`.  If this file is in the same folder as the notebook, then the line below should appear as follows:\n",
    "```\n",
    "env = UnityEnvironment(file_name=\"VisualBanana.app\")\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:unityagents:\n",
      "'Academy' started successfully!\n",
      "Unity Academy name: Academy\n",
      "        Number of Brains: 1\n",
      "        Number of External Brains : 1\n",
      "        Lesson number : 0\n",
      "        Reset Parameters :\n",
      "\t\t\n",
      "Unity brain name: BananaBrain\n",
      "        Number of Visual Observations (per agent): 1\n",
      "        Vector Observation space type: continuous\n",
      "        Vector Observation space size (per agent): 0\n",
      "        Number of stacked Vector Observation: 1\n",
      "        Vector Action space type: discrete\n",
      "        Vector Action space size (per agent): 4\n",
      "        Vector Action descriptions: , , , \n"
     ]
    }
   ],
   "source": [
    "from sys import platform as _platform\n",
    "\n",
    "if _platform == \"linux\" or _platform == \"linux2\":\n",
    "   # linux\n",
    "    env = UnityEnvironment(file_name=\"./VisualBanana_Linux/Banana.x86_64\")\n",
    "elif _platform == \"darwin\":\n",
    "   # MAC OS X\n",
    "   env = UnityEnvironment(file_name=\"VisualBanana.app\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Environments contain **_brains_** which are responsible for deciding the actions of their associated agents. Here we check for the first brain available, and set it as the default brain we will be controlling from Python."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the default brain\n",
    "brain_name = env.brain_names[0]\n",
    "brain = env.brains[brain_name]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Examine the State and Action Spaces\n",
    "\n",
    "The simulation contains a single agent that navigates a large environment.  At each time step, it has four actions at its disposal:\n",
    "- `0` - walk forward \n",
    "- `1` - walk backward\n",
    "- `2` - turn left\n",
    "- `3` - turn right\n",
    "\n",
    "The environment state is an array of raw pixels with shape `(1, 84, 84, 3)`.  *Note that this code differs from the notebook for the project, where we are grabbing **`visual_observations`** (the raw pixels) instead of **`vector_observations`**.* A reward of `+1` is provided for collecting a yellow banana, and a reward of `-1` is provided for collecting a blue banana. \n",
    "\n",
    "Run the code cell below to print some information about the environment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of agents: 1\n",
      "Number of actions: 4\n",
      "States look like:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJztnXuMJNd1n3+nqp/TszOzsy/uS+JK2kimlIiSaD1CR5FIMaIVRTJgWRBjGIatQElgJ/QDsMgEiGMgQWwgsC3AgRBBtMMEip6WbIKRJdI0ZUd5UKRERqJILnfJ3eXuch/DfczOTL+7Tv64t+qe3q6ert6p7plinQ8YTPWtulW3uur2Offcc88hZoaiKPnC2+wGKIoyfbTjK0oO0Y6vKDlEO76i5BDt+IqSQ7TjK0oO0Y6vKDlkQx2fiO4koiNEdIyI7kmrUYqiTBa6XgceIvIBPA/gDgCnATwO4C5mfia95imKMgkKG6j7TgDHmPlFACCiLwH4KIChHX/7/Czv37NjA5dUFGU9zpy/iMvLqzTquI10/P0ATonPpwG8a90Ke3bgq3907wYuqSjKevzcr/6HRMdN3LhHRJ8ioieI6IlLy6uTvpyiKAnYSMc/A+Cg+HzAlvXBzJ9j5luY+ZbF+dkNXE5RlLTYSMd/HMBhIjpERCUAnwDwQDrNUhRlklz3GJ+Zu0T0qwC+DcAH8MfM/OPUWqYoysTYiHEPzPxNAN9MqS2KokwJ9dxTlByiHV9Rcoh2fEXJIdrxFSWHaMdXlByiHV9Rcoh2fEXJIdrxFSWHaMdXlByiHV9Rcoh2fEXJIdrxFSWHaMdXlByiHV9Rcoh2fEXJIdrxFSWHaMdXlBwysuMT0R8T0QUielqULRLRw0R01P7fPtlmKoqSJkkk/n8BcOc1ZfcAeISZDwN4xH5WFCUjjOz4zPw3AC5dU/xRAPfb7fsB/EzK7VIUZYJc7xh/DzOftdvnAOxJqT2KokyBDRv32GTdHJp5UzPpKMrW43o7/nki2gsA9v+FYQdqJh1F2Xpcb8d/AMAv2u1fBPDn6TRHUZRpkGQ674sA/g+ANxLRaSL6JIDfBXAHER0F8AH7WVGUjDAykw4z3zVk1+0pt0VRlCmhnnuKkkO04ytKDtGOryg5RDu+ouQQ7fiKkkO04ytKDtGOryg5RDu+ouQQ7fiKkkNGeu6lCYFR4E6iI8cnvd+wUjDYxmCd8/OQ5gY0eEDcsXFl610PAFh/s5UNoG+PouSQqUp8BsA0pjTnjf42jV+/R+vVce0PYvbGS/TB8wUjvoZhWoQ7YMR+RVkHlfiKkkO04ytKDpmqqg/QCDXaItX767Hz9TH+bxvBT3RcnLYdjLi/+DojrhNzTp/jBhqKkgyV+IqSQ6Zu3AuSXHLDUj7u4sl/42K1Elt/lNHteox2o+10cW1Xia9cP0lCbx0kokeJ6Bki+jER3W3LNZuOomSUJGKwC+A3mfkmAO8G8CtEdBM0m46iZJYkMffOAjhrt1eI6FkA+2Gy6bzPHnY/gO8A+PTI801CjQdGq/JjXLcX93u4Tv2h95TS8EBR0mYs4x4R3QjgbQAeQ8JsOjKhxmVNqKEoW4LExj0imgXwpwB+jZmvkvDAY2YmolgbFTN/DsDnAODNf+u1k/M3o/SMXWk1klNsk6KkSSKJT0RFmE7/BWb+ui1OnE1HUZStRRKrPgG4D8CzzPz7Ypdm01GUjJJE1b8VwC8A+BERPWXL/hVM9pyv2Mw6JwF8PMkFaWIeZ2me9/r9mqQhb/3lPA4/Zmwx2giqvlfK9ZPEqv9dDLdpazYdRckgKjYUJYdMOQIP4IVq7YYs3uPUTXsiYcS14+c2kp8+/F5GViknP6eiXINKfEXJIdNdpMMB2q0GACD0A/B9twQ23Pb9Ph+BaLvb7QIAekE3KqtWq6as14vKer3BmHme5w1syzIpyZvN1kDbSn74Vbk6YdsC0R5JdI8FYeYLgr7/ph2ujjtnMFBHfhd+sTpwP3J/+H3Efb8A0G63AQBra2tRWa1Wi70P5dWHSnxFySHa8RUlh0xV1ff9Inbs2AXAqe2tViva3+l07P+eqOPU02KxCACoFpya22jYoYP4CfM8eVuDanJ4Hamiy/3zs2aFcagOy+vIIUWoopdsuwCgUHAN8X2v715N/fDaTpXvCW/nUG0veO6+C+WiPZ8ru7LmhjPhkCJueCCHM0XRzvBc27ZtG6ijvPpRia8oOWSqEr/T6eLs2YsAgFKpBAAol920VLlitqVU7Xad1K03TDmzKwulWF/U7r4FRLaOmB8r2DqlkpN2UhquXh1cRVitzpi6BfeVhRI91CAAoF53dZuNqwCA+Xl5nbJtYvx8XdCzBkzxHTSbzYGyyra90XZYLhdOhe2MM46atvsD96MSPz+oxFeUHKIdX1FyyFRV/ZXVNTz63e8BAHbu3AkAOHDgQLR/1y5j+JupzUZlpbKcAw9VVTnnbtTgYowREAB6VnVuCyNiu27qBKtuyCDV5FrFGA+larzaNMdS4M4TGvLCYQsAlGvz7jzbFkzdVaf+F0umTpFkCG8xp18wbS8Ix7wqBo13Laf1R+q8vAf5HYRIY6UcNij5QyW+ouQQ7fiKkkOmHFffQ5uMW+jSVaNGXzl6JtofHDkJwKnvANDuONU6VPWlSlsqmVvYuWsxKpPDh927zZCiWl2IyqyBvs/iLbfRM9uVqpyTN6q5VJE7TTO33xRDglZLzB54pp0Fce1wlVK95e6x0XD3yHaeX852VCol2wb3uOQwJNyOu5/+WQg3vOh3V+6vo7z6UYmvKDlkqhI/AGGtZ+exbUxpagspFW1Voi2v5LbL1oOtWHLNXlpaAgDULzgD2vmrL0Tb/pEXAVzrG9AZKJPSLqjXAQB797q58kOHDgEA9uzZFZVVy8YIyb6TvuE8PACEZ2803Tx/rWYMh+U5tyDGr7j9od+C1Go6gTnTWsMZ5xZm3PcSHSf8CcJtadCT0j+U+Dp3n0+SxNyrENH3iOj/2Uw6v2PLDxHRY0R0jIi+TESlUedSFGVrkETVbwG4jZnfCuBmAHcS0bsB/B6AP2DmNwC4DOCTk2umoihpkiTmHgMI9eii/WMAtwH4x7b8fgD/FsBn1z0X+Qism2yoysa5roZutgDQFapos2WNWE1nDJvbtd8c1xOuvXLhjz1WLsgpFMIhw4wrKwrDl2/KV5qubT9+3hgef3TEDSN67XDI4FRsXyzSKVqX3p2Lzri3e/duU7bTGSMrZbnIxwyFSsVBQ567Q+DixYvR9vbtZlHRzIy7n3CdvVwEJY170T30LTpSk09eSBpX37cRdi8AeBjACwCuMHPYm07DpNWKqxtl0mnYsbOiKJtLIuMeGxF8MxEtAPgGgDclvYDMpHPDDfsYnpFo3V4oLQeNS54vo+WICDLeoDfapWUj2aTm4Iv6xUrNlrnzhBqFlHathtMIKrZJXbE8GOG21FCsgZLhzh30xPJfa+g7/tLZqGzpslm4UzjqjHeNpouCE3RMnVJpcBpOtnffrKt/+PBhU7Zvn7hH7vsP9BsMiUbG71ZexYyl2zHzFQCPAngPgAUiCt/OAwDODK2oKMqWIolVf5eV9CCiKoA7ADwL8wPwMXuYZtJRlAyRRNXfC+B+IvJhfii+wswPEtEzAL5ERP8OwJMwabbWhdCDH1wCAETxNEe1QNr+YtaV1GLri0rdXv9/uOwg8levSO5EbIcXI/3YYn4242bFSUy5rw3GAQXkTKjd7AvfGXPfx1bdxY89aQ19Ty4NaWh4nbjWubKFBWOEfP3rXxeVSV+GXmAaL70Gq9UwvoAbOtSFLWd/7/L6bRqTSRgge15r9EFjMgn/iEKiLFTJrpvEqv9DmNTY15a/COCdia6iKMqWQudvFCWHTNVlV9nanD9/HgDQtIuPAODo0aPRdjj7IFX9cDGRnFWRvgN3vf8n023kJNYRBenPcBQK6Wc68nl0DAWiZLJcJb6i5BCV+K8K0jEkVSrGCimXRcvoQaGkJ8+J3bhAn1Ij+B8PfSeVtoV88IMfTPV8AFAupi/xm630DYadBAbDIEimEqnEV5Qcoh1fUXKIqvqZJf154nCOXGYykvPms7Mm/oAM5MnWyUDOW8voQY1Gumr0Aw//TarnA4APf+BdqZ+zWEo/AamfYB6fEvo5qMRXlByiEj/3uN9+l1NQpvAelA0y0k8Y4UcuIOrLd+gNRgraCNJwmBYPPfq/Uj/n7bffnvo5fR6tPQUJjgFU4itKLtGOryg5RFV9JaJQCBcLOVWf+hKQ2r2B3G9kR1lEESqV3HY95elsv1IdfdCYNNrpLiQCgG8+9Nepn9NPEP58OSbhaxwq8RUlh2jHV5Qcoqq+EhFa6KVVXkbociq+U/XDWP2yjnQb7aW8qKbTbo8+aEyK/szog8ZkEtkKiJIkOlWrvqIoQ1CJr0SEc/ZxGXeA+NTaoaSXc/uNhlvWW6ktDNTZWCMHQ4RvlE6Qvm/AJIKZcoI5ek5b4tsQ208S0YP2s2bSUZSMMo6qfzdMkM0QzaSjKBklkapPRAcA/EMA/x7Ab5DRY8bOpDMJcpvaOTZw5gh4/d9536rRgdDog768B4NqZJjpiERugZnqrLtkogCRmwylP+KdxFvZSxBdJ+l1k0r8PwTwW3DGyh24jkw69Xoj7hBFUabMyJ86IvowgAvM/H0iet+4F5CZdPbesCen4vnVRvIRos/pG87SppvyQqJJ4SXSnpIZ95LoOLcC+AgRfQgmcf0cgM/AZtKxUl8z6ShKhhj5083M9zLzAWa+EcAnAPwVM/88NJOOomSWjVg1Po0xM+koWWGjfl0iOWcWjHs5ZKyOz8zfAfAdu62ZdBQlo6jLrqLkEHXZVQSj5MD4bqjpO66mTzJr+RaAkkyKaVx9RVGGoBI/92xQyo/wBuQMyBaaiJ/dBEhRM9n6T0VRlNTRjq8oOURV/cwi1b60f783pt5LehNYAJM2/kTi5aRPcD0Ls4agEl9RcsjW/zlWEhBKggn/jq8r6eP3ZcNsln4cv0mQZNoxqaFSJb6i5BDt+IqSQ1TVVzbI+rIjyIDrXlYWErGX3sBJJb6i5BDt+IqSQ6au6nvB1rfzdin92O1pwyPaGBeElGMtvq4ssHHzZUx4GVefrKop99OIhSNzhSaA/lj7pUoZQH+Cx7m5xWh7ZdWE66rNuDJu24w97MJkeT3XtjJMos6qaC+3zbXb9atRmU8m/n+17F79Ru0nou1zS8cAALv3u+/3/JXnAADbd7k69c5atH11xcwK7Nv9FlfnnPle5mdvjMpWr7rcA/MLJnvPlSsnorLtu819+yWXxPPqigtsVesexCjUqq8oylCShtc+AWAFQA9Al5lvIaJFAF8GcCOAEwA+zswj8w0nSAaiJMAfIWnjpHu8xBfn9M3D6Zfo/We4tiwufLbUNi6vLAMAAvGq+TASnwruRIG/Ldr2iia3tgwn3eQ6AKBA9ais23VaRNAx0r0scvjVSma7tCAbbP6tCs3zlbUvRNs3/sRhAMCJ4+dd2WvebOo03L3uXdgebe+aM5K6y649h8xpsLLmUlEszrv64Ve0f7u77wvnjQa0a/uNUVkFe6Lt2VmnZQzD95LJ8nEk/vuZ+WZmvsV+vgfAI8x8GMAj9rOiKBlgI6r+R2ESacD+/5mNN0dRlGmQ1LjHAB4iY8n5zzZW/h5mPmv3nwOETpJxsjAaoZELSwbV+n4VPW5/eIDYFzM6kFXD88jzye3SnMmqw0ExKmuT2V5ruZj7XRF/nz3zWhaLTm0vFsI2OoNgZc6p/UxG1e9alR8A1gIj1zoFlwa77BvjoOe5VI+H59wX06qfBgDs3rUjKru0ZFT42dnDUdnqK67rXK1fMtdhd+3KvDEolqpCPS861+D6ihnOlLfVorLZ6j5zvosu0eh3//qlaPsffSS9dN5JO/5PMfMZItoN4GEiek7uZGamIeZdIvoUgE8BwNy2bXGHKIoyZRJ1fGY+Y/9fIKJvwETXPU9Ee5n5LBHtBXBhSN3+TDpjLOncNBLFNttcKBhMWd3HiJyCcVpNf548e9yIdM/RXnE9WaPeNFK5UnY/+iUryecXhEGPyqKW1SJ6Tlq2Oq8AAGa2Oc2gUGpF2z0r8angygIYqd4TM5/hGXtdd57OEZfrb+8NrzFlYqpwx3Yzrbi2eikq++KXXBqJm958s7mHgtMiTp0xxsG///73RmWLi2J6sme+l2//2f927WibOgVyU5Zra06LaLVG66JBQi/Ekb2QiGpEtC3cBvAPADwN4AGYRBqAJtRQlEyRROLvAfAN+8tfAPDfmflbRPQ4gK8Q0ScBnATw8ck1U1GUNBnZ8W3ijLfGlF8EcPu4F8zCog0vAxFZeJSqH8NotT3OM2+w/qjzSDyr1nvsVPCOzZpcEAoniSFgJfKqc+r4TNl4vfliSNFtuucU2BerXJyLysrVOdsGp8rXbZ162825/+RrfzbafubIU6ZuzXnZ+YUlc71gOSr72Z/7O9H25+8zyu5c7U1R2fa5vw0A+J9/4a7Tbb8SbR94jTHkrS25JNNdGFv5vte47+ItN7822gY534LhqOeeoihD0I6vKDlE1+PHkIX12d6IvPOx6jiP2O+F+0SRN6j2x6n/w6gWzdxzt+Pm3DtsVObFBWcFR8/NzxfsUGttxVn1ybr5BnVn/SfPqfB+cd5cp+f211fNvPlqx1nj2TdDpLmyu/bzx5xdev9BM38fzhIAQNfOz5cqbh6+QG7+/Z/+MzNUKHhuEc1n/8ics+Dvjcp27XJq+/GXjgMAWl2n/l9dOwkAeMfBW6KyQ4fno+3G8stIC5X4ipJDNkHib31pmoU2+iN+suMEcVKjXL9E54HyuIU7fSXCANd82UjyUtkZufbvNN/ve96xMyoricU1a1fOAQCOPH08KmutGkle8Z3UvLrirtOxtk6/4p7djGeXBMOtHStYqT075zwJ7/iYk8pnXjYSuNt2kvYvvmn81ebn3yDu0mkWnvVc6AXnorJf/ufvsDudEbYnFhhduWy0ntma89Ij7AYA7Nl9ICo7depstD03M9qgG+eRGYdKfEXJIdrxFSWHqHEvhiwkUSyOcCt2arnUy9dbmAP0bASekUZAecYRi3RuqJl56kbzZFTWXjbe3dRwcmd+0c2RHzho5vz3iwg882VjOGsuu7LH/6+LTnPiJTPHvW/BzYvf9PYbAQDbbnBq+1pg2tHoLEVlq3U3DJmdNQbDv3zou1HZ7t3mPGvLrr0doXUXK6Ybtbsrbr9nF+nUnP9Cs+GGHK95gxleXDjvvpeitwsAsHTRnbtQkJGWNNimoigbYMoSnzPhFZd2G8fxdEtKEKw/nbdVaHeNEUxOsy0vGwn7xA+eispuudlNj83sMa9lVRgEm/WTtsw9m/feekO0/ff+7iEAQI/EIp3Si6asJ+LWFY0Briri2j378l3R9gvHzDLY1qrTLMhaUksV9xw7IoZgvWmeRbXqDHWw9y3WGaFErr1Xzga2zN037CIpJtc2vyh295J4a6pxT1GUIWjHV5QcMnXjXha84ggpq9GTsBUmnK/dbJaaRtetll30GN83Ku9yyy2Eee6Em+O+umLU6BsWXZ39i8YYtnbZ6c69rvOkC2zwTM8Xi2vsPD6RM7r1AnOdbtd55j31mFPhl6+aGAHtnvt+K2HYAN95F6Io/Bu6VdMGdu3lXhh5SKjyEIa6cEjiXxG7Lw2WeSLKUGsfRpLwtVCJryg5RDu+ouSQqar6BAZh/HXk04aCrT8c4Qx8jwCwWjBqu19z5mkfZiHMasOpsRdfcN/58dOmzsEbnGr8+oNGHd+707nXFspOrw3DyXvijW62jWp+7qyz9L940myfO+/qLq2Jde72koVq1ZXZDELtrnC/7YoMRPZZtHsuY0/g2SGHcNPtw2b0geeGLl44JBFBO8Fu6JLEHTfpAFAlvqLkkKSZdBYAfB7AW2B+VH4ZwBGMm0mHAeIMSKqU25h04cQ4BBnwhwCAoGakd8N3y2CDlhGr7d6uqKwYCO+6NVPn8gkn+V5aMpKvvvrDqIwCJ8lDyVgsOH+BaskY2HxyQT07vXcBAHrCfuvPuOsUiibQJcMFvGy2jXxsd0RePmGoI5uBqCdDhIcGOs9JbJDb78VlIOrZ7tidE6Ximim+Rkkl/mcAfIuZ3wQThutZaCYdRcksSaLszgN4L4D7AICZ28x8BZpJR1EySxJV/xCAJQB/QkRvBfB9AHfjujLpMKjXGX3YJhOMiG4zLpNQ9TMQsxQA0O2a0V8rcHPcQdu0Xqrg5ZpItmJdVxstt3CnUzfq+Px2twgnEP6wnZaZ0282RcLOqzY5Z8+pzgUybrUF382vt/BMtO33jFFPuhh3WmFoIjdcKZbcUCCM39DrCUNemEBTuBCDxBDSNpNEhiEE1qDIIscAuyEFF8TqnWGkOI9fAPB2AJ9l5rfB5CToU+vZvNlDM+kQ0RNE9ES90Yw7RFGUKZNE4p8GcJqZH7OfvwbT8cfOpLNv9w7mLCwuSdm7cBISPxt+e0CpbXPElZ3k6rKRnBw47a/bckIhnE3tipDbftlI7eNn3dTbTMVJyxm7hLcoctFR2+zvtNxxrbaR1M2eePVbLlZe1y6oYnLvABXMNokoQWHsPgDoWi221XJlpZJpB7GLC+iJhxZYmUvs2kE21583JJd8kEBMpzadx8znAJwiojfaotsBPAPNpKMomSWpA8+/APAFIioBeBHAL8H8aGgmHUXJIEmTZj4F4JaYXWNl0mFwJtaRpz0cmcR6/Kz4Xs13jVpfKog02V2jGjc7bk170HVeb23rR9ETiqtfMsbB7TudEVCuT19r28U3bWfwC+xCm6LnjGWVmaptj/ArEPPzUbpucsMQKoReeG5RUEcYqdt2UReX3HNmzxozAzE8gPsOKAi7njvnqNckzSFjNt4eRVFSZeoReNL2ipsEFKRrOpuEwA8ykMobAGbtFBW1XXt7XWPwKwppV6g4yVi0ErQhJDpbz7xGQ8T2IyG3rCGwIGbZCjYZiAeRvy4wxsFQ6wAAj9xUY6fXsud2xj1r20NPeN4FLNpuDZdV4d/fWrOJMoTxjoX0hzVw9mkEXOy7l2u3mRN0V12WqyjKMLTjK0oOma6qz26ONrVTTiQSTbq6Oac8dDBs/SETAJwrxrSzGL52ci5dHmCMcX1SyWrZJYyPNKp5dvGNPHc32BFt+5FLnVhQY5fjen1eeE7Vhx02tJuijrima4fYjt5b1yF6dhg8LEpV0KvGlvdfI9m7qxJfUXKIdnxFySGaSUdR+lIHhWq2VLfjVG8pM/2B04xOvGrrsz9QxsPksRxeDEXj6iuKMoSpSnwG0EvZujcZ4166ZKGNeSYQ8/yhwJQRcoLQ0CefY6wNzR/YHmpqi+bnpewNtQAaLAMAT7RzKMn6l0p8Rckh2vEVJYdM3bjHQ9YaX//5Uj3dRMhCG/MMYTBADA/9EJbJ93hwzt5VHaK2h9fmGNkbVwYAnhr3FEXZAJvguafGPWVrwbHTZHFGN1kWI8k5ps5Q1tN8498XSuCsmfRVU4mvKDlEO76i5JCRqr6NtfdlUfQ6AP8GwH/FuJl0wKmrvVlQo7PQxjzDFBdxSaji4Tr4YfPrUXmM+i7U/4DkMDdcDCTfjbhhsIgL0EuyRCmlRTrMfISZb2bmmwG8A0AdwDegmXQUJbOMq+rfDuAFZj4JzaSjKJllXKv+JwB80W6PnUmHAfQyELM+bbLQxlzTF1w1lIVS1Q/6/w/sN3WC/hX34cljyoAowEDffmu2p2Hr8Wtxpf0fE/rJJJb4NrT2RwB89dp9STPpNJpJHBAURZk040j8nwbwA2YOU5mMnUlnz84FHuqVdL1kQZpmoY15hmLm52Pn5IdkuIkktKwzuLxXBvCMJuWlD0E0US+PE/Wj9NlSysuFQcPbeC3j9MK74NR8QDPpKEpmSdTxiagG4A4AXxfFvwvgDiI6CuAD9rOiKBkgaSadNQA7rim7iDEz6UzCZTcLTOKeJ5OdJ5+QyEbpot/EzdPHGe8AjgKfykSaRoWvzLi5d/Lce9Bs2cw/gVP1y5Xwmu64VtutwfdsklGGzDfQb0TkhMZz9dxTlByiMfcUhYVUjspEHrxY5Uoa4KIjo6JemPmn6UJye0US2+a/L07TtCnFPaEZFEuubR4GZ8XIev5F/z1dlqsoyhC04ytKDpl+sM1pXnCLMAlzppr20sMP1pd/HPME+5KWWoOanKdf3GHm3JcuLUVlzWUX6Wdm1iTqrFaFN55NJtpqO5W+23VPuttZDa/u6lyj6gdq3FMUZRja8RUlh2xCsM38ua/m8Z6zRHwAWOEqG1nr5dz9oFutfMyXLl4BABSLLtHlzI7t4qJG5q6uunn6ZsOUzczsjMq2L7g6jbUr9jpuwBzYBUaMjm2LJs1UFGUIUzbucS499yaBfo/p0RfEMi53nj1AGu8CuZ/7/gEAuvZDoedCb7cawkOQTSrwanExKqtVjXbQaroGnTlVj7YLnumuvu/OWSiYRTp+wdbhaxftxKMSX1FyiHZ8RckhatzLKPo9podc9OKi7AxGxuE+g9/gUItFd6rNLAAAul1X1qyLOfluxWxU56Myvz0LADj/8qWo7PiJ5Wh7pmIMeLWaMxhuXzT+APML5j/z8Kw+EpX4ipJDpp9JRyVVKqjETw8/EMtcQwNen2feKInv2TIn0ZevrAEAarXdUVlVSPdLrxjvvBePnYvKrl4x1/H9bVHZbPVQtN3prAAA2k23cKe+aiS9T0YLCLrJurRKfEXJIdrxFSWHJNILiOjXAfwTmKnKHwH4JQB7AXwJJjLP9wH8AjNrGF0le8SGs44pGxL2Oo4dO4z33YsvuBi0J088H213W8aQVyq5oUC5ZOb0iWejMu46Q16tbAyGvW4nKru8ZBb2LJ0zBsFWKy4r0CAjJT4R7QfwLwHcwsxvgQnr+QkAvwfgD5j5DQAuA/hkoisqirLpJFX1CwCqRFQAMAPgLIDbAHzN7tdMOoqSIUaq+sx8hoj+I4CXADQAPASj2l9h5lCvOA1gf6IrppxJZxJkwWDu8WAM98DriDL7aEharPvOYOqIeV+/YNTKtRWnLs4tuBirPVvycH9wAAAGPklEQVRcb7mFJcWicRENg0sCgF90bfNWzPlrs05lXb56wV7PvQvVGfcqdrrm/L7v3E+7HXOs54l5apav72Cgymhb+uTSYAabOtai7Z51ed2xY29UduGCWQdfFGp5W6jgHBjVvBs4Ff30RdOeC0tukU2949pRKhvLfeC79fjdnrl2qTQTlVXn3EwAwbj51uunXNuWnwMA7Dto7ocuyHdgOElU/e0wefIOAdgHoAbgzkRnR38mnWZLTQCKshVIYtz7AIDjzLwEAET0dQC3AlggooKV+gcAnImrLDPp7Fqcz4AszQhS4lNcWbg95CuPFnO4VyDomTqdjqvTrMtQzuZCva67TsE39VksGgrPAwBk1ScpvUs2gCR58YaocAGSDCE+Sb8FGQVntW6j4IiFMmwj9FTKbn6d4ebSL5w3C2nOnn8lKvN9I+nXVly7g0BoV56pP1N1WkKhYCR6seC0iXK5Em3X14zgLPjunPMLJtLPgQNGSzhyLD3PvZcAvJuIZsg8idsBPAPgUQAfs8doJh1FyRAjOz4zPwZjxPsBzFSeByPBPw3gN4joGMyU3n0TbKeiKCmSNJPObwP47WuKXwTwztRbpCRErLsOVXwaXCPenxFG1rflwri3bd7MIzfrV0Wdktg2dXwS68F9sz/ouVdJeruurV0EAMzUnMoaqvCeJ9eny6gy5j7k8GCSmYOob7hjjGPttmvP2qop88gZzupN1/bLF00QzaXzq1HZgo2c43vOUFeqOLW+UjYqeqXihhlFq+rLUAv1uluPf/mS+S57uOzaS8aN17NDBxo2tLsG9dxTlBwy5WW5nInFJRloIjiQkVZszDepBYRid1hGGGvcYzEldvqUCQV96qTzNmM7hQQA4VqWnliMUi4b6d9ouCkxeZ1dNfNBTueRF6aIloteBiW+JNQO+t6fvgc1GBcvaVmj7iR5YKfUgq7TakJJe/GV81FZu+MkdadtpPq2mpsC9MlI9HJlISorFpz0971y338A8GyEnXbbheFeWXHaV8ELY/vJ1Npde54w5p5KfEVRhqAdX1FyCE1T9SaiJQBrAF4ZdWyG2Am9n63Kq+legGT381pm3jXqRFPt+ABARE8w8y1TvegE0fvZurya7gVI935U1VeUHKIdX1FyyGZ0/M9twjUnid7P1uXVdC9Aivcz9TG+oiibj6r6ipJDptrxiehOIjpCRMeI6J5pXnujENFBInqUiJ4hoh8T0d22fJGIHiaio/b/9lHn2koQkU9ETxLRg/bzISJ6zD6jLxNRadQ5tgpEtEBEXyOi54joWSJ6T5afDxH9un3XniaiLxJRJa3nM7WOT0Q+gP8E4KcB3ATgLiK6aVrXT4EugN9k5psAvBvAr9j23wPgEWY+DOAR+zlL3A3gWfE5y7EUPwPgW8z8JgBvhbmvTD6fice6ZOap/AF4D4Bvi8/3Arh3WtefwP38OYA7ABwBsNeW7QVwZLPbNsY9HIDpDLcBeBAmpMcrAApxz2wr/wGYB3Ac1m4lyjP5fGBC2Z0CsAizpuZBAB9M6/lMU9UPbyQkeZy+LQYR3QjgbQAeA7CHmc/aXecA7NmkZl0Pfwjgt+AC0O3A9cZS3HwOAVgC8Cd26PJ5Iqoho8+Hmc8ACGNdngWwjI3EurwGNe6NCRHNAvhTAL/GzFflPjY/w5mYJiGiDwO4wMzf3+y2pEQBwNsBfJaZ3wbjGt6n1mfs+Wwo1uUoptnxzwA4KD4PjdO3VSGiIkyn/wIzf90WnyeivXb/XgAXhtXfYtwK4CNEdAImMcptMGPkBRtGHcjWMzoN4DSbiFGAiRr1dmT3+USxLpm5A6Av1qU95rqfzzQ7/uMADlurZAnGUPHAFK+/IWy8wfsAPMvMvy92PQATcxDIUOxBZr6XmQ8w840wz+KvmPnnkdFYisx8DsApInqjLQpjQ2by+WDSsS6nbLD4EIDnAbwA4F9vtgFlzLb/FIya+EMAT9m/D8GMix8BcBTAXwJY3Oy2Xse9vQ/Ag3b7dQC+B+AYgK8CKG92+8a4j5sBPGGf0Z8B2J7l5wPgdwA8B+BpAP8NQDmt56Oee4qSQ9S4pyg5RDu+ouQQ7fiKkkO04ytKDtGOryg5RDu+ouQQ7fiKkkO04ytKDvn/Tks/L7R2u+YAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "States have shape: (1, 84, 84, 3)\n"
     ]
    }
   ],
   "source": [
    "# reset the environment\n",
    "env_info = env.reset(train_mode=True)[brain_name]\n",
    "\n",
    "# number of agents in the environment\n",
    "print('Number of agents:', len(env_info.agents))\n",
    "\n",
    "# number of actions\n",
    "action_size = brain.vector_action_space_size\n",
    "print('Number of actions:', action_size)\n",
    "\n",
    "# examine the state space \n",
    "state = env_info.visual_observations[0]\n",
    "print('States look like:')\n",
    "plt.imshow(np.squeeze(state))\n",
    "plt.show()\n",
    "state_size = state.shape\n",
    "print('States have shape:', state.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess image: YCbCr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJztnXuMXcd52H/ffeyTSy6XoihKpEm5ViRLaiTLlCLHaeJIlqPYgu20hiHFSIzErRE0cZUH4Mht0SRo/3CAIo6QBkZYKYkc2HpEkWpBCey4jIKmaCCLsmW9qTffpEhRS3J3yd37+PrHzDkzyz279+zuvXf38nw/4OKeO+ecmTmPud8338x8n6gqhmEUi9JKV8AwjO5jDd8wCog1fMMoINbwDaOAWMM3jAJiDd8wCog1fMMoIMtq+CJyq4jsEZHXROSudlXKMIzOIkudwCMiZeAV4BbgAPAUcIeqvti+6hmG0Qkqyzj3BuA1VX0DQEQeAD4FzNvw14+VdPOW5RRpGMZCHD5Q590TTWl13HJa4SXA/uj3AeAnFjph85YK33p80zKKNAxjIX7xtqO5juu4cU9Evigiu0Vk9/iJZqeLMwwjB8tp+AeBrdHvLT5tFqq6U1V3qOqO0TEbRDCM1cByWuJTwGUicqmI9AG3A4+1p1qGYXSSJffxVbUuIr8BfBcoA3+uqi+0rWaGYXSMZZnYVfXvgL9rU10Mw+gS1uk2jAJiDd8wCog1fMMoINbwDaOAWMM3jAJiDd8wCog1fMMoINbwDaOAWMM3jAJiDd8wCog1fMMoINbwDaOAWMM3jAJiDd8wCog1fMMoINbwDaOAWMM3jALSsuGLyJ+LyNsi8nyUNiYi3xORV/33+s5W0zCMdpJH4v8lcOs5aXcBu1T1MmCX/20YRo/QsuGr6v8BTpyT/CngPr99H/DpNtfLMIwOstQ+/iZVPey3jwAWHscweohlG/fURd2cN/KmRdIxjNXHUhv+URHZDOC/357vQIukYxirj6W2xMeAz/vtzwPfbk91DMPoBnmG8+4H/hm4XEQOiMgXgK8Ct4jIq8BH/W/DMHqElpF0VPWOeXbd3Oa6GIbRJazTbRgFxBq+YRQQa/iGUUCs4RtGAbGGbxgFxBq+YRQQa/iGUUCs4RtGAbGGbxgFpOXMvV6hPP8CwbbQQJZc9kLnznd+q3MMYzmYxDeMAtJViS8sXjLnlXzzHdcuTWAhqbxc6Zx1/nK1CMNYCJP4hlFArOEbRgFZlca9XlZj26Wix8d12nBpFA+T+IZRQFalxO+EhFuMFrGQIS/et5R6mvQ2VgN5XG9tFZEnRORFEXlBRO706RZNxzB6lDyqfh34HVW9ErgR+HURuRKLpmMYPUueSDqHVfUHfvs08BJwCassmk4DWfCzXMrokuYg9LKh0jh/WZRxT0S2Ax8AniRnNJ04oMa7FlDDMFYFuY17IrIG+BvgN1X1lEiQZKqqIpIpDlV1J7AT4Kof7+uYZWs1Gs1WY50MA3JKfBGp4hr9N1X1EZ+cO5qOYRirizxWfQHuBV5S1T+Kdlk0HcPoUfKo+h8Gfgl4TkSe8Wn/ERc95yEfWWcv8NlWGSm9PSsvD+1aYmvdBKOT5Imk839h3jfXoukYRg9iU3YNo4Cs2JTd5aiyne4uLFS3VmUvt26m4hvdwCS+YRSQrkr8upY5Uh8BoCxuMs+wzKT7h0o1n1ZP02IJOt7sA+CsVtO07ZUJACab4T9sSudeVlXC5KEBafjvbOm6vz7k61FL09aVanOOS3KczCgPoOqPGCmFsqd9kWe1nFm3hoo/Luw/6/OP78XG8pnoelymtehykjrF93KkFM4/0nD5vzwT5l1d0Xc08zqM8w+T+IZRQKzhG0YB6aqqP1JqcvPgNAAT6r4P1YN+eqI54L51KDrnbLq9oeTOWVcK3YM36v0A9DFXlYegHjc1qLlJOXGXIVajr+s7DcCRkA1v1NcBMNnsj8px6v+G8mRU39AlGPEq+HjUDTnty6xFqnzMgFfNhyIV/YJy0gUK+eypDaTbJd9VqM3qHrhyTkfdlRkN9zLpAlzbfyg6J7tOxvmHSXzDKCBdlfgnmlUemNgIwEWVcQAuLp9O92+rTAFwOpKQJyMJ+5aXurXImDZWdsa92EBWiobEsobXxspO8m30GgTA+nLQMp6dmft/+N7KSQBGS2HfUMlJ1RONoIHsqa0N9a1dAMBPDLyVpiVllucZ9UuMlLGBcv/MMACnm4NpWiypE0NerPWMeK0o1nTie5kYUkdLQbM4ayOJhcEkvmEUEGv4hlFAuqrqHzo1yn/e9W8AGLnYqfi3bXsh3f+JdW4N0OXVMEZ9UTmo42c1pCfsrTv1d6AULHFjpdiY5lTZI42g5u6bcd2NZyLjXl9kENxecWWe0qib0XBdgWb0XzlScvXZWA7nXl49lW5f0+e6Ic/VQjdiY8l1Z4Yl2ylJMuZ/UWTce5/Pc1rHw3VFpyfqfCmalzAWqfAJxyNj5VSzOme/URxM4htGAbGGbxgFZMUW6Zw+7izVD57+YJr2QON6AHQqjCeXzob/Jml4C3UpqLTNAbc9fHEYHbh120vp9qdHnwbCiAHA9soxAGY0yieqW8Mnb4u6D0P9zgo+1Qzj4od8fcYja3k8Fp7MJ0imFQMkCvbRRlC1X69tjM536cmoB8DWilP1hyJV/nSkqo97a388gtFUd73xvIKhSP3v10jv99RMDhQGe9KGUUC6K/FVKPkxcq076aRnotliXqDFQ9zaH0t3J7HKA0FalQ+4GWyT+8L4+cPHdoTtitco6iFXqZXmpkUiv3zG1217MCZ+4vLnAfjX63enaZf4OQRj0ezCeHFNYnR7q74mTbui6mb5XdUXbv3FlcPp9rivR7w8d0Zdfd9oDKdpV1bDbMEtOG0ndmJ8ouFnJzZCOeuieQt9/oKTvI1ikcfn3oCIfF9EfuQj6fyBT79URJ4UkddE5EER6et8dQ3DaAd5/u6ngZtU9RrgWuBWEbkR+EPga6r6PuBd4Audq6ZhGO0kj889BRLrVNV/FLgJ+EWffh/w+8DXF8xMlGa/10cXclQTTR2VZqSOTzo1ujkRqi0XOzVba0HFlqj7IBPu/NQwCGjVFZDWBdBq2G72+//DiWBA+/az17jvxrWhcr7bIvW5eQNQcdvrN4Wx/Z/c/BYAHxt9Lk27pBK6LokxLp6LMN6cuwjnu1PvSbd/ZnAvAP+iEuYLNPyCnEONkTRtiGDoS6b3no7mMsSLm4zzm7x+9cvew+7bwPeA14FxVU3MxAdwYbWyzk0j6TQmJrMOMQyjy+Qy7qlqA7hWREaBR4Er8hYQR9Lp37ZFE0mfSuB6hugvB6mp0TapwAtpcswNpUXOZdBKZBAcSqxlUT7JZmzcm4xuRTJsFmkJTPv/yLi6iZKgseEw2l9z6af2jKVJf3vUGSH/tnJ1Ztky45cRD8TWxrn16dsQDIqff/+TAPyyH7oEqPlHGy9oqkaDltXz28u50YJFmXRVdRx4AvgQMCoiyVu1BTjY5roZhtEh8lj1N3pJj4gMArfgIuY+AXzGH2aRdAyjh8ij6m8G7hORMu6P4iFVfVxEXgQeEJH/BvwQF2arNV7b1ESdrrZYBK7zbCfZDbSIwJvsbi6s28bGPdGcenBG1yOzCkNR3sla/3jNf9xN8fMWZtUhoztUOx7W5t/zTx9x33ykVY0XpHqBm7dw+xWhy/C50e+n21NN97rEi5eSWYnxkp9Xo3kLo9Ech3ZQpcXzXiVMr3JvRnms+s/iQmOfm/4GcEMnKmUYRmexaVuGUUBWbJGOsfpo7HNTgr8xdWOa9q3+69Pt+qRX6OOuh58+LXF35WxQcx/9uT9pax2nWd0qdMJIRhyG5ZJHSueNxGQS3zAKiEl8I6Ux7KX3VHgtmuPBbFfxcwzidT3NWrJUOqSVZoL4/4XvfqmtdWy3BgHZUZKWy6HGUOuDOkDepdUm8Q2jgFjDN4wCYqq+EUjEQAgTMGvKdH3UdwXixU3J7miehIYpBsjZ9hrj2t11ALj/YwuvLVsKcVDTblIx455hGPNhEt9ISRYIzVrCXJ4rQXQ68oM4nbE0OVokFW+3pY7T7ZdVd+z6tbbn+fBH/7TteeYhK3JUFibxDaOAWMM3jAJiqr6Ron1OLdf5DETJwqF4nUySFC1Eoj/y5HOmvcY9HWi/lyDJCJK6XD7znd9oe555tPgjJ+/OlZVJfMMoINbwDaOAmKpvpCRRi2ZZ4mP1MkPLTroHs1ybRaMC0sIPwqKptV9WaX9vrPFvJybxDaOAmMQ3UtTHJJwl8eeR5Ofujw1k5YmwXV/XZmNcm+cFAHTEqU8nnJm2Mc/cEt+72P6hiDzuf1skHcPoURaj6t+Jc7KZYJF0DKNHyRtQYwvwCeAe/1twkXQe9ofcB3y6ExVsWTeVtn+KilbUqflNwqcm4ROnn7NfRdNPfaSRflBW/6fUgY904NNG8kr8Pwa+TOgNbcAi6RhGz5LHr/5twNuq+nSrY7NQ1Z2qukNVd5TXDLc+wTCMjpPHqv9h4JMi8nFgAFgL3I2PpOOlvkXSMYweoqXEV9WvqOoWVd0O3A78g6p+DoukYxg9y3Im8Pwu8Nsi8hquz58vko5x/hMbpFbacJfnU0AWNYFHVf8R+Ee/bZF0DKNHsSm7hlFAbMqukZ/iTnE47zCJbxgFxCS+sTCtpHxBjWMrQhvvtUl8wygg1vANo4CYqm/MxdT71clKrMc3DOP8wSS+kR+T9OcNJvENo4BYwzeMAmKqvmH0ShfGjHuGYSwHa/iGUUC6rOoX25llO1FZWD/NvM8t/MeXGj6STin2q59RZpx1q8fpA2jKRHjVdNClVU5U07T6hlqox0l3bHNdPU1LI+jEdYuj9FTcxUlfuEiddpUvTYSL0NLsOgCUhkI5lb0Drj7vOZumlQ+6tNqmmVBeFAy0Ou62dfuZUM7BQQAaF4RzSuPherlg2tX3yEA45yJXpkSxDJrH+8P+vvb1SUziG0YBySXxReQt4DQuelpdVXeIyBjwILAdeAv4rKq+u3BO2lJSrQZ6Qitp8Zetzbn3WUoLX1caSSd+RksRDZFmUX7HS7nonja8tG1WQzlSiSR1f0bdZ/z55ZBPaSpUrlQr+zyjavjQ3c3BDFUnigqUSHmAvitPuo0X181Ja0wE6Tv2ntPp9szFrux6I9Rn5KoJAI6PrwnXsCloGckVDr7vZJo2eWAEgP7NwRv1maHQRKvDQSuaD8kZaWgxj/VnVfVaVd3hf98F7FLVy4Bd/rdhGD3AclT9T+ECacAKBtQwDGPx5DXuKfD3IqLAn6nqTmCTqh72+48AmzpRQWOJZGj1sQqfFb56llEvTWxRTJJPfFykWTeSoJmxtu3V/vKZSP0/EUIvJga4Wep/RkiGRmSgayTlR8E70zpVQz6lPndOKTKgrdkylW5PnnHqfO2SYJSbOeLU9b6xYPA7MR4qpOOu7ml3BJhaM+TSBiIjYnQ9jZPunEZfZMBc51T5MycG06Ttj4TdBz9P28jb8H9KVQ+KyIXA90Tk5Xinqqr/U5iDiHwR+CJAeWx0WZU1DKM95Gr4qnrQf78tIo/ivOseFZHNqnpYRDYDb89z7k5gJ0D/tq3aE4azXqDF0NxS7rPUMs7Jm808mkEylNYcCpJP/BBf7YLowHJGBrUgvWXKD5+tDRKyFEly9ZuzbJq+7omUj9MaM2E47uQrY+l2/3ZvtIu0n6ELnbHtzOlgBHz/7x9Lt4/e4qLHNarhZm14yWkMr98RyhnZFDSLk3V3bRv/cihNK59NjJ6hun0ngpZRmwrGxfnQrFDmGeQJoTUsIiPJNvAx4HngMVwgDbCAGobRU+SR+JuAR12AXCrAt1T1OyLyFPCQiHwB2At8tnPVNAyjnbRs+D5wxjUZ6e8AN3eiUitNT8w1yDDOtT4p53GxHhidkxr/FlG0knHOaa/LxsbEWEWNDGJpPmsSo1xQ75tRV0D9tkTq/8AaNztuoC+Mf094dbk2EfTpKz64N91+efc2l99wyGfGG+X0bFDbX/zyRen2j/37fwagsnVLOGf7RgAufSjkU54ORju52tWj/52g/pfOum7M+JUjadqhfxXmASCtx/HzPhubuWcYBcQavmEUEFuPn8F5MfLQ4hIyuzPJ2xCLg1gdT/JcxJTe8oBTXxvT0UIZr9YPXBAWtTTqUUY+/9r4wJy05kxk8o7GxUu+nPi6z/jzz56NJgEk+/tDd+Llp7an29XtbqptOepKNZKpuFE3Ir4vr/zP62fXAbjsV19wp6wJZdeveE+6feFuV05pIljtOewGxk7ddlWaNHzViXR7/Fik9s9Hzl6qSXzDKCAm8bNY/ba97Fl2Mcv5S8+S8hAkfau84zkGh5zUlWjhTfVCJ+n/7IN/lab1ESTw/5u6DICvP/vTaVp90kn68nCYUdc4Fc328xpFvNw23RfPEfCGw7Xrg1HtRx+/P93+xik3uWCqGcbMv/bIJ115F0az7GbNF0i0kaDV7Pkf3h4e36vonL5jrunV1oVxfC2tB2Bs6/E0bfzVMMeAoRaTN86t1wKYxDeMAmIN3zAKiKn6WfSCba/VX3beMffYUJcY2OI5ArHanxg9Y40zaz5BtL+50avmp4JRbmbcqdH7axvStOsH9qXbXxx9EYCrb9ifpv3LPufq4Y16UI3/3Q9+OVT9VTf2XdkU1rL/1x9/DICfGTycph2ou1d+X319mvbM9HS6/WN9RwG4Y9evpWmy1XvLORUv9g/XrX46spwpzdkfdz3Kp0NXoHy5mxpc3x8Mdjrg7vWJg2FNi1Q70+80iW8YBURUu2fJ6t+2VTffdWfXylsttDTELSnT9mfZEfqd+C+dDsplshy3sSUMZX3puifS7c+MPA/A8UaQsG83nGS8pHIqTWtE6kzZ35CaBllWlbnGsGR/fG5iTAS459WfBOD0vrVpWurrLjISpouGAElWHo9EC5FmVkamHv7q3Uzv3d9SZzWJbxgFxBq+YRSQ7hv3ekFFbbNxb0kLas4TSuPeVfZAePB17xBSp8Lr9803b0i3X9jo1rd/ZDT4e/mFYWeg2z0TFrqMN4Kh76y6bsFAtJDloopzZNkXWRsn/XHjzXDu154Oa83kuJsbUIm86dT9wqBZsx1nbSeL/OMVTed8Q/Z7ldUe5ntd8rQdG8c3DGM+rOEbRgGxcfwscsyMNPIhdT+eHbu/Sozkp4JlfPz5MKa/a8iNse/etjVNe/PSZwC4beRHadqGaljkk1jwRyIV/FDDqe0PjV+fpj36mptKO7M/LJ6J1fpkUKA+GPJJpvxKtJAoua6Y2NlmJnm7ufN1D9rYTTaJbxgFJG8knVHgHuBq3P/OrwJ7WHQknR6h3QbI82GZ7xJpJotzIsNXIhlLGVIToHrayaOJl8PsunsPuwU73xj/2ZBPvB7HFxM7qmx4LzoaRZdJJLVERTeiBUTNgbnqnky7+pTmkehBI4j2d+KRr0CY7LuB76jqFTg3XC9hkXQMo2fJ42V3HfDTwL0AqjqjquNYJB3D6FnyqPqXAseAvxCRa4CngTtZSiQdZfZY5yolY6bn8uiFuQsdopT46m9GAS69ytyMQnA34umu/h0pT4ZzkgUutQsjh5NRHADxjjDj6DzV8blyrenf+GYUcnpWl8Nb9yR6T8szs3a5+vbHxj+fT+wLMzl9ueP4i3132jiOXwGuA76uqh8AJjlHrVc34X/eSDoisltEdjcmJrMOMQyjy+SR+AeAA6r6pP/9MK7hLz6SztatKnMdpKw62u5du8gz97wU1PhNS2LjxS7soiWtaXq8vNd7nxl8PXjdqa+JZgOOuP310XBSYmwrTcuctPI8hrosA16iJczy5BOJzOSccuQ+r9k66M1cWr133TTuqeoRYL+IXO6TbgZexCLpGEbPkncCz5eAb4pIH/AG8Cu4Pw2LpGMYPUjeoJnPADsydi06ko70gnGv3d2RAk+TSlT8ZvSmlbw2Xp6J0upz1fG4y6UVl3Z2c2QEjIx75Qk/1h4Z2JI8m5EXm4afkdeIjHtxPlnvp2YE9Iy7BGmZWUa5VrPw8hr8Fkpf7DEU+pU0jOLS9bn6PRCWru2TrrTAc/+TIbtZhrzES3Xs7m8oksBeqMcaQSKJK6fmmR2XzNwLtj8agxmz8Lx0r56N5udHw4qxITC9Bj8bcNZK3EgrTMqsrwnlVSYyZGrWux9XsQ0v3lLCIxqGURCs4RtGAemqqi9azFlxpQKr+mmAnKxx80jsVM4srKSW6hmJGc9e4uPmWQQ055x4xXBlbqZZ3dO4e5CcXz3ZO3K0d2pqGEbbsIZvGAXEGr5hFBBr+IZRQLo/jt/uWXE9YDjrhbkLRSbLQVKrZxafs9Cx8zlfyjQYysL55XLkZDP3DMOYD2v4hlFAuqvqz+uuY+n0hBpdYGebvcBS3qG85ywm71bHtvNdN4lvGAVkBYx77ZV+PSHxe8AAaSxAlv+8xexf6JzFYMtyDcNYDtbwDaOAtFT1va+9B6Ok9wL/BfgGS4ik035Hlm3OrwP0RHfEmJ9Wz28pMfESWqn8HXp38jjb3KOq16rqtcAHgSngUSySjmH0LItV9W8GXlfVvVgkHcPoWRZr1b8duN9vLymSTtun7PaAGt12HwTG6qSLfvHnLaLdVn3vWvuTwF+fuy93JJ0pi6RjGKuBxaj6Pw/8QFWP+t9HfQQdWkXSUdUdqrqjPDS8vNpmFtADH6MYSIvPKmIxDf8OgpoPFknHMHqWXA1fRIaBW4BHouSvAreIyKvAR/1vwzB6gLyRdCaBDeekvcMiI+lIJ4x7PUAnjHu27md1k0TXqa8JafEzq0y577g9NAaTA0NaHIgzDTwa+/c/N2KPTdk1DGM+ur5IxzCKQMlL8spESEsi8kCIJViKtIDymeTkkNaIw21naXk+LdEmNKcoN4lvGAXEGr5hFJDuq/pFHNfuwDWbbW91M3WRe+hDh8OT6h8PL8L0qEuvR1NbErW/Ehn04shAeZx65jUkm8Q3jAJiDd8wCkj3XW8VUNXvyDi+/WWvKPF7nOUPf+iIS2z2hbTJS6IOmj+2ejokVSdc4szacNzZC0Om1VMuPX6f0u3ksJx9QHt9DKOAdN29dhFn7nUCu4+rh8zh9eT5ROG0K2eiA7yEjo17M6P+lOi4kTejU5Kx+qjVNvrmpuXBJL5hFBBr+IZRQGwcv0cpopG0q8T3N0uXb+E4c2adT4rG4auRH5okXaOuAE2Xwdq3gvVu/dPHwu6RAZf3+oE0bfJip+uf2Zj0AzLqlYFJfMMoIN0fzjP/c23BJH4XWYL77IHj7kcyQw9gZiTsHzrq9m/8p2DJqx4ed9kMhDHA+gVhXW95yq31LU+HRlSddNuNPifD8xp9TeIbRgGxhm8YBSSXqi8ivwX8W5wy8xzwK8Bm4AGcZ56ngV9S1ZkO1dMweoqpze57w3NB9173w+CPVqad2q5rw0B+c+2QS5O5M/wAZsaci55SLaj6w/udK5+R1105r0/m0/VbSnwRuQT4D8AOVb0aNyXhduAPga+p6vuAd4Ev5CrRMIwVJ6+qXwEGRaQCDAGHgZuAh/1+i6RjGD1ES1VfVQ+KyH8H9gFngL/HqfbjqpqMUh4ALslTYE9Yo3uhjq3Iu2gjutZkQUm8bnzqwpBByT/teDw6mTKaOJeM84Ew/TQZ1wYYPObyj11R1daEcsrTzNlf8p3IWePerch5D0pRB1Wa7qTJLSFt+ID7bgyGjOLx+XBu2F73mj/30HRmmc0xZ+Jv9ocmKA1voR8MF35mU+R7yxc/cDzc7PJeF+Zi5kpXYa3kk+V5VP31uDh5lwIXA8PArblyxyLpGMZqJI9x76PAm6p6DEBEHgE+DIyKSMVL/S3AwayTVXUnsBNg8KKt54MsXb20mm2WRXRcMgYcS8Bq5Cwy0dZKtVBQsyqz9sX5QJCgWgkF1RPJOU8dE8k5Ox/3vSiJn5NaNL7ed8p9V6Yi6e6voTYUjitHgnztPle5kdfDGtvmgJPalePhBiYSHaDpJXNtNEj0ZCy+MRDKrg0F2dx/quHPDfubF28E4J33u9l89efyPfg8esE+4EYRGRIRwfnSfxF4AviMP8Yi6RhGD9Gy4avqkzgj3g9wQ3klnAT/XeC3ReQ13JDevR2sp2EYbSRvJJ3fA37vnOQ3gBvaXiNj6cRaXpZhK6srEKVNj7kfsfGOjCHlWNVMjHuzpopG5/Sf9F5l1smc/bPU9lldBf8jKqejHocyujuxKp9cQ9KtAahMhQoPH3TeMeWVfWlaeZuzdWs1NLHmUFDr62vd9sxIuAmNvrkedhL1HmBor3swUgtpMuMNfaVkVVDWBc7FZu4ZRgHpugeeXhgq64WFRHEctgWHSDN8w0H2NY6+4r7Hnj2ZnVe96csLmTaGncgvj09FmUfGqQud5Wx6NCwlnTUzzVOaxziYnlOav94uT78/6160eOdiv3dJ2XEdEok+HJmvSzONOdty0cY0rdnvJHljKIxtNgaDdE+0plh7SjSgytlQ4YFjkaU1GaprxFZPd/5iNSKT+IZRQKzhG0YBEdXu6d4icgyYBI53rdDOcwF2PauV8+laIN/1bFPVjS2O6W7DBxCR3aq6o6uFdhC7ntXL+XQt0N7rMVXfMAqINXzDKCAr0fB3rkCZncSuZ/VyPl0LtPF6ut7HNwxj5TFV3zAKSFcbvojcKiJ7ROQ1Ebmrm2UvFxHZKiJPiMiLIvKCiNzp08dE5Hsi8qr/Xr/SdV0MIlIWkR+KyOP+96Ui8qR/Rg+KSF+rPFYLIjIqIg+LyMsi8pKIfKiXn4+I/JZ/154XkftFZKBdz6drDV9EysCfAj8PXAncISJXdqv8NlAHfkdVrwRuBH7d1/8uYJeqXgbs8r97iTuBl6LfvexL8W7gO6p6BXAN7rp68vl03NelqnblA3wI+G70+yvAV7pVfgeu59vALcAeYLNP2wzsWem6LeIatuAaw03A47i1XceBStYzW80fYB3wJt5uFaX35PPBubLbD4zh1tQ8Dvxcu55PN1X95EIScvvpW22IyHbgA8CTwCZVPex3HQE2rVC1lsIfA18GkqUvG1iiL8VVwKXAMeAvfNflHhHfXINdAAABeklEQVQZpkefj6oeBBJfl4eBkyzD1+W5mHFvkYjIGuBvgN9U1VPxPnV/wz0xTCIitwFvq+rTK12XNlEBrgO+rqofwE0Nn6XW99jzWZavy1Z0s+EfBLZGv+f107daEZEqrtF/U1Uf8clHRWSz378ZeHu+81cZHwY+KSJv4QKj3ITrI496N+rQW8/oAHBAnccocF6jrqN3n0/q61JVa8AsX5f+mCU/n242/KeAy7xVsg9nqHisi+UvC+9v8F7gJVX9o2jXYzifg9BDvgdV9SuqukVVt+OexT+o6ufoUV+KqnoE2C8il/ukxDdkTz4fOu3rsssGi48DrwCvA/9ppQ0oi6z7T+HUxGeBZ/zn47h+8S7gVeB/A2MrXdclXNtHgMf99nuB7wOvAX8N9K90/RZxHdcCu/0z+l/A+l5+PsAfAC8DzwN/BfS36/nYzD3DKCBm3DOMAmIN3zAKiDV8wygg1vANo4BYwzeMAmIN3zAKiDV8wygg1vANo4D8f6UOP/woQkVLAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from skimage.color import convert_colorspace\n",
    "\n",
    "xi = np.round((np.squeeze(state)*255.)).astype(np.uint8)\n",
    "x = convert_colorspace(xi, fromspace='rgb', tospace='YUV')[..., 0] # Get only the luminance\n",
    "plt.imshow(np.squeeze(x))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Take Random Actions in the Environment\n",
    "\n",
    "In the next code cell, you will learn how to use the Python API to control the agent and receive feedback from the environment.\n",
    "\n",
    "Once this cell is executed, you will watch the agent's performance, if it selects an action (uniformly) at random with each time step.  A window should pop up that allows you to observe the agent, as it moves through the environment.  \n",
    "\n",
    "Of course, you'll have to change the code so that the agent is able to use its experience to gradually choose better actions when interacting with the environment!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# env_info = env.reset(train_mode=False)[brain_name] # reset the environment\n",
    "# state = env_info.visual_observations[0]            # get the current state\n",
    "# score = 0                                          # initialize the score\n",
    "# while True:\n",
    "#     action = np.random.randint(action_size)        # select an action\n",
    "#     env_info = env.step(action)[brain_name]        # send the action to the environment\n",
    "#     next_state = env_info.visual_observations[0]   # get the next state\n",
    "#     reward = env_info.rewards[0]                   # get the reward\n",
    "#     done = env_info.local_done[0]                  # see if episode has finished\n",
    "#     score += reward                                # update the score\n",
    "#     state = next_state                             # roll over the state to next time step\n",
    "#     if done:                                       # exit loop if episode finished\n",
    "#         break\n",
    "    \n",
    "# print(\"Score: {}\".format(score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When finished, you can close the environment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. It's Your Turn!\n",
    "\n",
    "Now it's your turn to train your own agent to solve the environment!  When training the environment, set `train_mode=True`, so that the line for resetting the environment looks like the following:\n",
    "```python\n",
    "env_info = env.reset(train_mode=True)[brain_name]\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pixel_dqn(n_episodes=2000, max_t=1000, eps_start=1.0, eps_end=0.01, eps_decay=0.995, prefix='dqn'):\n",
    "    \"\"\"Deep Q-Learning.\n",
    "\n",
    "    Params\n",
    "    ======\n",
    "        n_episodes (int): maximum number of training episodes\n",
    "        max_t (int): maximum number of timesteps per episode\n",
    "        eps_start (float): starting value of epsilon, for epsilon-greedy action selection\n",
    "        eps_end (float): minimum value of epsilon\n",
    "        eps_decay (float): multiplicative factor (per episode) for decreasing epsilon\n",
    "    \"\"\"\n",
    "    scores = []  # list containing scores from each episode\n",
    "    scores_window = deque(maxlen=100)  # last 100 scores\n",
    "    eps = eps_start  # initialize epsilon\n",
    "    for i_episode in range(1, n_episodes + 1):\n",
    "        env_info = env.reset(train_mode=True)[brain_name]           # reset the environment\n",
    "        image = env_info.visual_observations[0]                     # get the current image\n",
    "        score = 0                                                   # initialize the score\n",
    "\n",
    "        agent.on_new_episode(image)\n",
    "\n",
    "        for t in range(max_t):\n",
    "            action = agent.act(image, eps)                          # select an action\n",
    "            env_info =  env.step(action)[brain_name]                # send the action to the environment\n",
    "            next_image = env_info.visual_observations[0]            # get the next image\n",
    "            reward = env_info.rewards[0]                            # get the reward\n",
    "            done = env_info.local_done[0]                           # see if episode has finished\n",
    "            \n",
    "            agent.step(state, action, reward, next_image, done)\n",
    "            image = next_image\n",
    "            score += reward\n",
    "            if done:\n",
    "                break\n",
    "        scores_window.append(score)  # save most recent score\n",
    "        scores.append(score)  # save most recent score\n",
    "        eps = max(eps_end, eps_decay * eps)  # decrease epsilon\n",
    "\n",
    "        print('\\rEpisode {}\\tAverage Score: {:.2f}, Recent Score: {:.2f}'.format(i_episode, np.mean(scores_window), score), end=\"\")\n",
    "        if i_episode % 100 == 0:\n",
    "            print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))\n",
    "        if np.mean(scores_window) >= 13.0:\n",
    "            print('\\nEnvironment solved in {:d} episodes!\\tAverage Score: {:.2f},  Recent Score: {:.2f}'.format(\n",
    "                i_episode - 100,\n",
    "                np.mean(scores_window), score))\n",
    "            torch.save(agent.qnetwork_local.state_dict(), '{}_checkpoint.pth'.format(prefix))\n",
    "            break\n",
    "\n",
    "    return scores\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 100\tAverage Score: -0.03, Recent Score: 0.00\n",
      "Episode 200\tAverage Score: 0.01, Recent Score: 0.0000\n",
      "Episode 300\tAverage Score: -0.01, Recent Score: 1.000\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from agents.dqn_agent import VisualAgent\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "device = torch.device(\"cuda:1\" if torch.cuda.is_available() else \"cpu\")\n",
    "method='doubledqn'\n",
    "alpha = 0.6\n",
    "beta0 = 0.4\n",
    "\n",
    "agent = VisualAgent(state_size=(state_size[-1], state_size[1], state_size[2]),\n",
    "                    action_size=action_size, sample_method='uniform', method=method, \n",
    "                    seed=0, device = device, alpha = alpha, beta = beta0)\n",
    "scores = pixel_dqn(eps_start=1., prefix='Viz_uniform_ddqn_'+ method)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot the scores\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(len(scores)), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}