Fleet_Scheduling_MADDPG.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Fleet Scheduling MADDPG.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "xNcUwydFGgAg",
        "PxNyfvi_5dQR",
        "IVjZpbiGGlQm",
        "lm_mQoGdGsVO",
        "O1HBs0Mc67Cf",
        "QUZ_0UD164Ga",
        "Rx0JF_jyVJSH",
        "JY9ODifXBlfY",
        "j5thjY-EJuji",
        "r_qCgpK_Jske",
        "2XXTI-PCanVZ"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "\"We certify that the code and data in this assignment were generated independently, using only the tools and resources defined in the course and that we did not receive any external help, coaching or contributions during the production of this work"
      ],
      "metadata": {
        "id": "xYX1sDRal25i"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Libraries"
      ],
      "metadata": {
        "id": "xNcUwydFGgAg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import gym\n",
        "from gym import spaces\n",
        "import copy\n",
        "from copy import copy\n",
        "import matplotlib.pyplot as plt\n",
        "import gym\n",
        "from gym import spaces\n",
        "import copy\n",
        "from copy import copy\n",
        "import matplotlib.pyplot as plt\n",
        "from google.colab import widgets\n",
        "import time"
      ],
      "metadata": {
        "id": "QW3De0D7NW-B"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import torch\n",
        "import torch as T\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim"
      ],
      "metadata": {
        "id": "VjIiapVJGkYi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#req files"
      ],
      "metadata": {
        "id": "PxNyfvi_5dQR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/openai/multiagent-particle-envs.git"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xZQrgT6k5e6q",
        "outputId": "8d331bca-93d2-4689-8a36-442228d75984"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'multiagent-particle-envs'...\n",
            "remote: Enumerating objects: 242, done.\u001b[K\n",
            "remote: Counting objects: 100% (5/5), done.\u001b[K\n",
            "remote: Compressing objects: 100% (5/5), done.\u001b[K\n",
            "remote: Total 242 (delta 0), reused 3 (delta 0), pack-reused 237\u001b[K\n",
            "Receiving objects: 100% (242/242), 107.24 KiB | 17.87 MiB/s, done.\n",
            "Resolving deltas: 100% (127/127), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%pwd"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "bqGNVPTh8pnp",
        "outputId": "68391b2c-ef5e-4046-ea4e-6c4bc46bd510"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/content'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/multiagent-particle-envs/"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6qyxkU0u9HOh",
        "outputId": "2ad0877a-e0b1-49da-b661-98dbac316c09"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/multiagent-particle-envs\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -e ."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wzEMOPVC7rAj",
        "outputId": "ca74b411-1f35-4460-9be9-ddf4240c73b6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Obtaining file:///content/multiagent-particle-envs\n",
            "Requirement already satisfied: gym in /usr/local/lib/python3.7/dist-packages (from multiagent==0.0.1) (0.17.3)\n",
            "Collecting numpy-stl\n",
            "  Downloading numpy-stl-2.16.3.tar.gz (772 kB)\n",
            "\u001b[K     |████████████████████████████████| 772 kB 16.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym->multiagent==0.0.1) (1.3.0)\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from gym->multiagent==0.0.1) (1.5.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from gym->multiagent==0.0.1) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.10.4 in /usr/local/lib/python3.7/dist-packages (from gym->multiagent==0.0.1) (1.21.6)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym->multiagent==0.0.1) (0.16.0)\n",
            "Requirement already satisfied: python-utils>=1.6.2 in /usr/local/lib/python3.7/dist-packages (from numpy-stl->multiagent==0.0.1) (3.1.0)\n",
            "Building wheels for collected packages: numpy-stl\n",
            "  Building wheel for numpy-stl (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for numpy-stl: filename=numpy_stl-2.16.3-cp37-cp37m-linux_x86_64.whl size=137060 sha256=70d6aa51e0ea74a6d905eab313103a7638b2b7810fa76adeb3796342c2983092\n",
            "  Stored in directory: /root/.cache/pip/wheels/06/f4/db/7fac39962a6ba79b7e740892042332083924bff552d4bef41e\n",
            "Successfully built numpy-stl\n",
            "Installing collected packages: numpy-stl, multiagent\n",
            "  Running setup.py develop for multiagent\n",
            "Successfully installed multiagent-0.0.1 numpy-stl-2.16.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install torch==1.4.0\n",
        "!pip uninstall gym --y\n",
        "!pip install gym==0.10.5"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nR5pMtMZ-hlD",
        "outputId": "b87d8755-f5cb-4527-a394-47b23cde5969"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting torch==1.4.0\n",
            "  Downloading torch-1.4.0-cp37-cp37m-manylinux1_x86_64.whl (753.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 753.4 MB 6.5 kB/s \n",
            "\u001b[?25hInstalling collected packages: torch\n",
            "  Attempting uninstall: torch\n",
            "    Found existing installation: torch 1.11.0+cu113\n",
            "    Uninstalling torch-1.11.0+cu113:\n",
            "      Successfully uninstalled torch-1.11.0+cu113\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "torchvision 0.12.0+cu113 requires torch==1.11.0, but you have torch 1.4.0 which is incompatible.\n",
            "torchtext 0.12.0 requires torch==1.11.0, but you have torch 1.4.0 which is incompatible.\n",
            "torchaudio 0.11.0+cu113 requires torch==1.11.0, but you have torch 1.4.0 which is incompatible.\u001b[0m\n",
            "Successfully installed torch-1.4.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "torch"
                ]
              }
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found existing installation: gym 0.17.3\n",
            "Uninstalling gym-0.17.3:\n",
            "  Successfully uninstalled gym-0.17.3\n",
            "Collecting gym==0.10.5\n",
            "  Downloading gym-0.10.5.tar.gz (1.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.5 MB 27.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.10.4 in /usr/local/lib/python3.7/dist-packages (from gym==0.10.5) (1.21.6)\n",
            "Requirement already satisfied: requests>=2.0 in /usr/local/lib/python3.7/dist-packages (from gym==0.10.5) (2.23.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from gym==0.10.5) (1.15.0)\n",
            "Requirement already satisfied: pyglet>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym==0.10.5) (1.5.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from pyglet>=1.2.0->gym==0.10.5) (0.16.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0->gym==0.10.5) (2021.10.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0->gym==0.10.5) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0->gym==0.10.5) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0->gym==0.10.5) (1.24.3)\n",
            "Building wheels for collected packages: gym\n",
            "  Building wheel for gym (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gym: filename=gym-0.10.5-py3-none-any.whl size=1581307 sha256=7cdea559b5d9cb2c6185b46b32b5beaf5ba497b85ed6637784fa93c712a97c0d\n",
            "  Stored in directory: /root/.cache/pip/wheels/7a/2c/df/a05b548a40fae16ca400ecbeda0067e1a296499c1fbd7e0c9a\n",
            "Successfully built gym\n",
            "Installing collected packages: gym\n",
            "Successfully installed gym-0.10.5\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "gym"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "torch.__version__"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-5WcSWBlNJoz",
        "outputId": "1c993671-d59c-48c1-dc13-c8812a8ab5d9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'1.11.0+cu113'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Basic Multiagent Grid world"
      ],
      "metadata": {
        "id": "IVjZpbiGGlQm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class MAGridWorld(gym.Env):\n",
        "    \n",
        "    metadata = { 'render.modes' : ['human', 'rgb_array'] }\n",
        "        \n",
        "    def __init__(self, size, n_agents):\n",
        "        self.size = size\n",
        "        self._grid_size = (self.size, self.size)\n",
        "        self.world = np.zeros((self._grid_size))\n",
        "        self._timestep = None\n",
        "        self._max_timestep = int(self.size*4)\n",
        "        self.n_agents = n_agents\n",
        "        self.dist = int((self.size-1)/(n_agents-1))\n",
        "        self._step_cost = -0.1\n",
        "        self.observation_space = spaces.Box(low=0, high=self.size, shape=(self.size, self.size))\n",
        "        self.action_space = spaces.Discrete(5)\n",
        "\n",
        "    def reset(self):\n",
        "        self.agent_pos = []\n",
        "        self.goal_pos = []\n",
        "        self._timestep = 0\n",
        "        self.done = [False for i in range(self.n_agents)]\n",
        "        for i in range(self.n_agents):\n",
        "            self.agent_pos.append([self.size-1, i*self.dist])\n",
        "            self.goal_pos.append([0, i*self.dist])\n",
        "        self.goal_pos.reverse()\n",
        "        observation = self.agent_pos\n",
        "        return observation\n",
        "\n",
        "\n",
        "    def _compute_distance(self, x, y):\n",
        "        dist = np.abs(x[0]-y[0]) + np.abs(x[1]-y[1])\n",
        "        return dist\n",
        "\n",
        "\n",
        "    def _is_inside_grid(self, pos):\n",
        "        return (0 <= pos[0] < self._grid_size[0]) and (0 <= pos[1] < self._grid_size[1])\n",
        "\n",
        "\n",
        "    def _is_collision(self, new_pos, agent):\n",
        "        for i in range(self.n_agents):\n",
        "            if i == agent:\n",
        "                continue\n",
        "            elif new_pos == self.agent_pos[i]:\n",
        "                return True\n",
        "            elif new_pos == self.goal_pos[i]:\n",
        "                return True\n",
        "\n",
        "    def step(self, actions):\n",
        "\n",
        "        self._timestep += 1\n",
        "        old_pos = self.agent_pos.copy()\n",
        "        rewards = [0 for _ in range(self.n_agents)]\n",
        "\n",
        "        for i in range(self.n_agents):\n",
        "            self.update_agent_pos(i, actions[i])\n",
        "            old_dist = self._compute_distance(old_pos[i], self.goal_pos[i])\n",
        "            new_dist = self._compute_distance(self.agent_pos[i], self.goal_pos[i])\n",
        "            \n",
        "            if new_dist < old_dist:\n",
        "                rewards[i] = 1 #-1 worked\n",
        "            elif new_dist > old_dist:\n",
        "                rewards[i] = -3 #-2 worked\n",
        "\n",
        "            elif (new_dist == old_dist) and not self.done[i]:\n",
        "                rewards[i] = -5 #-2 worked\n",
        "            \n",
        "            if (self.agent_pos[i] == self.goal_pos[i]):\n",
        "                self.done[i] = True\n",
        "                rewards[i] = 100\n",
        "            else:\n",
        "                self.done[i] = False\n",
        "\n",
        "        if self._timestep >= self._max_timestep:\n",
        "            self.done = [True for i in range(self.n_agents)]\n",
        "            \n",
        "        observation = self.agent_pos\n",
        "        info = {}\n",
        "        return observation, rewards, self.done, info #all(done)\n",
        "\n",
        "\n",
        "\n",
        "    def update_agent_pos(self, agent, action):\n",
        "        next_pos = copy(self.agent_pos[agent])\n",
        "        if action == 0: # Down\n",
        "            next_pos[0] += 1\n",
        "        elif action == 1: # Up\n",
        "            next_pos[0] -= 1\n",
        "        elif action == 2: # Right\n",
        "            next_pos[1] += 1\n",
        "        elif action == 3: # Left\n",
        "            next_pos[1] -= 1\n",
        "        elif action == 4: # No move\n",
        "            pass\n",
        "        else:\n",
        "            raise Exception('Action is not valid')\n",
        "            \n",
        "        if self._is_inside_grid(next_pos) and not self._is_collision(next_pos, agent):\n",
        "            self.agent_pos[agent] = next_pos\n",
        "        else:\n",
        "            pass\n",
        "        \n",
        "\n",
        "    def render(self):\n",
        "        state = np.copy(self.world)\n",
        "        for i in range(self.n_agents):\n",
        "            state[tuple(self.agent_pos[i])] = 1 - (0.1 * i)\n",
        "            state[tuple(self.goal_pos[i])] = 1 - (0.1 * i)\n",
        "        return plt.imshow(state)"
      ],
      "metadata": {
        "id": "dSVdjU5xNbrq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class RandomAgent:\n",
        "    \n",
        "    def __init__(self, env):\n",
        "        self.env = env\n",
        "        self.observation_space = env.observation_space\n",
        "        self.action_space = env.action_space\n",
        "\n",
        "    def step(self, observation):\n",
        "        return np.random.choice(self.action_space.n)"
      ],
      "metadata": {
        "id": "WNhccf-bNvbc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#1. Q-Learning on Multiagent"
      ],
      "metadata": {
        "id": "lm_mQoGdGsVO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Q-learning\n",
        "\n",
        "def q_learning(evaluation = False):\n",
        "  env = MAGridWorld(4, 2)\n",
        "  agent = RandomAgent(env)#creating a random agent to explore the given environments \n",
        "  obs = env.reset()#resets the environment to its initial configuration\n",
        "  \n",
        "  #Intialize parameters\n",
        "  learning_rate = 0.15 #alpha\n",
        "  discount_factor = 0.99 #how much weightage to put on future rewards\n",
        "  det_epsilon = 0.99 # For all states in deterministic environment p(s', r/s, a) = {0, 1}: Either action taken or No action taken\n",
        "\n",
        "\n",
        "  #Intial state\n",
        "  current_state = [0, 0] #s1\n",
        "  action_val = [0,1,2,3]\n",
        "\n",
        "  #Q table representing 16 rows: one for each state (i.e., 0,1,2,...15) -> (i.e., s1, s2, s3,....s16) and 4 columns: one for each action (i.e., 0,1,2,3) -> (down,up,right,left)\n",
        "  # (0-15, 0-3) remember the dimension is one less\n",
        "  q_tables = [np.zeros((16,4)), np.zeros((16,4))] \n",
        "\n",
        "\n",
        "  #mapping next_state co-ordinates to q_table co-ordinates\n",
        "  states = {(0,0): 0, (0,1): 1, (0,2): 2, (0,3): 3,\n",
        "                (1,0): 4, (1,1): 5, (1,2): 6, (1,3): 7,\n",
        "                (2,0): 8, (2,1): 9, (2,2): 10, (2,3): 11,\n",
        "                (3,0): 12, (3,1): 13, (3,2): 14, (3,3): 15} #16 states\n",
        "\n",
        "  #Empty lists to store values\n",
        " \n",
        "  epsilon_values = []\n",
        "  \n",
        "  total_episodes = 10\n",
        "  epsilon = 1 #multiply by 0.995 for each episode(#after 20 iterations# or terminal state reached)\n",
        "  decay_factor = (0.01/1)**(1/total_episodes)\n",
        "  rewards_val = []\n",
        "  if evaluation == False:\n",
        "    for episode in range(1, total_episodes+1):\n",
        "      \n",
        "      obs = env.reset()\n",
        "      current_states = [0, 0]\n",
        "      total_rewards = [0, 0]\n",
        "      timestep = 0\n",
        "\n",
        "      while timestep < 10: #(i.e., considering untill the terminal is reached or 20 timesteps completed)\n",
        "      \n",
        "        actions = []\n",
        "        next_states = []\n",
        "        for agent in range(2):\n",
        "\n",
        "          #e - greedy algorithm\n",
        "          rand_num = np.random.random()\n",
        "          if epsilon > rand_num:\n",
        "            action = np.random.choice(action_val)\n",
        "          else:\n",
        "            action = np.argmax(q_tables[agent][current_states[agent]]) #action in current state s with max_q value\n",
        "          \n",
        "          actions.append(action)\n",
        "\n",
        "        #Taking the action\n",
        "        next_state_poss, rewards, done, _ = env.step(actions)\n",
        "\n",
        "        # print(\"Rewards: \", rewards)\n",
        "        for pos in next_state_poss:\n",
        "          next_states.append(states[tuple(pos)])\n",
        "\n",
        "\n",
        "        #Choosing action with max Q value\n",
        "        for agent in range(2):\n",
        "\n",
        "          max_q_action = np.argmax(q_tables[agent][next_states[agent]])\n",
        "\n",
        "          #Update function\n",
        "          # print(actions[agent])\n",
        "          q_tables[agent][current_states[agent]][actions[agent]] = q_tables[agent][current_states[agent]][actions[agent]] + learning_rate*(rewards[agent] + discount_factor*q_tables[agent][next_states[agent]][max_q_action] - q_tables[agent][current_states[agent]][actions[agent]])\n",
        "\n",
        "          total_rewards[agent] += rewards[agent]\n",
        "\n",
        "        current_states[0] = next_states[0] #next_state is assigned to current_state\n",
        "        current_states[1] = next_states[1]\n",
        "        \n",
        "        if done[0] and done[1]:\n",
        "          done[0] = False\n",
        "          done[1] = False\n",
        "          break     \n",
        "\n",
        "      rewards_val.append(total_rewards)\n",
        "      epsilon_values.append(epsilon) #Append epsilon values in every episode\n",
        "      \n",
        "\n",
        "      if epsilon > 0.01: #keeping epsilon in [0.01 - 1] range as if it falls below 0.01 it will exploit more: choosing best actions. We want our agent to explore a bit: choosing random actions\n",
        "          epsilon = epsilon*decay_factor\n",
        "      else:\n",
        "          epsilon = 0.01\n",
        "\n",
        "\n",
        "      if (episode % 1) == 0:\n",
        "        print(\"Episode: {}, epsilon: {}, rewards: {}\".format(episode, epsilon, rewards))\n",
        "        \n",
        "\n",
        "    #Plotting the results\n",
        "    #x, y co-ordinates\n",
        "    x = [episode for episode in range(1, total_episodes+1)]\n",
        "    ye = epsilon_values\n",
        "    yr1 = [rewards_val[episode][0] for episode in range(total_episodes)]\n",
        "    yr2 = [rewards_val[episode][1] for episode in range(total_episodes)]\n",
        "    # yr2 = total_rewards[1]\n",
        "\n",
        "    #Plots showing episodes vs epsilon, episodes vs rewards\n",
        "    fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(10,4))\n",
        "    #episodes vs epsilon\n",
        "    ax1.plot(x, ye)\n",
        "    ax1.set_title(\"Epsilon decay\")\n",
        "\n",
        "    #episodes vs rewards\n",
        "    ax2.plot(x,yr1)\n",
        "    ax2.set_title(\"Agent1: Rewards per episode\")\n",
        "\n",
        "    #episodes vs rewards\n",
        "    ax3.plot(x,yr2)\n",
        "    ax3.set_title(\"Agent2: Rewards per episode\")\n",
        "\n",
        "  else:\n",
        "    eval_rewards_val = []\n",
        "\n",
        "    for i in range(10):\n",
        "      obs = env.reset()\n",
        "      current_states = [0, 0]\n",
        "      total_rewards = [0, 0]\n",
        "      timestep = 0\n",
        "      agent1_states = [13]\n",
        "      agent2_states = [16]\n",
        "\n",
        "      while timestep < 10: #(i.e., considering untill the terminal is reached or 20 timesteps completed)\n",
        "      \n",
        "        actions = []\n",
        "        next_states = []\n",
        "        \n",
        "        for agent in range(2):\n",
        "          action = np.argmax(q_tables[agent][current_states[agent]]) #action in current state s with max_q value\n",
        "          actions.append(action)\n",
        "\n",
        "        #Taking the action\n",
        "        next_state_poss, rewards, done, _ = env.step(actions)\n",
        "\n",
        "        # print(\"Rewards: \", rewards)\n",
        "        for pos in next_state_poss:\n",
        "          next_states.append(states[tuple(pos)])\n",
        "\n",
        "        #Choosing action with max Q value\n",
        "        for agent in range(2):\n",
        "\n",
        "          max_q_action = np.argmax(q_tables[agent][next_states[agent]])\n",
        "\n",
        "          #Update function\n",
        "          q_tables[agent][current_states[agent]][actions[agent]] = q_tables[agent][current_states[agent]][actions[agent]] + learning_rate*(rewards[agent] + discount_factor*q_tables[agent][next_states[agent]][max_q_action] - q_tables[agent][current_states[agent]][actions[agent]])\n",
        "\n",
        "          total_rewards[agent] += rewards[agent]\n",
        "        \n",
        "        agent1_states.append(next_states[0] + 1)\n",
        "        agent2_states.append(next_states[1] + 1)\n",
        "        current_states[0] = next_states[0] #next_state is assigned to current_state\n",
        "        current_states[1] = next_states[1]\n",
        "        \n",
        "        if done[0] and done[1]:\n",
        "          done[0] = False\n",
        "          done[1] = False\n",
        "          break     \n",
        "\n",
        "      eval_rewards_val.append(total_rewards)\n",
        "      env.render()\n",
        "      \n",
        "      # print(\"Evaluation rewards: \", eval_rewards_val)\n",
        "      print(\"Agent 1 route: \", agent1_states)\n",
        "      print(\"Agent 2 route: \", agent2_states) \n"
      ],
      "metadata": {
        "id": "CjXhhqZ4N0ok"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "q_learning()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 455
        },
        "id": "5-E-phT0OcpB",
        "outputId": "ec910036-fc20-4420-f91d-5be788892f14"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Episode: 1, epsilon: 0.6309573444801932, rewards: [1, -3]\n",
            "Episode: 2, epsilon: 0.39810717055349726, rewards: [1, 1]\n",
            "Episode: 3, epsilon: 0.251188643150958, rewards: [100, 1]\n",
            "Episode: 4, epsilon: 0.15848931924611134, rewards: [100, 100]\n",
            "Episode: 5, epsilon: 0.09999999999999999, rewards: [100, 100]\n",
            "Episode: 6, epsilon: 0.06309573444801932, rewards: [100, -5]\n",
            "Episode: 7, epsilon: 0.03981071705534972, rewards: [100, 100]\n",
            "Episode: 8, epsilon: 0.025118864315095798, rewards: [100, 100]\n",
            "Episode: 9, epsilon: 0.015848931924611134, rewards: [100, 100]\n",
            "Episode: 10, epsilon: 0.01, rewards: [100, 100]\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 720x288 with 3 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlkAAAEICAYAAABswuGIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXxcZ3X4/8/RMtqt3askO16S2GSxHWdfCFloFkjya1mSUjBtSgoECg2/shTaAoU2lJatULaGJoGEpZQmBhJCyAJZnMVxEieWk9hxvGjxomUkax2NdL5/3OfKY0W2RtLM3DvSeb9eemnmztXVM9LcmXOf5zznEVXFGGOMMcakVk7QDTDGGGOMmYksyDLGGGOMSQMLsowxxhhj0sCCLGOMMcaYNLAgyxhjjDEmDSzIMsYYY4xJAwuyAiAi7xKR3ybcVxFZnoHfe6uIfCHdv8eYsBGRC0WkKeh2pJKI3Csi61N8zM+KyI9SeUwTPnY+JH3MaZ8PFmRNQER2iUi/iPQkfH1zOsdU1TtU9c2paqPJXiLysIh0ikhBBn/nEUG9iERE5Ofuta4icuEkj/ewiAy4c6NNRH4hIgtS3nBzBFW9XFVvC7odqRSS8+EsEblfRDpE5KCI/M9kXs92PgQjrOeDBVnJeauqliZ8fSjoBpnsJyJLgPMBBa4KtDHwKPBnwL4p/vyHVLUUWA6UAv+WqoZNlojkBfW7j0VEcoNuQ5iF6HyoBL4HLAEWA4eA/57kMex8mMBsOR8syJoGEXmviDwmIt8UkS4ReUlELh7z+E4ROSQir4nIuxK2P3qUY5aLyO3uCmq3iHxGRHISf05E/s1d7b0mIpcfo31rRGSz+/0/BQrHPP4WEXlORKIi8riInJLwWL27AjsoIu1+752ILBORB922NhG5Q0Qq3GN/KyL/O+Z3fENEvj7pP+7s8B7gCeBW4IhubhGpFpFfiki3iDwtIl9IfM2IyIkJV9svi8g7Eh67VUS+JSK/dv/7J0VkmXvsD263592V9jtVNaaqX1PVR4Hh6TwhVY0CdwGrJ2qriBznXnv+6/v7InIg4ed+KCIfdbf/XES2ueezU0T+KmG/C0WkSUQ+ISL7gP8WkSL3d+gUkUbg9DF/30+ISLM73suJ5+2Y/W4Vke+49h8Skd+LyOKJnlvCz35bRO4RkV7gTeMcv1xEbhGRVteeL/gfPkm8vzwsIn/pbi93bety5+VPE/Y7x72Gutz3cxIeO8793CERuR+oGdO+s9x7Q1REnpdJ9nJOUljOh3tV9X9UtVtV+4BvAudO5QnZ+WDnA6pqX8f4AnYBlxzlsfcCceBvgHzgnUAXUAWUAN3ACW7fBcAbEn7u0YTjKLDc3b4duBsow7uSegW4PuHnhoD3AbnAB4AWQMZpWwTYndC2t7mf/YJ7fA1wADjTHWu9e64F7v7zwFfd8ygEznM/txy41O1XC/wB+FrCc+wFKtz9PPc7Tgv6/xjGL2AH8EHgNPe/mZfw2E/cVzGwCtjrv2bc/2Qv8Ofub7wGaANWucdvBdqBM9zjdwA/Ge/1Nk6bmoALx2z7U2DLMZ7Hw8BfutvVwO+Au5Ns6x7/9QG8DOwEViY8tsbdvhJYBgjwRqAPWOseuxDvPPySe10WATcDj+Cdi/XAi0CT2/8E16aF7v4SYNlRntuteD0ZF7hjf32S/4cuvA/oHKBwnOP/H/Bdd6y5wFPAX030/jLO3/3HwKf938Ph87UK6ATe7dp4nbtf7R7fCHzFPbcL3HP9kXtsEd7r6Ap33Evd/drZcj64xz8KPGHng50PTOF8CPyDJuxfeIFHDxBN+Hpfwj/9iCDHvSje7V4kUeBPgKIxx3wv4wRZeMFNzH9Rusf+Cng44ed2JDxW7H52/jjtvmCctj3O4SDr28A/jfmZl/FO2LOBg0BeEn+fa4BnE+7fm/D3eQvQGPT/MIxfwHl4HyQ17v5LwN+427nusRMS9v8Ch9/M3gk8MuZ43wX+0d2+FfivhMeuAF4a+3o7SrteF2Ql8VwexnuT73LHfg5oSLKtPwRuAua719+/Au8HjnPnT85RfuddwEfc7QvdeVOY8PhO4LKE+zdw+ENlOV7wfwmQP8Fzu5UjP5BL8Xr76pP8P9x+jGPPAwZJeH/Ae9N/yN1+7zjn8FPAuxP+7v6Hyu14Q1x1Y37Hu4Gnxmzb6I7dgPehVZLw2J0c/lD5BPDDMT97H7B+Fp0PpwAdwPl2Ptj5wBTOBxsuTM41qlqR8PX9hMea1f21nd14VwS9eC+69wOtrqv6xAl+Tw1ehL57zPEWJdwfzZlRrysbvBf6WAuP0jbfYuBjrtszKiJRvBNlofu+W1XjYw8qIvNE5CeuK7cb+BFHdqnehpfbg/v+w6M+29ltPfBbVW1z9+/k8BBJLd5V1t6E/RNvLwbOHPO/exfeG7MvMbeqj/FfI6n016pajvehVAnUJdnW3+N9KFyA1yv6MF6g/0a8N+wRABG5XESecMMQUbwPysTX3UFVHUi4v5Aj/2ajr31V3YHXO/FZ4IB7PS88xnMbPY6q9uB96C5M4rkd8bPjWIx3vrcm/Px38a7gfeO+v4xzrI/j9Wo8JSJbReQv3PaFHHne+8dY5B7rdO9ViY8ltu/tY57feXg91qkWuvNBvGT4e/GCl0cm82Sw88HOByeUCXFZZpGISMI/vgHYAKCq9wH3iUgR3pXX9/ESO4+mDe+KbTHQmHC85im0q/UobXvV3d4LfFFVvzj2B0XkbKBBRPLGCbT+Ge/q7GRV7RCRa/ByFnx3Ad8WkZPwerI+PoW2z2ju9fAOINflTIDXPV0hIqfideXH8d6YX3GP1yccYi/we1W9NENNTpqqviBemZBvichaJm7r74Ev4/Wg/R4vAf87wIC7j3gzzf4XL2fnblUdEpG78N5ER3/1mOO24v3Ntrr7DWPaeSdwp4jMwXsj/xLeVe54Rv/2IlKKN+TQksRzG69difbiXbnXjHdB4xz1/WXM89mHl0aAiJwH/E68fKMWvPeTRA3Ab/D+RpUiUpLwwdKQ0Oa9eFfu7zvGc5i2MJ4PLs/od3i9/VO+ULTz4XVm3flgPVnTNxf4axHJF5G3AyuBe1yPz9UiUoL3wukBRo51IFUdBn4GfFFEytyJfhNeb9FkbcR7Y/Lb9sd4OQm+7wPvF5EzxVMiIleKSBleF2wrcLPbXigifuJnmXsuXSKyCPjbMc9hAPg53pXoU6q6Zwptn+muwetiX4WXELsa73XzCPAe9zr4BfBZESl2PaDvSfj5XwHHi8i73f82X0ROF5GVSf7+/cDSxA0iUiAi/sSIiPufy+t/NCm34XX9XzVRW1V1O9CP1+v5e1Xtdu37E9yHCl5+YQHeEHZcvMkeE5VA+RnwKRGpFJE64MMJz/UEEbnIfVgNuN9/rHPzChE5T0QiwD/h5efsnei5TURVW4HfAv8uInNEJEe8iSVvTNht3PeXsccSkbe75wlejom653SPa+OfikieiLwT73X3K1XdDWwCPideGY/zgLcmHPZHwFtF5I9EJNe9Ji5M+D2pEqrzwb2vPQh8U1W/M90nh50Ps/p8sCArOb+UI+tk/V/CY08CK/B6ob4IvE1V2/H+tjfhRc4deN29H0jid30YL3l8J95VzJ3ADybbYFWNAX+MN9bcgTd0+YuExzfhRfrfxHsR7nD7+sHeW/HG6vfgXVW90/3o54C1ePkGv048ZoLbgJOxocKjWQ/8t6ruUdV9/hfe/+Jd4k25/hBQjjfM8UO8RM5BAFU9hPemei3e62sfh5Nck/FZ4Dbxurz92T8v4725LsLLM+jHXfGJVzx363gHGo977X0d+Psk2/p7oN29Ufv3Bdic8Hz/Gu+DohMv8fh1V69jfA6vq/81vDfuxNdiAV4icJtrz1zgU8c41p3AP+KdR6fhhsNT8H8AL1iI4PVcd+JdoCQOPxzt/WWs04EnRaQH72/zEVXd6fZ9C/AxvCTdjwNvSRiW+1O8yS8d7jne7h/Q/T+uBv4O7wN9L95FVao/N8J2PvwlXtD12cT3fX9nOx/sfGAS54McObxpJkNE3ouXaHde0G0JExFpwEtcne+uxMw0iciX8P6e6yfc2aSMiNyKlyD8mQB+93ux95dx2fkQDDsfJs96skxKiVfj5Sa8GSgWYE2RePVmThHPGcD1eNObjZl17Hww2coS303KiJd/th+vW/qygJuT7crwhkQW4v1N/x2vfpoxs5GdDyYr2XChMcYYY0wa2HChMcYYY0waBDZcWFNTo0uWLAnq1xtzhGeeeaZNVWuDbIOdEyZM7Jww5khTOScCC7KWLFnCpk2bgvr1xhxBRMZWAc44OydMmNg5YcyRpnJO2HChMcYYY0waWJBljDHGGJMGFmQZkyQRqRCRn4vISyKyTUTOFpEqEblfRLa775VuXxGRb4jIDhHZIt66ZcYYY2YRC7KMSd7Xgd+o6onAqcA24JPAA6q6AnjA3Qe4HG/5hxXADcC3M99cY4wxQbIgy5gkiEg5cAFwC3jrkalqFG8tq9vcbrfhLXaL2367ep4AKkRkAcYYY2aNCYMsEfmBiBwQkReP8rgNi5jZ4Di8RUH/W0SeFZH/chXu57nV48FbEHWeu70IbwFRX5PbdgQRuUFENonIpoMHD6ax+cakh4jkunPiV+7+cSLypPtM+KmIRNz2And/h3t8SZDtNiYTkunJupVjL5FiwyJmNsgD1gLfVtU1QC+HhwYBUG/5hEktoaCq31PVdaq6rrY20JJExkzVR/CGzn1fAr6qqsuBTrx1BnHfO932r7r9jJnRJgyyVPUPQMcxdknpsEhTZx9fvu8lWqL9Uz2EMenQhLf6/JPu/s/xgq79/uvdfT/gHm8G6hN+vs5ty0qqys+e3ku0LxZ0U0yIiEgdcCXwX+6+ABfhnR/w+iF0f2j958DFbn8zDfe+0MqBQwNBN8McRSpyspIaFoHkhkYODcT51kOv8uRr7SlomjGpoar7gL0icoLbdDHQCGwA1rtt6zm8aO0G4D1uOP0soCthWDHr7G7v4+P/u4XP/6ox6KaYcPka8HFgxN2vBqKqGnf3Ez8PRj8r3ONdbv8j2BB68roHhvjAHZu588k9QTfFHEVGE9+TGRo5fl4ZpQV5bN4dzWTTjEnGh4E7RGQLsBr4Z+Bm4FIR2Q5c4u4D3APsBHYA3wc+mPnmpk5bzyAAv9jczIvNXQG3xoSBiLwFOKCqz6TyuDaEnrzmTm/Exz8/TfikYlmdlA6L5OYIq+sr2Lync9oNMyaVVPU5YN04D108zr4K3Jj2RmVIW483TJibI3zh1438+H1nYSM9s965wFUicgVQCMzBK3NSISJ5rrcq8fPA/6xoEpE8oBywIYtp8NNq2ntsGD+sUtGTlfJhkbUNFWxr7aZ3MD7xzsaYtOvo9d7E/+qCpTyxs4MHth2Y4CfMTKeqn1LVOlVdAlwLPKiq7wIeAt7mdhs7hO4Prb/N7T+piSLmSM1+kNVrQVZYJVPC4cfARuAEEWkSketF5P0i8n63S8qHRdYsrmRE4fkmGzI0Jgza3XDEhy5aztLaEv753m0MDY9M8FNmlvoEcJOI7MDLubrFbb8FqHbbb2LM7Fwzef5wYbsNF4bWhMOFqnrdBI+nfFhkbX0lAM/uiXLOsppUHtoYMwXtvTHKCvIojuTxqctX8r7bN/Hjp/bwnrOXBN00EwKq+jDwsLu9EzhjnH0GgLdntGEznN+T1WE9WaEVyorv5cX5LJ9byubdlpdlTBh09MaoLo0AcMnKuZy1tIqv/W473QNDAbfMmNnLD7Ki/UPErWc5lEIZZIGXl7V5Tyc2ZG9M8Np7B6kq8YIsEeEzV66iozfGfz70asAtM2b2aon2kyOgCp19dsETRiEOsirp7BvitbbeoJtizKzX3hOjurRg9P5Ji8r54zWL+MFjr7G3oy/AlhkzO8XiIxw4NMiy2lLAhgzDKrxB1mIvL2vzHkt+NyZo7b0xql1Plu///6MTEODL970cTKOMmcX2dQ2gCifXlQOW/B5WoQ2ylteWUlaYZ/WyjAnYyIjS2RsbHS70Lawo4n3nL2XD8y08t9cuhozJpKao14N8al0FYGUcwiq0QVZOjrCmodKS340JWPfAEPERPWK40Pf+C5dRUxrhi79utPxJYzLIL99gPVnhFtogC7zk95f3H+KQzWAyJjD+FfLY4UKA0oI8/ubS43l6Vyf3bd2X6aYZM2u1RL1FoVctmIOI5WSFVciDrEpU4fm9tlaaMUHxl+zwSziM9c519ayYW8rN975ELG7TyI3JhOZoH3PLCijMz6WqOEKbBVmhFOoga3VDBSLwjA0ZGhOYjl5vGGJsTpYvLzeHv7tyJbva+/jRE7sz2TRjZq2W6AALK4oA79zssPULQynUQdacwnyOn1tmye/GBMhfHLpmnJws34XH13Le8hq+8eB2uqxejzFp1xztZ1GlF2RVl0Zo77WcrDAKdZAFsHZxBc/u6WRkxJJqjQmCn+tRWTx+TxZ4BUr/7oqVdPUP8c2HtmeqacbMSqrqBVmuJ6u6pMBmF4ZU6IOsNQ2VdA/E2dnWE3RTjJmV2nsGmVOYRyTv2G8XqxbO4e2n1XHb47vZ024FSo1Jl7aeGLH4yGiQVVUSGc2dNOES+iDrNFeU1PKyjAlGe29s3PIN4/nYm08gN0f40m9eSnOrjJm9/DULR3uySiN09Q8xZOsXhk7og6ylNSVUFOezebcVOzQmCO09r6/2fjTz5hRywwVL+fULrXZhZEyatLgga+HocKF3fnbakGHohD7IEhHW1FdY8rsxAekYp9r7sfzVG5cyt6yAL1iBUmPSwi9Eejjx3etptrys8Al9kAVevaztB3ro6rdZS8Zk2mSGCwGKI3l87M3H8+yeKL9+oTWNLTNmdmqO9lNakMecwjzgcHkVK0gaPlkRZPl5Wc9ab5YxGTUyonT2JT9c6HvbafWcOL+ML/3mJQbjw2lqnTGzkz+zUEQAqHGFgttsaZ3QyYog69T6CnIENu+xvCxjMqmrf4jhET1qtfejyc0RPn3lSvZ29HP741ag1JhUaon2s7CicPR+VYnX02w9WeGTFUFWSUEeJ8yfYz1ZxmRY+wTV3o/l/BW1vPH4Wv7jwe2WkGtMCiUWIgWoKMonR7AyDiGUFUEWeItFP7cnyrAVJTUmY0bXLSxJPicr0aevXEnPYJxvPGgFSo1Jhd7BONG+IRZVFI9uy8kRr1aWXcyETtYEWactruTQYJztBw4F3RQzS4nILhF5QUSeE5FNbluViNwvItvd90q3XUTkGyKyQ0S2iMjaYFs/Nf6b9mSHC33Hzyvjnac38MONu3mtrTeVTTNmVjpcvqHwiO3VJQW0W05W6GRNkLW2wUt+t3pZJmBvUtXVqrrO3f8k8ICqrgAecPcBLgdWuK8bgG9nvKUpMBpkTWG40Pc3l66gIC+Hm+/dlqpmGTNrNbkgqy5huBDcItHWkxU6WRNkLa4upqokYvWyTNhcDdzmbt8GXJOw/Xb1PAFUiMiCIBo4Hf6VceU0gqy5ZYW8/43LuG/rfp56rSNVTTNmVhpbiNTnLRJtQVbYZE2QJSKsbbCipCZQCvxWRJ4RkRvctnmq6heD2gfMc7cXAXsTfrbJbTuCiNwgIptEZNPBgwfT1e4p6+iNUV6UT37u9N4q/vL8pcyfU8i/WG+WMdPS3NlPXo4wt2zscGHEhgtDKGuCLIC1iyvZebDXZiqZoJynqmvxhgJvFJELEh9Ur7z5pGZmqOr3VHWdqq6rra1NYVNTo70nNuV8rERFkVyuPaOeZ/dEGRiyulnGTFVztJ/55YXk5sgR26tLC+geiBOL2/qFYZJdQZbLy3p2r/VmmcxT1Wb3/QDwf8AZwH5/GNB9P+B2bwbqE368zm3LKu29g9PKx0q0pLoEgKbOvpQcz5jZqMUVIh3LL7PS2WedEGGSVUHWKXXl5OaIJb+bjBOREhEp828DbwZeBDYA691u64G73e0NwHvcLMOzgK6EYcWs4S0OPbXyDWPVV3kfDHvdumvGmMlr7jyyRpbPqr6HU17QDZiM4kgeKxeU8cxu68kyGTcP+D+3jEUecKeq/kZEngZ+JiLXA7uBd7j97wGuAHYAfcCfZ77J09fRG+P041LTk1VX6dX1aeqwnixjpiI+PMK+7oGj9GRZ1fcwyqogC+C0hkr+55km4sMj5E0zGdeYZKnqTuDUcba3AxePs12BGzPQtLQZHlE6+mLUpGi4sLa0gEhejvVkGTNF+7oHGFGOOVxoVd/DJeuilLWLK+mLDfPyfitKakw6RftiqE5tSZ3x5OQIdZVF7LWeLGOmpCU6ALy+fAMcHi60Mg7hkn1Bll+U1BaLNiatOkarvacmJwugvrKYvZb4bsyUNEe9c2e8nKw5hfnk5ggdvZaTFSZJBVkicpmIvOyWCPnkOI83iMhDIvKsW0LkitQ31VNXWURNaQGbLS/LmLRq65l+tfex6quK2Nthw4XGTEWzG2pfWP76IGt0/UIbLgyVCYMsEckFvoVXG2gVcJ2IrBqz22eAn6nqGuBa4D9T3dCE9nDaYitKaky6pasnq6t/iO6BoZQd05jZojk6QHVJhKJI7riPV9si0aGTTE/WGcAOVd2pqjHgJ3hLhiRSYI67XQ60pK6Jr7e2oZLd7X02VdWYNGp3ww6pysmCxBmG1ptlzGQ1R8cv3+CrLrWq72GTTJCVzPIgnwX+TESa8Kauf3i8A6VqCZG1i11RUsvLMiZt/GGHyuL8lB3zcK0sy8syZrJaov3jDhX6qkoKrIRDyKQq8f064FZVrcOrDfRDEXndsVO1hMjJi8rJyxGrl2VMGrX3DlJZnJ/SUin1rifLZhgaMzmqetRCpL5qy8kKnWTePZNZHuR64GcAqroRKARqUtHA8RTm5/KGReWWl2VMGnX0xlI6VAhQUZxPaUEeTVYrK+uJSKGIPCUiz4vIVhH5nNt+nIg86SZK/VREIm57gbu/wz2+JMj2Z5to3xD9Q8Pjlm/wVZdEODQYZzBu64OGRTJB1tPACnfiRPAS2zeM2WcPriCjiKzEC7KmPh6YhLUNFWxpijI0bIthGpMObT2xlCa9gzdxxWplzRiDwEWqeiqwGrjMLSH1JeCrqroc6MS7CMd973Tbv+r2M0lqjnoXJuMVIvX556sNGYbHhEGWqsaBDwH3AdvwZhFuFZHPi8hVbrePAe8TkeeBHwPvdRWv02ZtQyUDQyO81GpFSY1Jh47eWErLN/jqq6xW1kygnh53N999KXAR8HO3/TbgGnf7ancf9/jF4tapMhPze3+PFWRZ1ffwSWpZHVW9By+hPXHbPyTcbgTOTW3Tjs1Pfn9mdwcn15Vn8lcbMyu09wxy1tKqlB+3vrKYx3a0oarYZ2x2cyV+ngGW45X6eRWIuotzOHKi1OgkKlWNi0gXUA20jTnmDcANAA0NDel+Clmjxe/JOkZOllV9D5+sq/juW1heyPw5hVb53Zg0iA+PEO0fGl10NpXqKovoiw3bkMYMoKrDqroaL1f3DODEFBwzJROkZprmaD9F+bnHnO3r92RZ1ffwyNogS0RYa0VJjUmLzr4hVA9fGadSfZWbYWjJ7zOGqkaBh4CzgQoR8UdJEidKjU6ico+XA+0ZbmrWaon2s7Ci8Ji9v35Olg0XhkfWBlng5WU1dfZzoHsg6KYYM6P4vUypnl0ICbWyLPk9q4lIrYhUuNtFwKV4ebsPAW9zu60H7na3N7j7uMcfTHfu7kziFSItPuY+cwrzyM8VGy4MkawOstaMLhZtvVnGpJJfNbo6DcOFo7WyLPk92y0AHhKRLXiz0O9X1V8BnwBuEpEdeDlXt7j9bwGq3fabgNetg2uOriXaz6KKwmPuIyJUFlvV9zBJKvE9rE5aNIdIbg6b90S57KQFQTfHmBmjfXTdwtT3ZJUU5FFVErGForOcqm4B1oyzfSdeftbY7QPA2zPQtBlnYGiYtp7YMWcW+qpLrep7mGR1T1ZBXi4nLZrDZqv8bkxKjS4OnYbhQoD6yiKarCfLmKT4NbKOVYjUZ4tEh0tWB1ng5WVtae4iFreipMakSnvPICJQUZyeIKuustiqvhuTpJYkCpH6vEWiLcgKi+wPshZXEouPsLWlK+imGDNjtPfGqCqOkJuTnjpWdVVFNHf2MzJiec/GTKS5c+IaWb6qkogNF4ZI9gdZo8nvVi/LmFRp70n9uoWJ6iuLiQ2PsP+QzQw2ZiIt0X5yBObNOXbiO0BNaQE9g3EGhmz9wjDI+iBrfnkhiyqKbIahMSmUjsWhE43WyrLkd2Mm1BTtZ/6cQvJzJ/7IPlyQ1HqzwiDrgyyANQ0VlvxuTAq19Q5Sk+LFoRPVV1qtLGOS5RUinXioEA5PVrG8rHCYEUHW2oZKWrsGaO2yq2JjUiHdPVl+bonVyjJmYl4h0iSDrNH1C61WVhjMjCDLLRa9ebflZZn0EpFcEXlWRH7l7h8nIk+KyA4R+amIRNz2And/h3t8SZDtnoyh4RGifUNpqZHlK8jLZd6cApthaMwEhkeU1ujAJHqybGmdMJkRQdaqBXMoyMuxvCyTCR/BWzrE9yXgq6q6HOgErnfbrwc63favuv2yQmdfemtk+eori2240JgJHDw0SHxEkyrfAFBVajlZYTIjgqxIXg6n1JXzjOVlmTQSkTrgSuC/3H0BLgJ+7na5DbjG3b7a3cc9frEca2XXEPGvgKvTmJMFXvK79WQZc2zNUe9CJNnhwrKCPCK5ObTZcGEozIggC7y8rK0tXTZt1aTT14CPA37l22ogqqpxd78JWORuLwL2ArjHu9z+RxCRG0Rkk4hsOnjwYDrbnrR0Lg6dqL6yiNaufoaGrZCwMUfTHPXKnCTbkyUiXq0sGy4MhRkTZK1pqGRoWK0oqUkLEXkLcEBVn0nlcVX1e6q6TlXX1dbWpvLQU9bmFpetSWNOFkBdVTEjeriatTHm9fxCpMnmZIGr+m7DhaEwY4KstYsrAEt+N2lzLnCViOwCfoI3TPh1oEJE/IXW64Bmd7sZqAdwj5cD7Zls8FQd7slK83BhpdXKMmYiLdF+yovyKS3Im3hnp8rWLwyNGRNkzS0rpL6qyPKyTFqo6tt7CU8AACAASURBVKdUtU5VlwDXAg+q6ruAh4C3ud3WA3e72xvcfdzjD6pqVqwh094TI0egoig/rb+nvsq7MreFoo05uuZof9JDhb6a0gLaeywnKwxmTJAFXl7W5j2dZMlnmZkZPgHcJCI78HKubnHbbwGq3fabgE8G1L5Ja3c1snLStG6hb/6cQnJzxGplGXMMzZ3J18jy2fqF4ZF8/2MWWNtQyd3PtdAc7afODUUYk2qq+jDwsLu9EzhjnH0GgLdntGEp0tE7OFprJ53ycnNYWFFow4XGHENLtJ+zl71uzswxVZVE6IsN0x8bpiiSm6aWmWTMqJ6s0xbbYtHGTFe6F4dOVF9ZbD1ZxhxFV/8QhwbjUxgutKrvYTGjgqwT55dRlJ9r6xgaMw0dvbG0VntP5BUktZ4sY8bjz7ydzMxCODxpxYYMgzejgqy8XK8oqVV+N2bq2noG017t3VdfVURbzyD9sczUt1NVegbjE+9oTAj45Rsmm5M1un6h1coK3IwKssBbx7CxpTtjb9rGzCSx+AjdA/G0V3v31Vd5uZN+Vet0u+eFfaz9/P1sfDUrqmmYWa6ly+/JKpzUz/kXSVbGIXgzLsg6raGS+IiypcnysoyZLH/dwkzlZNW5K/RMDRk+vauD2PAIH7pzM81WBNWEXHNnP5G8HGomORHFv0iyMg7Bm3FB1poGV5TUkt+NmbTRdQszmPgOZCz5vbG1m4aqYmLxEd7/w2dsGS4Tak2uRtZky6mURHKJ5OVYTlYIzLggq7q0gCXVxZaXZcwU+LORMjVcWFtWQEFeDns70h9kqSrbWrq54PgavvLO1bzQ3MWn/+9Fq6tnQqsl2j/poULw1i+sKYnQZjlZgZtxQRbA6UuqeHJnO4Nxu0o1ZjIytTi0T0SoqyzKyHBhU2c/hwbjrFpQzqWr5vHRS1bwv5ubuH3j7rT/bmOmorlz8tXefVWlETqshEPgZmSQdcUpC+geiPP7lw8G3RRjsop/5ZvuxaET1VdlplbW1pZuAFYtnAPAX1+0gktWzuWfftXIkzstEd6Ey2B8mAOHBiddvsFXXVJgie8hMCODrPOX11BdEuHu51qCbooxWaWjd5DcHGFOYXrXLUzk1cpKf5DV2NpNjsAJ88oAyMkRvvLO1TRUFXPjnZtp7bJEeBMe+7oGAKbck1VdErESDiGQVJAlIpeJyMsiskNExl2DTUTeISKNIrJVRO5MbTMnJy83h7ecsoDfbdvPoYGhIJtiTFbxq72ne93CRHWVRXQPxOnqT++52tjSzdLa0iOWGZlTmM/33nMa/bFh3v+jzZYIb0LDn/065SCrNGIV30NgwiBLRHKBbwGXA6uA60Rk1Zh9VgCfAs5V1TcAH01DWyflqtWLGIyP8JsX9wXdFGOyRntvLGMzC31+raymNA8ZbmvtZuWCOa/bvnxuGf/+jtU8vzfKP9xtifAmHKZaiNRXVVLAwNAIfTErvhukZHqyzgB2qOpOVY0BPwGuHrPP+4BvqWongKoeSG0zJ29tQwUNVcVseN6GDI1JVnvPYMaW1PGNlnFIY/J7tC9Gc7SfVeMEWQCXnTSfD1+0nJ9tauKOJ/ekrR3GJKs52o8IzC+f/OxCsKrvYZFMkLUI2Jtwv8ltS3Q8cLyIPCYiT4jIZalq4FSJCFevXshjO9o4cGgg6OYYkxU6emOj655lSn2Vd6Wezp6sxtYjk97H89FLjudNJ9TyuV9uZdOujrS1xZhktET7qS0toCAvd+Kdx2FV38MhVYnvecAK4ELgOuD7IlIxdicRuUFENonIpoMH0z/z7+rVCxlR+OXzrWn/XcbMBO09mR8uLC/Kp6wgL63J79taDwEctScLIDdH+Nq1a1hUUcQH7tjM/m67ODPBaY72T3moEKzqe1gkE2Q1A/UJ9+vctkRNwAZVHVLV14BX8IKuI6jq91R1naquq62tnWqbk7Z8bhlvWDiHu58b21xjzFiD8WEODcYzHmSJCHVVxeztTN9wYWNLN7VlBdSWHbuXrrwon+++ex29g3E+8KNnrNaeCUxLdGDK5RvAerLCIpkg62lghYgcJyIR4Fpgw5h97sLrxUJEavCGD3emsJ1Tds3qRWxp6mLnwZ6gm2JMqHX2erP7MlXtPVFdZVHahwuP1YuV6IT5Zfzb209l854on93QmLY2GXM0IyNKc7SfumkEWX5BYVtaJ1gTBlmqGgc+BNwHbAN+pqpbReTzInKV2+0+oF1EGoGHgL9V1VBU93vrqQsRwWpmGTOBNjeskKlq74m8Wln9aZnZF4uPsOPAoWPmY411xckL+OCFy/jxU3u40xLhTYa198aIxUem1ZNVHMmlMD/HhgsDllROlqreo6rHq+oyVf2i2/YPqrrB3VZVvUlVV6nqyar6k3Q2ejLmlxdy9tJq7n6u2aZmG3MM/hVvJqu9++qriugfGk7L0Mb2A4cYGtake7J8H3vzCVxwfC3/uOFFntlta6GazJlujSzwhuGt6nvwZmTF97GuXr2QXe19bGnqCropxoSWX7gwqJ4sIC3J740tE88sHE9ujvCNa1ezoLyID/zoGQ5YIrzJkOnWyPJVl1rV96DNiiDrspMWEMnN4S5LgDfmqPw34+oMl3CAwwVJ05H83tjaTVF+LkuqSyb9sxXFEb777tM4NBDng3dsJhYfSXn7spWI1IvIQwkrfXzEba8SkftFZLv7Xum2i4h8w60cskVE1gb7DMKrxfVkTWe4ELwLJsvJCtasCLLKi/K56MS5/PL5VuLD9iZpzHjae2Pk5QhzivIy/rvr3BV7unqyTphfRu4UlwpauWAO//q2U9i0u5N/+pUlwieIAx9T1VXAWcCNbjWQTwIPqOoK4AF3H7xVQ1a4rxuAb2e+ydmhOdpPWUEe5UXTW0O0uqTAcrICNiuCLIBr1iykrWeQx18NRT6+MaHT4dYtFMncuoW+koI8qksiKZ9hqKrezMJJDhWO9dZTF/JXFyzlh0/s5mdP7534B2YBVW1V1c3u9iG8iVGL8FYEuc3tdhtwjbt9NXC7y+F9AqgQkQUZbnZWaI72T7sXC/z1C2OWjxygWRNkXXjCXMoK82yWoZkSESkUkadE5Hk3NPI5t/04EXnSDYH81JU5QUQK3P0d7vElQbY/Ge29g4GUb/B5ZRxSO1zYHO3n0EB80knv4/nbPzqB85bX8Jm7XuS5vdEUtG7mcK/vNcCTwDxV9StA7wPmudvJrB6S8aLVYdTcOb1CpL7qkgiD8RF6Y1bvLSizJsgqzM/l8pPmc9/WfQwM2QvOTNogcJGqngqsBi4TkbOALwFfVdXlQCdwvdv/eqDTbf+q2y/UglgcOlFdVXHKhwunmvQ+nrzcHP7jujVUlUT419+8NO3jzRQiUgr8L/BRVe1OfEy9LpRJdaNkumh1GHk9WVNbszDRaK0sS34PzKwJssArTNozGOd32/YH3RSTZdwQh1/RNt99KXAR8HO3fezQiD9k8nPgYgliHG4S2ntiGV8cOlF9ZTHN0X6GR1I3tNHY2o0InDi/LCXHqyyJcOUpC9i0u9Mu1gARyccLsO5Q1V+4zfv9YUD3/YDbnszqIbNez2Ccrv4hFlUUT/tYNa5nuq3X8rKCMquCrDOXVjNvToENGZopEZFcEXkO70PjfuBVIOoK9sKRwx+jQyPu8S6gepxjhmZoxFscOsAgq6qIoWFN6ZqBjS3dHFdTQnEkdcn85y6vJhYfYfMsr53lLhpuAbap6lcSHtoArHe31wN3J2x/j5tleBbQlTCsaBx/ZmEqhgutJyt4syrIys0R3nrKQh5++QDRPnvRmclR1WFVXY13BX4GcGIKjhmKoZGBoWF6BuOjV75BSEetrMksp5Os05dUkZsjPPZqW0qPm4XOBd4NXCQiz7mvK4CbgUtFZDtwibsPcA/ecms7gO8DHwygzaF3uBDp9IcL/Z7pduvJCsysCrIArlmziKFh5Z4X9gXdFJOlVDWKt3zU2XgzpPxuksThj9GhEfd4ORDaqa1+LZ1ge7JSWyurq3+Ips7+lORjJSorzOeUuvJZP1NZVR9VVVHVU1R1tfu6R1XbVfViVV2hqpeoaofbX1X1RrdyyMmquino5xBGo4VIUzBc6Ne8s6rvwZl1QdYbFs5hWW2JFSY1kyIitSJS4W4XAZfiTVl/CHib223s0Ig/ZPI24EEN8Tzqw4VIgwuyFlYUIkLKyjhsa3VJ7ynuyQI4d1kNW5q6ODQwlPJjm9mtJdpPXo5QWzb9XuWiSC7FkVyr+h6gWRdkiQjXrF7EU691jI59G5OEBcBDIrIFeBq4X1V/BXwCuElEduDlXN3i9r8FqHbbb+JwQcZQ8ocTgkx8L8jLZV5ZIXs7UnNepnJm4VjnLKtmeER56rWOlB/bzG7N0X4WVBROuXjuWFb1PVizLsgCuGr1QgA2PG8J8CY5qrpFVde4oZGTVPXzbvtOVT1DVZer6ttVddBtH3D3l7vHdwb7DI7NfxMOYkmdRPVVRexNUU9WY2s3NaUR5pZNP7dlrLWLK4nk5cz6IUOTes2d/Swsn37Su6+6JGLDhQGalUHW4uoS1jRUcNezNmRoDBweLqwKsCcLvOT3phQlvm9r7WZlGoYKwau7t25xpQVZJuVaoqkpROqrLrWldYI0K4Ms8GpmvbTvEC/vOxR0U4wJXHtvjEhuDmUFmV+3MFFdVTGt3QPTXog5Fh9h+/6etAwV+s5ZVs221m77ADMpMzQ8wr7uAepSsKSOz4YLgzVrg6wrT1lAbo5wtyXAG0N7z2Bg6xYmqq8sQpVp50u+erCH2PBIWpLefWcvqwHgiZ2Wl2VSY3/3ACNKStYt9FWXRmjvsfULgzJrg6ya0gLOW17D3c+1MJLCCtPGZKOgC5H66lytrOmuYegnvb8hjT1Zp9aVU1qQx+NWL8ukyGj5hlQOF5ZEiA2P0DMYn3hnk3KzNsgCuGbNQpqj/TyzZ3ZXbjamrTfYJXV89VXeh8t0k98bW7spzM/huJrSVDRrXHm5OZxxXJXlZZmUaenygqyU9mT5tbKsjEMgZnWQ9eZV8ynKz7UhQzPrdfQOBlojy7egvIi8HJl21ffGlm5OmD8nZdPgj+acZdW81tZr5WBMShwuRJrCnKzRqu8WZAVhVgdZJQV5XLpqHr/e0srQ8PQSbY3JZt7i0MGWbwBv6auFFUXTqvquqmlZTmc857i8rI3Wm2VSoDnaT3VJhML83JQds2a0J8smaARhVgdZAFevXkhn3xB/eCXYxXmNCUp/bJi+2HAocrLA1cqaRk9WS9cAXf1DaZ1Z6DtxfhlVJRFbx9CkRHN0IKX5WHC4J8tmGAZj1gdZFxxfS2VxPnc9Z4VJzezkV3uvCUFOFrhaWdPIyRqt9J6BnqycHOHspdVsfLXdZm+ZaWvu7EvpUCEcXirLhguDMeuDrPzcHK48ZQH3N+6j12ZfmFno8OLQwQ8XAtRVFtHWE6MvNrXzcVtrNyJeL1MmnL2smtauAXa1p6aIqpmdVJWW6EBKk97BK5xbYusXBmbWB1kAV69exMDQCL9t3Bd0U4zJuNHFocPSk1XllXFonmJeVmNLN0uqSyjJUGHVc5ZVA/DYDhsyNFPX2TdE/9BwynuywFV977WcrCBYkAWc1lDJoooi7nrWhgzN7NM+um5hOIIsv1bWVMs4ZCrp3XdcTQkLygst+d1Miz9DNdU9WWBV34NkQRZeXsXVqxfy6I422mwGhpll/FlHYZhdCAm1sjom35PVPTDEno6+jCS9+0SEs5dVs3FnuxU2NlPmF+CtS3HiO3j5lm02XBgIC7Kca9YsYnhE+fWW1qCbYkxGdfTGiOTlUBJJ3bTx6agtLaAwP2dKMwxfavXWIs1kTxZ4pRw6emO8ZGuhmilqTntPlnUgBMGCLOf4eWWcOL+Mu6wwqZll2ntj1IRg3UKfiFBXWTyl4cLGli6AjPZkweG8LFtix0xVS7SfovxcKovzU37s6tICOnpt/cIgWJCV4Jo1i3h2T5Td7b1BN8WYjGnvGRytpRMW9ZVFUxoubGztprokwtyyzA59Lqwo4riaEsvLMlPW3NnPosqitFzsVJdEGBpWugdsBn2mWZCV4KpTFyICG6xmlplFOnpjo+ubhUXdFGtlNbZ2s2rhnEB65c5eVs2Tr3UQt9UjzBS0dPWnZagQGC00bMnvmWdBVoKFFUWcsaSKu55rtm5VM2u09cRCM7PQV19VRPdAnK7+oaR/Zmh4hFf29WQ8H8t37rIaegbjbGnuCuT3m+zW3NmflvINcHhSiy2tk3kWZI1x9epFvHqwl62uarQxM11Hbyw0S+r46v0yDpNIft95sJfY8EjG87F8Zy2tAmwdQzN5A0PDtPfGWFRRmJbjW9X34CQVZInIZSLysojsEJFPHmO/PxERFZF1qWtiZl1x8nzyc4W7nrUEeDPz9cXi9A8Nh6Z8g88vSDqZIcPGVpf0HlBPVnVpASfOL7PkdzNp/szCVK9b6PMLDVvV98ybMMgSkVzgW8DlwCrgOhFZNc5+ZcBHgCdT3chMqiiOcOEJc/nllhaGreaNmeFGq72Hticr+eT3xpZuInk5HFdTkq5mTejc5TVs2tXJwNBwYG0w2cdf3WBhebpzsmy4MNOS6ck6A9ihqjtVNQb8BLh6nP3+CfgSMJDC9gXimtWL2N89yBM7rdvfzGyj1d5DNruwvDifssK8SZVxaGzt5sT5ZeTlBpcFcc6yagbjI2ze0xlYG0z2aUlzT1ZBXi5lBXlWkDQAybwbLQL2JtxvcttGichaoF5Vf32sA4nIDSKySUQ2HTx4cNKNzZSLV86luiTCNx7YbgnwZkbzr2zDlpMF/gzD5HqyVJXGlswupzOeM46rIjdHLC/LTEpztJ8cgflz0pOTBVBVakvrBGHal3wikgN8BfjYRPuq6vdUdZ2qrqutrZ3ur06bwvxc/ubS43nytQ5+27g/6OaYEBCRehF5SEQaRWSriHzEba8SkftFZLv7Xum2i4h8w+UxbnEXIqHjX9nWhCwnC/xaWcn1ZO3rHqCzbyiwpHdfWWE+p9SV22LRZlKao/3Mn1OY1l7Y6pKILRIdgGT+o81AfcL9OrfNVwacBDwsIruAs4AN2Zz8DnDt6fWsmFvKv9yzjVjc6t4Y4sDHVHUV3mv8Rpeb+EngAVVdATzg7oOXw7jCfd0AfDvzTZ6Yf2Ubxp6s+iqvJyuZ3uRGNxs46J4s8IYMn2/qomfQCj+a5PiFSNOpqqTAEt8DkEyQ9TSwQkSOE5EIcC2wwX9QVbtUtUZVl6jqEuAJ4CpV3ZSWFmdIXm4Of3flSna19/HDJ3YH3RwTMFVtVdXN7vYhYBvesPnVwG1ut9uAa9ztq4Hb1fMEUCEiCzLc7Am19wxSmJ9DcUjWLUxUX1lE/9BwUnkkfpB1YiiCrBqGR5SnX+sIuikmS6SzEKmvpjRiJRwCMGGQpapx4EPAfXgfLD9T1a0i8nkRuSrdDQzShcfXcv6KGr7xwHaiffbiNB4RWQKswZtJO09V/VXF9wHz3O0JcxndsQLNU2x31d7Dsm5hIr+MQzLJ742t3SypLqa0IC/dzZrQaYsrieTl2JChScrwiNIaHUhbIVJfVUmEzt4YIzZrPqOSGgBW1XtU9XhVXaaqX3Tb/kFVN4yz74XZ3ovlExE+feVKDg0M8Y0HdgTdHBMCIlIK/C/wUVU9omKteuNak3oHCzpPsb0nFrqZhb7RICuJvCx/OZ0wKMzP5bSGSh635HeThAOHBoiPaNp7sqpLC4iPKN0Dya+iYKbPKr5P4MT5c3jn6Q3cvnEXOw/2BN0cEyARyccLsO5Q1V+4zfv9YUD3/YDbPlEuYyiEsdq7z7+yn2iGYc9gnN3tfaHIx/Kds6yaxtZuOm14xkwg3eUbfFb1PRgWZCXhpkuPpyAvh5vvfSnoppiAiDeedguwTVW/kvDQBmC9u70euDth+3vcLMOzgK6EYcXQCOPi0L6SgjyqSyITVn1/qdUlvYekJwvgnOU1AGy0WntmAv5FRF3ae7Ks6nsQLMhKQm1ZAR9803J+27jf6t/MXucC7wYuEpHn3NcVwM3ApSKyHbjE3Qe4B9gJ7AC+D3wwgDYfk6rS1jMY2uFCgLqq4gmrvje6IGtliHqyTqkrpySSa0vsmAm1RL363ekeLrSq78EIPks0S1x/3nHc8cRuvvDrRn75ofPIyQlforBJH1V9FDjaP/3icfZX4Ma0Nmqa+mLDDMZHQrekTqL6yiJeaO465j6NLd1UFuentZDjZOXn5nDm0mrLyzITao72UVGcT0maJ234PdY2XJhZ1pOVpML8XD5x+YlsbenmF7Z4tJkB/GGDsOZkgZf83hLtP+Y6on7Se9hmSJ6zrJqdB3vZ15X1K42ZNGqJDqRtzcJE/nluw4WZZT1Zk/DWUxbyg8d28eX7XuKKk+dTHLE/n8lefvXnMFZ799VXFjM0rOzrHn+Ke3x4hJf2HWL92YsDaN2xnb2sGoDHX23jj9fWBdya1BORHwBvAQ6o6kluWxXwU2AJsAt4h6p2upzGrwNXAH3Ae/26c7Ndc2c/DdXFaf89kbwcygrzQrO0zvCIsrWli0e2t/HI9oM8tzd6zIupTPny207lmjWvq7YzZRYlTEJOjvD3V67kbd/ZyPf/8BofuWRF0E0yZsqyoyfLC6z2dvSNG2TtbOslFh8JVdK7b+X8OVQW5/P4q+0zMsgCbgW+CdyesM1fAeFmEfmku/8JjlwB4Uy8FRDOzGhrQ0hVaY72jwbk6VZTWkBbT3A5WU2dfTy6vY1HdrTx2I42on1eOYlVC+Zw7ekNoSiKvHxuaUqPZ0HWJK1bUsUVJ8/nO79/lWvPqGdeiPJAjJmMMC+p46ur9K7wj1bG4fByOuUZa1OycnKEs5dV8/iONlQ1dMOZ06Wqf3CFeRNdDVzobt8GPIwXZI2ugAA8ISIVIrIgjDNuM6l7IE7PYDzthUh9VSWZXST60MAQG19t59EdbTy6vY2dbb0AzJtTwCUr53H+ihrOXV4T6t706bIgawo+cdmJ/K7xAP9238t8+e2nBt0cY6akzQ0Xhnl24cKKQkSOXpC0sbWbSF4OS2tLMtyy5Jy9rIZ7XtjH7vY+ltSEs40pNtkVEF4XZInIDXjrfdLQ0JC+loZAc2dmamT5qksi7G5PbtH1qYgPj/B8UxePbD/Io9vbeNYNARbl53LW0ireddZiLlhRw/K5pTPuouNoLMiagsXVJbz33CV8/5GdrD9nCSctCt9VtDET6eiJUZSfG+rcwoK8XObPKTzq0jrbWrs5YV4Z+bnhnMNz7mheVvtsCbJGqaqKyKSTbFT1e8D3ANatWxd8kk4a+YVI012+wVddGmHznmjKj/vYjjZu37iLx19t59BAHBE4ZVE573/jUs5bXsvaxRUU5AU/FBiE8L67htyNb1rO/2zayxd/vY0733fmrInKzczR3hveJXUS1VcW0zROrSxVpbGlm0tWzhvnp8LhuJoS5s8p5LFX2/jTM2d2r4yz3x8GzMYVEDKt2a/2nqkgq6SAzj5v/cJUlSEaGh7h/T98hsJILleevIDzV9RyzrJqKkOchpBJ4bz8ywLlRfn8zaXHs3FnO7/bdmDiHzAmZLzFocP/RlhXVTRuT9aBQ4O098ZCmfTuExHOWVbNE6+2z5aFebN6BYRMa472E8nLydh5WFUSYXhE6epP3fqFz+2Ncmgwzj9d/QZu/pNTuPKUBRZgJbAgaxquO6OBpbUl/Ms92xgaHgm6OcZMSnvPINVZkHBaX1nMvu4BBuPDR2wfTXoPcZAF3hI77b0xXjlwKOimpJSI/BjYCJwgIk0icj1ZvAJCEJ58rYMT55dlrLj16NI6Kaz6/sgrB8kRL//QvJ4FWdOQn5vDp69Yyc62Xu54YnfQzTFmUsK8OHSiusoiVKE1emRRT385nRPnlwXRrKT50/Mf2zGzqr+r6nWqukBV81W1TlVvUdV2Vb1YVVeo6iWq2uH2VVW9UVWXqerJqrop6PYHrTnaz/N7o1x20vyM/c7Rqu8pLEj6h+1trK6voLwoP2XHnEksyJqmi06cy7nLq/naA9vp6ktdF6wx6aSqtPdkSU5WlVfGYeyQYWNLNw1VxZQVhvvNfVFFEUuqi9lo6xiaBL95cR8Al5+0IGO/83BPVmqCrK6+IbY0RTl/RW1KjjcTWZA1TSLCp69YRVf/EP/x4Pagm2NMUnoG48SGw71uoW80yBqT/N7Y2s2qEC0KfSznLK/hyZ0dxC2twDi/ebGVE+eXcVwGZ53653uqgqzHX21jROGC422o8GgsyEqBVQvn8I7T6rlt4y52uWJrxoSZX5DQHz4Is/lzCsnPlSN6snoG4+xq7w19PpbvnGXVHBqMT7jYtZkd9ncPsGl3J1ecnLleLGA0Ib09RVXf/7C9jbKCPE6tq0jJ8WYiC7JS5GNvPp783BxuvveloJtizITa/CV1smC4MDdHWFhRdERB0pf3daNK1vRknb30cL0sY+7bug9VuDyD+Vjg5RGXF+WnpOq7qvKHVw5yzvJq8kJapy4M7C+TInPnFPL+Ny7jN1v38dRrHUE3x5hj8t9ka7KgJwu8GYZ7E5bWaWz1ZuplS09WdWkBJ84vY6MFWQa494V9LJ9byop5mZ+0UV0aSclw4a72Ppqj/ZaPNQELslLofecvZf6cQr7w68bZUhPHZCl/uCAberLAm2HYlNCT1djSTUVxPgvKs2ft0HOW1fD0rg4GhoYn3tnMWO09gzz5WnvGe7F81SWRlAwXPrL9IAAXWJB1TBZkpVBRJJePX3YCW5q6uPv5WV/M2IRY+2hOVnYEWfVVxbT3xuiLxYHDSe/ZtNLCOcuqGYyP8GwaljUx2eO3jfsZ0czOKkyUqkWi//BKG4uri2moLk5Bq2YuC7JS7JrVizh5UTn/+puX6Y/ZFasJp/aeGCWRXArzs2M9sTq3gG5TZz/x4RFe5bzHMQAAGfJJREFUyqKZhb4zl1aRmyNWymGWu+eFVpZUF7NyQTD13apLC6ZdJ2toeISNr7Zx/gqbVTgRC7JSLCdH+MyVK2ntGuC/HtkZdHOMGVdH72DWDBVCYhmHPna19zIYH8mafCxfWWE+Jy8q5zHLy5q1on0xNr7azmUnLQisF7a6JEJnX4zhaaS0PLsnSm9s2PKxkmBBVhqcubSaP3rDPP7z4Vd50aZsmxDy1i3MjqR38BLfwQuytrrldFZmWU8WeEOGz++N0jMYD7opJgD3N+4nPqJccXIw+VjgBVkj6gV8U/XI9oPk5sjoagbm6CzISpPPX30SVSUR3vvfTx8x9dyYMGjvyY7FoX01pREK83PY29lPY2s3kdwcltWWBt2sSTt3eQ3xEeXpXTYDeTa698V9LKoo4uRF5YG1ocqtVzqdvKw/bG9jTX0Fc0K+2kIYWJCVJvPmFHLbX5zO0PAI63/wVEoSDY1JlfbewaxYUscnItRVFrO3o4/Glm5WzCslkpd9b1+nLa4kkpvD4zssL2u26R4Y4tHtbVx+0vxAJ2zUuIurtinmZUX7YraUziRk37tUFlk+t4z/Wr+Opmg/19/2tCXCm1BQVbc4dPYMFwLUVxbR1NnPtixMevcV5ueydnGFFSWdhR7cdoDY8AiXZ7jK+1h+LuZUL/wf29GOKpxvS+kkJS/oBsx0py+p4hvXruYDd2zmwz9+lu/82VqrjpuFROQHwFuAA6p6kttWBfwUWALsAt6hqp3iXaZ+HbgC6APeq6qbg2j3eLoH4gwNKzVZ1JMFXvL7ozvaGBrWrEt6T3Tushq+8rtX+PFTe8jLCbYExZqGCpbPDWaW22xzzwutzJtTwJr6YJeg8XMx23unVivrke0HmVOYxykBDnlmEwuyMuCykxbw2be+gX/csJW/v3sr//z/nZRV9X0MALcC3wRuT9j2SeABVb1ZRD7p7n8CuBxY4b7OBL7tvoeCfwVblUU5WeAlvw8NezOisrUnC+BNJ87l3+9/hU/94oWgm8Jn37rKgqwM6B2M8/tXDnLdGQ3kBBxYVxZ7eVRTKeOgqjyyvY1zl9dYZ0GSLMjKkPXnLGFf9wDffvhVFpYX8uGLVwTdJDMJqvoHEVkyZvPVwIXu9m3Aw3hB1tXA7aqqwBMiUiEiC1S1NTOtPTa/2nN1aZYNF1YVjd5emcU9WSctKmfTZy4JRfpARbElLmfCQy8fYDA+wmUBVXlPlJebQ2Vx/pR6sna29dIc7efGNy1PQ8tmJguyMujjf3QC+7sG+Pf7X2FeeSHvWFcfdJPM9MxLCJz2AfPc7UXA3oT9mty21wVZInIDcANAQ0ND+lqaINuqvfvqXBmH+qqirJ/VVJNlAa6Znntf3EdNaYTTl1QF3RRg6lXfH3nFW0rHipAmL6n+PhG5TEReFpEdblhk7OM3iUijiGwRkQdEZHHqm5r9RISb/+QUzl9Rw6d+8QIPvXwg6CaZFHG9VpOu7qeq31PVdaq6rrY2M7N1/GGCbJpdCIcLkmbzUKGZffpjwzz00gH+6A3zyQ14qNBXXVowpdmFj2xvY0l18ei5aCY2YZAlIrnAt/DyTFYB14nIqjG7PQusU9VTgJ8D/5rqhs4Ukbwcvv1np3Hi/DJuvGMzW5psHbMstl9EFgC4737U3AwkdlPWuW2h0OGGCbItJ6u8KJ9zllXz5lXBD7kYk6zfv3KQvthwYGsVjqd6Cj1ZsfgIG3e2W+mGSUqmJ+sMYIeq7lTVGPATvJyTUar6kKr6FTefwPtQMUdRWpDHf//56VSVRPiLW59md3tv0E0yU7MBWO9urwfuTtj+HvGcBXSFJR8LvOHCsoI8CvKyY93CRHe+7yz+5DR7ezHZ4zcvtlJZnM+ZS8MxVAheL/Zkg6zNezrpiw3bUOEkJRNkHS2/5GiuB+4d7wERuUFENonIpoMHDybfyhloblkht/3FGcRHlPU/eGo0GdmEk4j8GNgInCAiTSJyPXAzcKmIbAcucfcB7gF2AjuA7wMfDKDJR9XeE8uqdQuNyVaD8WEe2HaAS1fNIz9Es/GqSgomvX6hLaUzNSn9r4vInwHrgC+P93gQ+Sdhtqy2lFvWn05r1wB/cdsm+mK2nllYqep1qrpAVfNVtU5Vb1HVdlW9WFVXqOolqtrh9lVVvVFVl6nqyaq6Kej2J+roza4ldYzJVo/taOPQYDzwAqRj1ZRGUIXOSaxf+Mj2NtY2VFCW5ZNOMi2ZICup/BIRuQT4NHCVqlq3TJJOW1zJf1y3hheaonzozmeJD48E3SQzw7X1DGZdtXdjstE9L+yjrDCPc5eFa4jNz8dMtlZWR2+MF5q7LB9rCpIJsp4GVojIcSISAa7FyzkZJSJrgO/iBVg2ZW6S3vyG+Xz+6pN48KUDfOauF/EmqhmTHtaTZUz6DQ2PcH/jfi5dOS9062yOBllJ1sp6bEebt5SO5WNN2oR1slQ1LiIfAu4DcoEfqOpWEfk8sElVN+AND5YC/+Mqme9R1avS2O4Z58/OWsy+rgG++dAO5pcX8v/au/PouOrrgOPfO9rGGu3SSJa8L5KNMRiMHWPjhRhIjaHQJQ3OnjaJ24YGSNKTk7RpTpNz0ixN05ATSutQCKEcaOJmgcQGWiC2zG5DgiW8SBY2NhpZM5K1jKxdt3/MaPUm8My8N6P7OcdHozmS3h3Ld3zf+/3evXddX+V0SCYFDc8tTLb2DcYkmxeOtNDe3e+6pUIY7dM22StZI6N0Zjo7EigZTaoZqaruILKZd+xzXx3z+PoYxzUlfeF9VQTae/j+/9VRnu/ltpWJaU5ppo6O7gEGhjTp2jcYk2x21gTwZaa58urPcP5P5g7D4VE6aytLXNPnK5m46xrmFBdpVnoZ66v8/N0vanjm4EmnQzIpJhRdHrCO48bEz8DgEE/VnmTjJWV4M9zXKqUwOxMRJnVX+5FgmEB7j+3HepesyHKZjDQP9354OUvK87j94dd45Wir0yGZFJKsw6GNSSYvH22lpauPzS6YVXg2aR6hMDtzZMTW+ew+HAJg7UL3XZFLBlZkuZAvK537P7GSsrwstmx7kR8+U/eO+pkYcy6jw6GtyDImXnbub8Kb4WHDIvde/Sn2ZU5qT1Z1XZD5JT4bpfMuWZHlUv7cLH51+1o2X1bOd586zJZtL3C89fSFv9GY8xgdDm3LhcbEw9CQ8kRtE+9dVEp25qS2PTtiMkOiewcGebGh1ZX7ypKFFVkulp+dwQ+2XMG/3raMA4FONt9dzS9fc80IPJOEhs9cbbnQmPjY99Ypgp29bHLpUuGwkpyskT2a57Lv2Cm6+wdtP9ZFsCLL5USEP75yJjvvXMei6bnc9d+/445HXqO9u9/p0EwSau3qI9eb7rq+Pcakip37m8hM97BxcanToZzXZK5kVdeFSPcIV9sonXfN3mmTxKyibB7dejWfv6GK3+wPsPnual5qaHE6LJNkQuFeu7PQmDgZGlJ21gRYX+l3/fiZ4pxM2k7303+eKSPVdUGWzykkJ8u9y55uZ0VWEklP83DHdZVs/6vVpKcJW370Iv/85MHzJokxY7V29dlSoTFx8vsTbQTae7jR5UuFwMjUh3PNL2wJ91LzdgfrbT/WRbEiKwldObuQHXes4wNXzeKeZ4/wp/c+T0Mw7HRYJgnYSB1j4ueJmiYy0oTrLylzOpQLKo5e0T7XkuGe+kjrBtuPdXGsyEpSvqx0vv3+y7n3w8s51nKam36wh0defsvmHprzCoVtpI4x8aCq7KgJsGZBCfnZ7l4qhAsPia6uC1GQncHSGfmJDCvlWJGV5G68rJwn71rP8jkFfPnn+9n60L5JjUowU8/QkHLqdJ+1bzCOEZFNInJIROpF5EtOxxNLtY0dHG/tZvNl7l8qBCjJGR4Sfeb/F5FROkGuWWijdC6WFVkpYHq+l4f+YhVfuekSdh0K8gff383uw0GnwzIu097dz6DNLTQOEZE04B7gRmAJ8EERWeJsVLGzsyZAmke4YUlyFFlFvuEh0We2cahrDnOyo9f2Y8WAFVkpwuMRPrVuPr+8/RoKpmXwsftf5muP19LTP+h0aMYlRhqR2nKhccZ7gHpVbVDVPuBR4FaHY4oJVWXn/iaunl+UNCcxBdMy8MjZ92QNn6Svtf1YF82KrBSzpCKPxz+7lk+smcsDzx3llh/uYcf+gI3lMaMjdWy50DhjBnB8zOcnos+NIyJbRWSviOwNBpPjivzhk2EaQl3cuLTc6VAmzeMRinyZhM6yJ6u6LsQCv48ZBdMciCy1WJGVgrwZafzjLZfywJ+vpKd/iM88/CrXfvdZ7t/zJuHeAafDMw6x4dAmGajqNlVdoaor/P7kuJKyY38AEXjfpe6/q3CsSEPS8cuFPf2DvPRmi91VGCNWZKWw9y4q5dm/vZZ//8hVlOV6+fqv32D1N5/mmzsO0NjW7XR4JsFC0SKrxJYLjTPeBmaN+Xxm9Lmk90RNEyvnFlGa63U6lHek2Jd1xt2Frx47RU//EOurbD9WLFiRleLSPMKmpdPZ/tdr+MVn1rC+ys+PqhtY/51nufPR19h/ot3pEE2CtEbfTAvtSpZxxitApYjME5FMYAvwmMMxXbQjwTCHTnayOQkakE5UlHPmaJ3ddSEy0oRV82yUTixYr/wp5MrZhdzzoUKOt57mweeP8ugrx/nV7xpZNa+IT62bz3WLS/HY7bopq6Wrl/xpGWSk2bmVSTxVHRCRvwGeBNKA+1W11uGwLtoTNU0AbEqi/VjDSnyZhCbcXVhdF+SqOYX4bJROTNi77RQ0qyibr9y8hOe/vJGv3HQJJ0518+mf7OW67+3ioReP0d1ndySmohbr9m4cpqo7VLVKVReo6jecjicWduwPsHx2AdPzk2upECJtHDp6BugbiIxmC4V7qW3ssP1YMWSl6hSW583gU+vm84k1c9lZ08R91Q38wy9r+JenDvGRVXP42Oo5lOYl3xuHW4jIJuBuImft96nqt5yMpyXca+0bzJTXNzDEa2+d4rn6ED0DQyz057CwLIeFpTnkvcOhzm+1nKa2sYO/33xJnKKNr+H3g1On+yjL8/JcdJTOeiuyYsaKLEN6moc/XFbBzZeXs/fYKe6rbuCe39azbXcDt1xRwQffM5srZhVY5993YEzjxRuI3Kr+iog8pqpvOBVTa1cf80p8Th3eGEeoKkeCYarrQuypC/FCQwun+wbxSOS9b/gqDkBZXhaVpbksLI0UXZXRj8Nz/ibaWRMAYFMS7seC0SHRLeFIkbX7cIjC7AwurchzOLLUYUWWGSEirJxbxMq5RRwNdfHAc2/y070n2L7vBAXZGayr9HNtlZ91VSVJdxeNA0YaLwKIyHDjRceKrJZwHyvmFjl1eGMSpiXcy576SFG1pz5EoL0HgDnF2fzJ8hmsXehn9YJicrLSOd56mvrmMHXNYeqaOznSHOZne4/TNWbbRJEv84zCq7I0lx01TVw+M59ZRdlOvdSLMlw8tnT1jozSWVvpt725MWRFljmruSU+vnbrUj7/vkXsOhzkt4ea2X04yOO/bwTg0oo8NlT5uXZRKVfOLrDN1Gc6W+PFVRO/SES2AlsBZs+eHbdgRucW2nKhST09/YPsO3aK6roQ1XVBahs7AMjzpnPNwhI+u9HPusqSsxZDc0t8zC3xcf2S0R5XqkqgvYe65jD1zWHqmzupOxnmN68HaO/uH/f9X9y0KL4vLo6Ge+a1dvVx+GSY5s5e1tkonZiyIsucV/60DG5ZVsEtyyoYGlLeCHSw63CQXYeC/MfuBv7tt0fIzYq8kW1Y5GdDlZ8K6xI8aaq6DdgGsGLFiri15W/r7mdIsSLLpARV5WBTJ3vqQlTXh3j5zRZ6+odI9wjL5xTyhRuqWFtZwuUz3902BxGhomAaFQXT2FA1uj9JVQmF+0aueAXae/jQe+J3chRvwz3zQuE+qusi3fWtyIotK7LMpHk8wtIZ+Sydkc/t711IR08/z9eHole6gjxRG7mVuaoshw1VfjZUlbJyXiFZ6WkOR+4IVzVeHB6pU3SOvSXGJJOf7TvBF7e/DsACv48tK2ezrrKEVfMjS4DxIiL4c7Pw52axZkHyFyN53gzSPEJrVy/73+6gsjSH8nw7SY4lK7LMu5bnzWDT0nI2LS1HValrDrPrUJBdh4M8+PwxflT9JtMy0li9oJjLZ+azeHoui6fnMbsoeyqs+Y80XiRSXG0BPuRUMMPDoUvsSpZJARuq/Hzn/ZezdmGJXTm/CMPzCwNtPbzU0MKHV81xOqSUY0WWiQkRoaosl6qyXD69fj6n+wZ4saGFXYeCVNeHePZQMxpdDJuWkUbV9FwumZ7LomjhtXh6bkp1Indb48Xh0RlF1sLBpICyPC8fWDHrwl9oLqjYl8nTB5vpHRhinY3SiTkrskxcZGems3FxGRsXRzaTnu4boO5kmINNHRxs6uRgoJMna5t49JXRveFleVkjBdfi8kjxtcCfQ2Z6cm6qV9UdwA6n4wBGhsAW+2y50Bgzqjgnk4NNnWSmeVg1z+4+jjUrskxCZGems2xWActmFYw8p6oEO3s50NTJoaYODgY6OdjUyQtHWugbjPSuSfcIC/yRW6bL871Mz/dSUTCN8ujHkpyslO/fVfN2O19//A3KC7wjm3Er8kcf53nTETn/30FoeG5h9jtrtmiMSW1F0ROvFXMLyc60kiDW7G/UOEZEKM3zUprnHXcHT//gEEdDXeOKrwOBDp4+eJKe/qFxPyPdI5TleSnP91IeLb4if6ZRURApykp8WUm9B6xvcAgE9h07xW9eDzAwNP4mRF9m2mjxVeClIj/yuLzAy4yCaUzP99La1UdBdgbp1mrDGDPG8B3HNkonPqzIMq6TkeahsiyXyrJcWFYx8ryq0na6n0B7D4H2bhrbewi0ddPU3kNjezevn2jjydqecR2cATLTPJTlZ7F1/QI+enXybexcPruQn/7lagAGh5RQuJe327oJtPXQ2NZNY3t35GNbD7WN7SNXrcZK9wizi5OzYaIxJn5GiyzbjxUPkyqyLjSDTUSygJ8AVwEtwG2qejS2oZqpTkQo9GVS6MtkyTnGPqgqrV19BNojBUikIIsUZf4UaF+QFr1yV5bnhXO05+npH4wUnm3dkWIs+nildXs3xkxw87IKBoaUJeU2SiceLlhkTXIG2yeBU6q6UES2AN8GbotHwMacj4hQnJNFcU4WS2fkOx2OI7wZaSNdrI0x5nzmlfj43A1VToeRsiazQWNkBpuq9gHDM9jGuhV4MPp4O3CdXGgnrjHGGGNMCptMkXW2GWwzzvU1qjoAtAPFE3+QiGwVkb0isjcYDL67iI0xxhhjkkBCbzVS1W2qukJVV/j9dieDMcYYY1LXZIqsycxgG/kaEUkH8olsgDfGGGOMmZImU2SNzGATkUwiM9gem/A1jwEfjz5+P/CMqirGGGOMMVPUBe8uPNcMNhH5OrBXVR8D/hN4SETqgVYihZgxxhhjzJQ1qT5ZZ5vBpqpfHfO4B/iz2IZmjDHGGJO8bMaGMcYYY0wciFNbp0QkCByL4yFKgFAcf77F8M64IY7zxTBHVR295dVyYkrFAO6Iw3LC3b8DiyHxYpoTjhVZ8SYie1V1hcXgfAxuicMNMTjJDa/fYnBXHG6IwUlueP0Wg3tiiEcctlxojDHGGBMHVmQZY4wxxsRBKhdZ25wOAIthLDfE4YYYnOSG128xjHJDHG6IwUlueP0WQ4QbYoAYx5Gye7KMMcYYY5yUyleyjDHGGGMcY0WWMcYYY0wcpFyRJSKzRORZEXlDRGpF5E4HY0kTkddE5NcOHb9ARLaLyEEROSAiqx2I4XPR30ONiDwiIt4EHfd+EWkWkZoxzxWJyP+KSF30Y2EiYnGa5cS440/JnLB8GM8tOeF0PkRjsJwYfS7mOZFyRRYwAHxBVZcAVwO3i8gSh2K5Ezjg0LEB7gaeUNXFwLJExyIiM4A7gBWqupTI7MtEzbX8MbBpwnNfAp5W1Urg6ejnU4HlxKipmhM/xvJhLLfkhNP5AJYTY8U8J1KuyFLVgKq+Gn3cSeQfzIxExyEiM4GbgPsSfezo8fOB9USGd6Oqfara5kAo6cA0EUkHsoHGRBxUVXcTGVY+1q3Ag9HHDwJ/lIhYnGY5MXL8KZsTlg/juSEnnM6HaAyWE+PFPCdSrsgaS0TmAlcCLzlw+O8DXwSGHDg2wDwgCDwQvRx9n4j4EhmAqr4NfBd4CwgA7ar6VCJjmKBMVQPRx01AmYOxOMJywnJijCmfD+BoTjidD2A5MVHMcyJliywRyQH+B7hLVTsSfOybgWZV3ZfI406QDiwH7lXVK4EuErwcEF3PvpVIIlcAPhH5SCJjOBeN9C6ZUv1LLCcsJ85lKuYDOJcTLskHsJw4p1jlREoWWSKSQSRxHlbVnzsQwjXALSJyFHgU2Cgi/5XgGE4AJ1R1+OxsO5FkSqTrgTdVNaiq/cDPgTUJjmGskyJSDhD92OxgLAllOQFYTkw0ZfMBHM8JN+QDWE5MFPOcSLkiS0SEyPryAVX9nhMxqOqXVXWmqs4lsoHvGVVNaGWuqk3AcRFZFH3qOuCNRMZA5PLv1SKSHf29XIezmzwfAz4effxx4FcOxpIwlhMjMVhOjDcl8wGczwk35EM0DsuJ8WKeEylXZBE5Q/gokTOD30X/bHY6KId8FnhYRF4HrgD+KZEHj54dbQdeBfYT+feWkNEJIvII8AKwSEROiMgngW8BN4hIHZGzp28lIhYXsJwYNSVzwvLhDJYToywn4pgTNlbHGGOMMSYOUvFKljHGGGOM46zIMsYYY4yJAyuyjDHGGGPiwIosY4wxxpg4sCLLGGOMMSYOrMgyxhhjjIkDK7KMMcYYY+Lg/wHFNzg06dNM+QAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "q_learning(evaluation=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 616
        },
        "id": "zSr47kef_-sb",
        "outputId": "28aa3722-a3fb-4b25-cbf3-ca95274a1573"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Agent 1 route:  [13, 13, 13, 9, 13, 9, 5, 9, 5, 5, 6, 10, 14, 14, 10, 6, 2]\n",
            "Agent 2 route:  [16, 16, 16, 12, 16, 12, 8, 12, 8, 8, 8, 7, 11, 15, 15, 11, 7]\n",
            "Agent 1 route:  [13, 9, 5, 6, 2, 6, 2, 2, 2, 2, 6, 2, 6, 2, 2, 2, 2]\n",
            "Agent 2 route:  [16, 12, 8, 7, 3, 7, 3, 3, 3, 3, 7, 3, 3, 3, 3, 7, 3]\n",
            "Agent 1 route:  [13, 9, 5, 6, 2, 6, 2, 6, 2, 2, 2, 2, 6, 2, 6, 6, 5]\n",
            "Agent 2 route:  [16, 12, 8, 7, 3, 7, 3, 7, 3, 3, 3, 3, 7, 3, 7, 8, 7]\n",
            "Agent 1 route:  [13, 9, 5, 6, 2, 2, 2, 3, 7, 11, 15, 15, 11, 7, 3, 3, 4]\n",
            "Agent 2 route:  [16, 12, 8, 7, 6, 10, 14, 14, 10, 6, 2, 6, 2, 2, 2, 1, 1]\n",
            "Agent 1 route:  [13, 9, 5, 6, 2, 3, 4]\n",
            "Agent 2 route:  [16, 12, 8, 7, 6, 2, 1]\n",
            "Agent 1 route:  [13, 9, 5, 6, 2, 3, 4]\n",
            "Agent 2 route:  [16, 12, 8, 7, 6, 2, 1]\n",
            "Agent 1 route:  [13, 9, 5, 6, 2, 3, 4]\n",
            "Agent 2 route:  [16, 12, 8, 7, 6, 2, 1]\n",
            "Agent 1 route:  [13, 9, 5, 6, 2, 3, 4]\n",
            "Agent 2 route:  [16, 12, 8, 7, 6, 2, 1]\n",
            "Agent 1 route:  [13, 9, 5, 6, 2, 3, 4]\n",
            "Agent 2 route:  [16, 12, 8, 7, 6, 2, 1]\n",
            "Agent 1 route:  [13, 9, 5, 6, 2, 3, 4]\n",
            "Agent 2 route:  [16, 12, 8, 7, 6, 2, 1]\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQcAAAD8CAYAAAB6iWHJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAMzUlEQVR4nO3df+xddX3H8edrpZYxRH6Z0JUKKozMHwykqRiShYBkQAxdImbwh4KBdDEycdkfM1vCMv8R94cmBsPSABkYoxhQ1pkSUgJGzQajNKWDMrTjH9o1QwsUGxRX8t4f99R9+fr5ttB77rn3y/f5SG56fnx6358byKvnnnPueaeqkKT5fmfaE5A0mwwHSU2Gg6Qmw0FSk+EgqclwkNQ0VjgkOTHJ5iQ/7f48YYFxryXZ1r02jlNT0jAyzn0OSf4BeKGqbk7yBeCEqvrrxrj9VXXsGPOUNLBxw+EZ4MKq2pNkJfCDqjqrMc5wkBaZccPhpao6vlsO8OLB9XnjDgDbgAPAzVV13wLvtx5YD3D0MTnvXe952xHPbVbtefKYaU9Bb9IfnP3KtKcwMY9vf/XnVfXO1r7DhkOSB4FTGrv+FrhzbhgkebGqfuu8Q5JVVbU7yXuAh4CLq+q/DlX3rA8eXbduPO2Qc1uMvvTes6c9Bb1JD/z3tmlPYWKWrdz5eFWtae076nB/uao+utC+JP+TZOWcrxXPL/Aeu7s/n03yA+Bc4JDhIGm6xr2UuRG4plu+Bvjn+QOSnJBkRbd8MnABsGPMupImbNxwuBm4JMlPgY926yRZk+S2bswfAluSPAE8zOicg+EgzbjDfq04lKraC1zc2L4FuL5b/lfgg+PUkTQ875CU1GQ4SGoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUZDhIauolHJJcmuSZJDu7zlfz969Icne3/9Ekp/dRV9LkjB0OSZYBXwcuA94HXJ3kffOGXceo4c0ZwFeBL49bV9Jk9XHksBbYWVXPVtWvgW8D6+aNWQfc2S3fA1zcdciSNKP6CIdVwHNz1nd125pjquoAsA84qYfakiZkpk5IJlmfZEuSLS+98Nq0pyMtaX2Ew25g9Zz1U7ttzTFJjgLeAeyd/0ZVtaGq1lTVmuNPXNbD1CQdqT7C4THgzCTvTvI24CpGbfLmmts270rgoRqnvbekiRur4xWMziEkuQF4AFgG3FFVTyX5IrClqjYCtwPfSLITeIFRgEiaYWOHA0BVbQI2zdt205zlXwGf6KOWpGHM1AlJSbPDcJDUZDhIajIcJDUZDpKaDAdJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqGqpX5rVJfpZkW/e6vo+6kiZn7AfMzumVeQmjblePJdlYVTvmDb27qm4Yt56kYfTx9Onf9MoESHKwV+b8cHhT9jx5DF9679k9TE8az5/8/jnTnsIE7Vxwz1C9MgE+nmR7knuSrG7sf107vP/l1R6mJulIDXVC8l+A06vqbGAz/99x+3XmtsNbzoqBpiapZZBemVW1t6oOHgrcBpzXQ11JEzRIr8wkK+esXgE83UNdSRM0VK/MzyW5AjjAqFfmtePWlTRZmdVm18flxPpwLp72NKS3tAfrnserak1rn3dISmoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUZDhIajIcJDX11Q7vjiTPJ3lygf1J8rWuXd72JB/qo66kyenryOGfgEsPsf8y4MzutR64tae6kiakl3Coqh8yeqr0QtYBd9XII8Dx8x5XL2nGDHXO4Q21zLMdnjQ7ZuqEpO3wpNkxVDgctmWepNkyVDhsBD7VXbU4H9hXVXsGqi3pCIzdDg8gybeAC4GTk+wC/g5YDlBV/whsAi4HdgKvAJ/uo66kyeklHKrq6sPsL+CzfdSSNIyZOiEpaXYYDpKaDAdJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNRkOkpqGaod3YZJ9SbZ1r5v6qCtpcnp5hiSjdni3AHcdYsyPqupjPdWTNGFDtcOTtMgMec7hI0meSHJ/kve3BtgOT5odfX2tOJytwGlVtT/J5cB9jDpuv05VbQA2AByXE2uguUlqGOTIoaperqr93fImYHmSk4eoLenIDBIOSU5Jkm55bVd37xC1JR2ZodrhXQl8JskB4JfAVV0XLEkzaqh2eLcwutQpaZHwDklJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNRkOkpoMB0lNhoOkprHDIcnqJA8n2ZHkqSQ3NsYkydeS7EyyPcmHxq0rabL6eIbkAeCvqmprkrcDjyfZXFU75oy5jFGfijOBDwO3dn9KmlFjHzlU1Z6q2tot/wJ4Glg1b9g64K4aeQQ4PsnKcWtLmpxezzkkOR04F3h03q5VwHNz1nfx2wFiOzxphvQWDkmOBe4FPl9VLx/Je1TVhqpaU1VrlrOir6lJOgK9hEOS5YyC4ZtV9d3GkN3A6jnrp3bbJM2oPq5WBLgdeLqqvrLAsI3Ap7qrFucD+6pqz7i1JU1OH1crLgA+CfxHkm3dtr8B3gW/aYe3Cbgc2Am8Any6h7qSJmjscKiqHwM5zJgCPjtuLUnD8Q5JSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUZDhIajIcJDUZDpKaDAdJTYaDpKah2uFdmGRfkm3d66Zx60qarKHa4QH8qKo+1kM9SQMYqh2epEVmqHZ4AB9J8kSS+5O8f4G/bzs8aUb08bUCOGw7vK3AaVW1P8nlwH2MOm6/TlVtADYAHJcTq6+5SXrzBmmHV1UvV9X+bnkTsDzJyX3UljQZg7TDS3JKN44ka7u6e8etLWlyhmqHdyXwmSQHgF8CV3VdsCTNqKHa4d0C3DJuLUnD8Q5JSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUZDhIajIcJDUZDpKaDAdJTYaDpKY+HjB7dJJ/73pSPJXk7xtjViS5O8nOJI92/S0kzbA+jhxeBS6qqj8CzgEuTXL+vDHXAS9W1RnAV4Ev91BX0gT10Q6vDvakAJZ3r/lPll4H3Nkt3wNcfPBR9ZJmU19NbZZ1j6V/HthcVfPb4a0CngOoqgPAPuCkPmpLmoxewqGqXquqc4BTgbVJPnAk72OvTGl29Hq1oqpeAh4GLp23azewGiDJUcA7aHS8qqoNVbWmqtYsZ0WfU5P0JvVxteKdSY7vln8XuAT4z3nDNgLXdMtXAg/Z8UqabX20w1sJ3JlkGaOw+U5VfT/JF4EtVbWRUS/NbyTZCbwAXNVDXUkT1Ec7vO3AuY3tN81Z/hXwiXFrSRqOd0hKajIcJDUZDpKaDAdJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNRkOkpqG6pV5bZKfJdnWva4ft66kyerj6dMHe2XuT7Ic+HGS+6vqkXnj7q6qG3qoJ2kAfTx9uoDD9cqUtMj0ceRA17PiceAM4OuNXpkAH0/yx8BPgL+squca77MeWN+t7n+w7nmmj/m9QScDPx+w3lD8XIvPkJ/ttIV2pM/GU13nq+8Bf1FVT87ZfhKwv6peTfLnwJ9V1UW9Fe5Bki1VtWba8+ibn2vxmZXPNkivzKraW1UHO+PeBpzXZ11J/RukV2aSlXNWrwCeHreupMkaqlfm55JcARxg1Cvz2h7q9m3DtCcwIX6uxWcmPluv5xwkvXV4h6SkJsNBUtOSD4cklyZ5JsnOJF+Y9nz6kuSOJM8nefLwoxePJKuTPJxkR3e7/o3TnlMf3sjPEAaf01I+59CdRP0Joyssu4DHgKurasdUJ9aD7oaz/cBdVfWBac+nL92Vr5VVtTXJ2xndfPeni/2/WZIAvzf3ZwjAjY2fIQxmqR85rAV2VtWzVfVr4NvAuinPqRdV9UNGV4beUqpqT1Vt7ZZ/weiy+Krpzmp8NTJTP0NY6uGwCph7G/cu3gL/oy0VSU4HzgVat+svOkmWJdkGPA9sXuBnCINZ6uGgRSrJscC9wOer6uVpz6cPVfVaVZ0DnAqsTTLVr4NLPRx2A6vnrJ/abdMM676T3wt8s6q+O+359G2hnyEMbamHw2PAmUneneRtwFXAxinPSYfQnbi7HXi6qr4y7fn05Y38DGFoSzocquoAcAPwAKMTW9+pqqemO6t+JPkW8G/AWUl2Jblu2nPqyQXAJ4GL5jxZ7PJpT6oHK4GHk2xn9I/W5qr6/jQntKQvZUpa2JI+cpC0MMNBUpPhIKnJcJDUZDhIajIcJDUZDpKa/g+9R//cPUXu4gAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#2. MADPPG on Simple Adversary"
      ],
      "metadata": {
        "id": "KWQiEj5Z-jpv"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Networks"
      ],
      "metadata": {
        "id": "O1HBs0Mc67Cf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "class MultiAgentReplayBuffer:\n",
        "    def __init__(self, max_size, critic_dims, actor_dims, \n",
        "            n_actions, n_agents, batch_size):\n",
        "        self.mem_size = max_size\n",
        "        self.mem_cntr = 0\n",
        "        self.n_agents = n_agents\n",
        "        self.actor_dims = actor_dims\n",
        "        self.batch_size = batch_size\n",
        "        self.n_actions = n_actions\n",
        "\n",
        "        self.state_memory = np.zeros((self.mem_size, critic_dims))\n",
        "        self.new_state_memory = np.zeros((self.mem_size, critic_dims))\n",
        "        self.reward_memory = np.zeros((self.mem_size, n_agents))\n",
        "        self.terminal_memory = np.zeros((self.mem_size, n_agents), dtype=bool)\n",
        "\n",
        "        self.init_actor_memory()\n",
        "\n",
        "    def init_actor_memory(self):\n",
        "        self.actor_state_memory = []\n",
        "        self.actor_new_state_memory = []\n",
        "        self.actor_action_memory = []\n",
        "\n",
        "        for i in range(self.n_agents):\n",
        "            self.actor_state_memory.append(\n",
        "                            np.zeros((self.mem_size, self.actor_dims[i])))\n",
        "            self.actor_new_state_memory.append(\n",
        "                            np.zeros((self.mem_size, self.actor_dims[i])))\n",
        "            self.actor_action_memory.append(\n",
        "                            np.zeros((self.mem_size, self.n_actions)))\n",
        "\n",
        "\n",
        "    def store_transition(self, raw_obs, state, action, reward, \n",
        "                               raw_obs_, state_, done):\n",
        "        # this introduces a bug: if we fill up the memory capacity and then\n",
        "        # zero out our actor memory, the critic will still have memories to access\n",
        "        # while the actor will have nothing but zeros to sample. Obviously\n",
        "        # not what we intend.\n",
        "        # In reality, there's no problem with just using the same index\n",
        "        # for both the actor and critic states. I'm not sure why I thought\n",
        "        # this was necessary in the first place. Sorry for the confusion!\n",
        "\n",
        "        #if self.mem_cntr % self.mem_size == 0 and self.mem_cntr > 0:\n",
        "        #    self.init_actor_memory()\n",
        "        \n",
        "        index = self.mem_cntr % self.mem_size\n",
        "\n",
        "        for agent_idx in range(self.n_agents):\n",
        "            self.actor_state_memory[agent_idx][index] = raw_obs[agent_idx]\n",
        "            self.actor_new_state_memory[agent_idx][index] = raw_obs_[agent_idx]\n",
        "            self.actor_action_memory[agent_idx][index] = action[agent_idx]\n",
        "\n",
        "        self.state_memory[index] = state\n",
        "        self.new_state_memory[index] = state_\n",
        "        self.reward_memory[index] = reward\n",
        "        self.terminal_memory[index] = done\n",
        "        self.mem_cntr += 1\n",
        "\n",
        "    def sample_buffer(self):\n",
        "        max_mem = min(self.mem_cntr, self.mem_size)\n",
        "\n",
        "        batch = np.random.choice(max_mem, self.batch_size, replace=False)\n",
        "\n",
        "        states = self.state_memory[batch]\n",
        "        rewards = self.reward_memory[batch]\n",
        "        states_ = self.new_state_memory[batch]\n",
        "        terminal = self.terminal_memory[batch]\n",
        "\n",
        "        actor_states = []\n",
        "        actor_new_states = []\n",
        "        actions = []\n",
        "        for agent_idx in range(self.n_agents):\n",
        "            actor_states.append(self.actor_state_memory[agent_idx][batch])\n",
        "            actor_new_states.append(self.actor_new_state_memory[agent_idx][batch])\n",
        "            actions.append(self.actor_action_memory[agent_idx][batch])\n",
        "\n",
        "        return actor_states, states, actions, rewards, \\\n",
        "               actor_new_states, states_, terminal\n",
        "\n",
        "    def ready(self):\n",
        "        if self.mem_cntr >= self.batch_size:\n",
        "            return True\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "S4-7bZPI70lC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "class CriticNetwork(nn.Module):\n",
        "    def __init__(self, beta, input_dims, fc1_dims, fc2_dims, \n",
        "                    n_agents, n_actions, name, chkpt_dir):\n",
        "        super(CriticNetwork, self).__init__()\n",
        "\n",
        "        self.chkpt_file = os.path.join(chkpt_dir, name)\n",
        "\n",
        "        self.fc1 = nn.Linear(input_dims+n_agents*n_actions, fc1_dims)\n",
        "        self.fc2 = nn.Linear(fc1_dims, fc2_dims)\n",
        "        self.q = nn.Linear(fc2_dims, 1)\n",
        "\n",
        "        self.optimizer = optim.Adam(self.parameters(), lr=beta)\n",
        "        self.device = T.device('cuda:0' if T.cuda.is_available() else 'cpu')\n",
        " \n",
        "        self.to(self.device)\n",
        "\n",
        "    def forward(self, state, action):\n",
        "        x = F.relu(self.fc1(T.cat([state, action], dim=1)))\n",
        "        x = F.relu(self.fc2(x))\n",
        "        q = self.q(x)\n",
        "\n",
        "        return q\n",
        "\n",
        "    def save_checkpoint(self):\n",
        "        T.save(self.state_dict(), self.chkpt_file)\n",
        "\n",
        "    def load_checkpoint(self):\n",
        "        self.load_state_dict(T.load(self.chkpt_file))\n",
        "\n",
        "\n",
        "class ActorNetwork(nn.Module):\n",
        "    def __init__(self, alpha, input_dims, fc1_dims, fc2_dims, \n",
        "                 n_actions, name, chkpt_dir):\n",
        "        super(ActorNetwork, self).__init__()\n",
        "\n",
        "        self.chkpt_file = os.path.join(chkpt_dir, name)\n",
        "\n",
        "        self.fc1 = nn.Linear(input_dims, fc1_dims)\n",
        "        self.fc2 = nn.Linear(fc1_dims, fc2_dims)\n",
        "        self.pi = nn.Linear(fc2_dims, n_actions)\n",
        "\n",
        "        self.optimizer = optim.Adam(self.parameters(), lr=alpha)\n",
        "        self.device = T.device('cuda:0' if T.cuda.is_available() else 'cpu')\n",
        " \n",
        "        self.to(self.device)\n",
        "\n",
        "    def forward(self, state):\n",
        "        x = F.relu(self.fc1(state))\n",
        "        x = F.relu(self.fc2(x))\n",
        "        pi = T.softmax(self.pi(x), dim=1)\n",
        "\n",
        "        return pi\n",
        "\n",
        "    def save_checkpoint(self):\n",
        "        T.save(self.state_dict(), self.chkpt_file)\n",
        "\n",
        "    def load_checkpoint(self):\n",
        "        self.load_state_dict(T.load(self.chkpt_file))"
      ],
      "metadata": {
        "id": "QTiyoPXx8G6f"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "class Agent:\n",
        "    def __init__(self, actor_dims, critic_dims, n_actions, n_agents, agent_idx, chkpt_dir,\n",
        "                    alpha=0.01, beta=0.01, fc1=64, \n",
        "                    fc2=64, gamma=0.95, tau=0.01):\n",
        "        self.gamma = gamma\n",
        "        self.tau = tau\n",
        "        self.n_actions = n_actions\n",
        "        self.agent_name = 'agent_%s' % agent_idx\n",
        "        self.actor = ActorNetwork(alpha, actor_dims, fc1, fc2, n_actions, \n",
        "                                  chkpt_dir=chkpt_dir,  name=self.agent_name+'_actor')\n",
        "        self.critic = CriticNetwork(beta, critic_dims, \n",
        "                            fc1, fc2, n_agents, n_actions, \n",
        "                            chkpt_dir=chkpt_dir, name=self.agent_name+'_critic')\n",
        "        self.target_actor = ActorNetwork(alpha, actor_dims, fc1, fc2, n_actions,\n",
        "                                        chkpt_dir=chkpt_dir, \n",
        "                                        name=self.agent_name+'_target_actor')\n",
        "        self.target_critic = CriticNetwork(beta, critic_dims, \n",
        "                                            fc1, fc2, n_agents, n_actions,\n",
        "                                            chkpt_dir=chkpt_dir,\n",
        "                                            name=self.agent_name+'_target_critic')\n",
        "\n",
        "        self.update_network_parameters(tau=1)\n",
        "\n",
        "    def choose_action(self, observation):\n",
        "        state = T.tensor([observation], dtype=T.float).to(self.actor.device)\n",
        "        actions = self.actor.forward(state)\n",
        "        noise = T.rand(self.n_actions).to(self.actor.device)\n",
        "        action = actions + noise\n",
        "\n",
        "        return action.detach().cpu().numpy()[0]\n",
        "\n",
        "    def update_network_parameters(self, tau=None):\n",
        "        if tau is None:\n",
        "            tau = self.tau\n",
        "\n",
        "        target_actor_params = self.target_actor.named_parameters()\n",
        "        actor_params = self.actor.named_parameters()\n",
        "\n",
        "        target_actor_state_dict = dict(target_actor_params)\n",
        "        actor_state_dict = dict(actor_params)\n",
        "        for name in actor_state_dict:\n",
        "            actor_state_dict[name] = tau*actor_state_dict[name].clone() + \\\n",
        "                    (1-tau)*target_actor_state_dict[name].clone()\n",
        "\n",
        "        self.target_actor.load_state_dict(actor_state_dict)\n",
        "\n",
        "        target_critic_params = self.target_critic.named_parameters()\n",
        "        critic_params = self.critic.named_parameters()\n",
        "\n",
        "        target_critic_state_dict = dict(target_critic_params)\n",
        "        critic_state_dict = dict(critic_params)\n",
        "        for name in critic_state_dict:\n",
        "            critic_state_dict[name] = tau*critic_state_dict[name].clone() + \\\n",
        "                    (1-tau)*target_critic_state_dict[name].clone()\n",
        "\n",
        "        self.target_critic.load_state_dict(critic_state_dict)\n",
        "\n",
        "    def save_models(self):\n",
        "        self.actor.save_checkpoint()\n",
        "        self.target_actor.save_checkpoint()\n",
        "        self.critic.save_checkpoint()\n",
        "        self.target_critic.save_checkpoint()\n",
        "\n",
        "    def load_models(self):\n",
        "        self.actor.load_checkpoint()\n",
        "        self.target_actor.load_checkpoint()\n",
        "        self.critic.load_checkpoint()\n",
        "        self.target_critic.load_checkpoint()"
      ],
      "metadata": {
        "id": "N1uGQxGy8HnQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class MADDPG:\n",
        "    def __init__(self, actor_dims, critic_dims, n_agents, n_actions, \n",
        "                 scenario='simple',  alpha=0.01, beta=0.01, fc1=64, \n",
        "                 fc2=64, gamma=0.99, tau=0.01, chkpt_dir='tmp/maddpg/'):\n",
        "        self.agents = []\n",
        "        self.n_agents = n_agents\n",
        "        self.n_actions = n_actions\n",
        "        chkpt_dir += scenario \n",
        "        for agent_idx in range(self.n_agents):\n",
        "            self.agents.append(Agent(actor_dims[agent_idx], critic_dims,  \n",
        "                            n_actions, n_agents, agent_idx, alpha=alpha, beta=beta,\n",
        "                            chkpt_dir=chkpt_dir))\n",
        "\n",
        "\n",
        "    def save_checkpoint(self):\n",
        "        print('... saving checkpoint ...')\n",
        "        for agent in self.agents:\n",
        "            agent.save_models()\n",
        "\n",
        "    def load_checkpoint(self):\n",
        "        print('... loading checkpoint ...')\n",
        "        for agent in self.agents:\n",
        "            agent.load_models()\n",
        "\n",
        "    def choose_action(self, raw_obs):\n",
        "        actions = []\n",
        "        for agent_idx, agent in enumerate(self.agents):\n",
        "            action = agent.choose_action(raw_obs[agent_idx])\n",
        "            actions.append(action)\n",
        "        return actions\n",
        "\n",
        "    def learn(self, memory):\n",
        "        if not memory.ready():\n",
        "            return\n",
        "\n",
        "        actor_states, states, actions, rewards, \\\n",
        "        actor_new_states, states_, dones = memory.sample_buffer()\n",
        "\n",
        "        device = self.agents[0].actor.device\n",
        "\n",
        "        states = T.tensor(states, dtype=T.float).to(device)\n",
        "        actions = T.tensor(actions, dtype=T.float).to(device)\n",
        "        rewards = T.tensor(rewards, dtype=T.float).to(device)\n",
        "        states_ = T.tensor(states_, dtype=T.float).to(device)\n",
        "        dones = T.tensor(dones).to(device)\n",
        "\n",
        "        all_agents_new_actions = []\n",
        "        all_agents_new_mu_actions = []\n",
        "        old_agents_actions = []\n",
        "\n",
        "        for agent_idx, agent in enumerate(self.agents):\n",
        "            new_states = T.tensor(actor_new_states[agent_idx], \n",
        "                                 dtype=T.float).to(device)\n",
        "\n",
        "            new_pi = agent.target_actor.forward(new_states)\n",
        "\n",
        "            all_agents_new_actions.append(new_pi)\n",
        "            mu_states = T.tensor(actor_states[agent_idx], \n",
        "                                 dtype=T.float).to(device)\n",
        "            pi = agent.actor.forward(mu_states)\n",
        "            all_agents_new_mu_actions.append(pi)\n",
        "            old_agents_actions.append(actions[agent_idx])\n",
        "\n",
        "        new_actions = T.cat([acts for acts in all_agents_new_actions], dim=1)\n",
        "        mu = T.cat([acts for acts in all_agents_new_mu_actions], dim=1)\n",
        "        old_actions = T.cat([acts for acts in old_agents_actions],dim=1)\n",
        "\n",
        "        for agent_idx, agent in enumerate(self.agents):\n",
        "            critic_value_ = agent.target_critic.forward(states_, new_actions).flatten()\n",
        "            critic_value_[dones[:,0]] = 0.0\n",
        "            critic_value = agent.critic.forward(states, old_actions).flatten()\n",
        "\n",
        "            target = rewards[:,agent_idx] + agent.gamma*critic_value_\n",
        "            critic_loss = F.mse_loss(target, critic_value)\n",
        "            agent.critic.optimizer.zero_grad()\n",
        "            critic_loss.backward(retain_graph=True)\n",
        "            agent.critic.optimizer.step()\n",
        "\n",
        "            actor_loss = agent.critic.forward(states, mu).flatten()\n",
        "            actor_loss = -T.mean(actor_loss)\n",
        "            agent.actor.optimizer.zero_grad()\n",
        "            actor_loss.backward(retain_graph=True)\n",
        "            agent.actor.optimizer.step()\n",
        "\n",
        "            agent.update_network_parameters()"
      ],
      "metadata": {
        "id": "RishMQYP8QDl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Main"
      ],
      "metadata": {
        "id": "QUZ_0UD164Ga"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from make_env import make_env\n",
        "\n",
        "def obs_list_to_state_vector(observation):\n",
        "    state = np.array([])\n",
        "    for obs in observation:\n",
        "        state = np.concatenate([state, obs])\n",
        "    return state\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    #scenario = 'simple'\n",
        "    scenario = 'simple_adversary'\n",
        "    env = make_env(scenario)\n",
        "    n_agents = env.n\n",
        "    actor_dims = []\n",
        "\n",
        "    print(\"Observatio space shape: \", env.observation_space)\n",
        "    \n",
        "    print(\"Action space: \", env.action_space)\n",
        "    print(\"Action space shape: \", env.action_space[0].n)\n",
        "\n",
        "\n",
        "    for i in range(n_agents):\n",
        "        actor_dims.append(env.observation_space[i].shape[0])\n",
        "        print(\"Observatio space shape: \", env.observation_space[i].shape[0])\n",
        "    print(\"Actor dims: \", actor_dims)\n",
        "    critic_dims = sum(actor_dims)\n",
        "\n",
        "    # action space is a list of arrays, assume each agent has same action space\n",
        "    n_actions = env.action_space[0].n\n",
        "    maddpg_agents = MADDPG(actor_dims, critic_dims, n_agents, n_actions, \n",
        "                           fc1=64, fc2=64,  \n",
        "                           alpha=0.01, beta=0.01, scenario=scenario,\n",
        "                           chkpt_dir='tmp/maddpg/')\n",
        "\n",
        "    memory = MultiAgentReplayBuffer(1000000, critic_dims, actor_dims, \n",
        "                        n_actions, n_agents, batch_size=1024)\n",
        "\n",
        "    PRINT_INTERVAL = 1\n",
        "    N_GAMES = 10000\n",
        "    MAX_STEPS = 25\n",
        "    total_steps = 0\n",
        "    score_history = []\n",
        "    evaluate = False\n",
        "    best_score = 0\n",
        "\n",
        "    if evaluate:\n",
        "        maddpg_agents.load_checkpoint()\n",
        "\n",
        "    for i in range(1, N_GAMES+1):\n",
        "        obs = env.reset()\n",
        "        score = 0\n",
        "        done = [False]*n_agents\n",
        "        episode_step = 0\n",
        "        while not any(done):\n",
        "            if evaluate:\n",
        "                env.render()\n",
        "                #time.sleep(0.1) # to slow down the action for the video\n",
        "            # print(\"Obs: \", obs)\n",
        "            actions = maddpg_agents.choose_action(obs)\n",
        "            # print(\"Actions: \", actions)\n",
        "            obs_, reward, done, info = env.step(actions)\n",
        "\n",
        "            state = obs_list_to_state_vector(obs)\n",
        "            state_ = obs_list_to_state_vector(obs_)\n",
        "\n",
        "            if episode_step >= MAX_STEPS:\n",
        "                done = [True]*n_agents\n",
        "\n",
        "            memory.store_transition(obs, state, actions, reward, obs_, state_, done)\n",
        "\n",
        "            if total_steps % 100 == 0 and not evaluate:\n",
        "                maddpg_agents.learn(memory)\n",
        "\n",
        "            obs = obs_\n",
        "\n",
        "            score += sum(reward)\n",
        "            total_steps += 1\n",
        "            episode_step += 1\n",
        "\n",
        "        score_history.append(score)\n",
        "        avg_score = np.mean(score_history[-100:])\n",
        "        if not evaluate:\n",
        "            if avg_score > best_score:\n",
        "                # maddpg_agents.save_checkpoint()\n",
        "                best_score = avg_score\n",
        "        if i % 50 == 0 and i > 0:\n",
        "            print('episode', i, 'average score {:.1f}'.format(avg_score))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9GjyKDh68Vqy",
        "outputId": "be4e9cf0-9c09-46ff-a614-a9d007b03633"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Observatio space shape:  [Box(8,), Box(10,), Box(10,)]\n",
            "Action space:  [Discrete(5), Discrete(5), Discrete(5)]\n",
            "Action space shape:  5\n",
            "Observatio space shape:  8\n",
            "Observatio space shape:  10\n",
            "Observatio space shape:  10\n",
            "Actor dims:  [8, 10, 10]\n",
            "episode 50 average score -20.9\n",
            "episode 100 average score -45.7\n",
            "episode 150 average score -61.5\n",
            "episode 200 average score -44.7\n",
            "episode 250 average score -33.2\n",
            "episode 300 average score -33.2\n",
            "episode 350 average score -34.5\n",
            "episode 400 average score -31.9\n",
            "episode 450 average score -35.3\n",
            "episode 500 average score -32.6\n",
            "episode 550 average score -29.9\n",
            "episode 600 average score -31.7\n",
            "episode 650 average score -30.3\n",
            "episode 700 average score -29.5\n",
            "episode 750 average score -29.3\n",
            "episode 800 average score -31.0\n",
            "episode 850 average score -31.3\n",
            "episode 900 average score -35.0\n",
            "episode 950 average score -36.7\n",
            "episode 1000 average score -37.7\n",
            "episode 1050 average score -37.5\n",
            "episode 1100 average score -31.5\n",
            "episode 1150 average score -31.2\n",
            "episode 1200 average score -31.8\n",
            "episode 1250 average score -33.9\n",
            "episode 1300 average score -36.0\n",
            "episode 1350 average score -35.8\n",
            "episode 1400 average score -34.4\n",
            "episode 1450 average score -31.2\n",
            "episode 1500 average score -30.6\n",
            "episode 1550 average score -33.0\n",
            "episode 1600 average score -33.9\n",
            "episode 1650 average score -31.8\n",
            "episode 1700 average score -30.4\n",
            "episode 1750 average score -30.1\n",
            "episode 1800 average score -31.6\n",
            "episode 1850 average score -33.3\n",
            "episode 1900 average score -34.1\n",
            "episode 1950 average score -31.7\n",
            "episode 2000 average score -29.8\n",
            "episode 2050 average score -31.9\n",
            "episode 2100 average score -34.4\n",
            "episode 2150 average score -35.5\n",
            "episode 2200 average score -33.2\n",
            "episode 2250 average score -31.6\n",
            "episode 2300 average score -33.0\n",
            "episode 2350 average score -30.5\n",
            "episode 2400 average score -29.8\n",
            "episode 2450 average score -31.9\n",
            "episode 2500 average score -31.2\n",
            "episode 2550 average score -31.4\n",
            "episode 2600 average score -30.7\n",
            "episode 2650 average score -32.4\n",
            "episode 2700 average score -33.5\n",
            "episode 2750 average score -34.3\n",
            "episode 2800 average score -35.7\n",
            "episode 2850 average score -34.1\n",
            "episode 2900 average score -34.8\n",
            "episode 2950 average score -31.8\n",
            "episode 3000 average score -29.1\n",
            "episode 3050 average score -34.6\n",
            "episode 3100 average score -36.3\n",
            "episode 3150 average score -36.5\n",
            "episode 3200 average score -36.1\n",
            "episode 3250 average score -30.2\n",
            "episode 3300 average score -30.7\n",
            "episode 3350 average score -31.4\n",
            "episode 3400 average score -29.4\n",
            "episode 3450 average score -33.5\n",
            "episode 3500 average score -35.3\n",
            "episode 3550 average score -34.4\n",
            "episode 3600 average score -32.3\n",
            "episode 3650 average score -30.8\n",
            "episode 3700 average score -32.2\n",
            "episode 3750 average score -32.8\n",
            "episode 3800 average score -33.9\n",
            "episode 3850 average score -30.0\n",
            "episode 3900 average score -30.2\n",
            "episode 3950 average score -33.3\n",
            "episode 4000 average score -28.4\n",
            "episode 4050 average score -30.9\n",
            "episode 4100 average score -33.4\n",
            "episode 4150 average score -32.1\n",
            "episode 4200 average score -31.5\n",
            "episode 4250 average score -29.9\n",
            "episode 4300 average score -31.5\n",
            "episode 4350 average score -32.2\n",
            "episode 4400 average score -28.7\n",
            "episode 4450 average score -28.4\n",
            "episode 4500 average score -33.9\n",
            "episode 4550 average score -33.5\n",
            "episode 4600 average score -29.5\n",
            "episode 4650 average score -29.7\n",
            "episode 4700 average score -33.1\n",
            "episode 4750 average score -34.7\n",
            "episode 4800 average score -32.6\n",
            "episode 4850 average score -33.0\n",
            "episode 4900 average score -35.7\n",
            "episode 4950 average score -33.9\n",
            "episode 5000 average score -34.4\n",
            "episode 5050 average score -32.5\n",
            "episode 5100 average score -29.9\n",
            "episode 5150 average score -32.2\n",
            "episode 5200 average score -32.3\n",
            "episode 5250 average score -31.4\n",
            "episode 5300 average score -33.0\n",
            "episode 5350 average score -33.8\n",
            "episode 5400 average score -34.1\n",
            "episode 5450 average score -33.9\n",
            "episode 5500 average score -31.9\n",
            "episode 5550 average score -30.1\n",
            "episode 5600 average score -32.8\n",
            "episode 5650 average score -36.2\n",
            "episode 5700 average score -38.2\n",
            "episode 5750 average score -36.1\n",
            "episode 5800 average score -30.7\n",
            "episode 5850 average score -31.2\n",
            "episode 5900 average score -32.9\n",
            "episode 5950 average score -33.1\n",
            "episode 6000 average score -32.5\n",
            "episode 6050 average score -30.0\n",
            "episode 6100 average score -27.5\n",
            "episode 6150 average score -27.0\n",
            "episode 6200 average score -25.8\n",
            "episode 6250 average score -23.8\n",
            "episode 6300 average score -34.8\n",
            "episode 6350 average score -48.7\n",
            "episode 6400 average score -29.7\n",
            "episode 6450 average score -7.4\n",
            "episode 6500 average score -6.1\n",
            "episode 6550 average score -9.2\n",
            "episode 6600 average score -10.5\n",
            "episode 6650 average score -10.0\n",
            "episode 6700 average score -10.0\n",
            "episode 6750 average score -10.8\n",
            "episode 6800 average score -9.6\n",
            "episode 6850 average score -7.4\n",
            "episode 6900 average score -9.3\n",
            "episode 6950 average score -12.2\n",
            "episode 7000 average score -11.0\n",
            "episode 7050 average score -10.5\n",
            "episode 7100 average score -11.2\n",
            "episode 7150 average score -10.2\n",
            "episode 7200 average score -9.2\n",
            "episode 7250 average score -9.7\n",
            "episode 7300 average score -10.4\n",
            "episode 7350 average score -8.1\n",
            "episode 7400 average score -8.0\n",
            "episode 7450 average score -7.6\n",
            "episode 7500 average score -8.2\n",
            "episode 7550 average score -8.1\n",
            "episode 7600 average score -7.5\n",
            "episode 7650 average score -10.1\n",
            "episode 7700 average score -8.4\n",
            "episode 7750 average score -9.8\n",
            "episode 7800 average score -12.0\n",
            "episode 7850 average score -12.4\n",
            "episode 7900 average score -10.4\n",
            "episode 7950 average score -4.6\n",
            "episode 8000 average score -8.1\n",
            "episode 8050 average score -14.0\n",
            "episode 8100 average score -11.9\n",
            "episode 8150 average score -9.6\n",
            "episode 8200 average score -9.8\n",
            "episode 8250 average score -7.5\n",
            "episode 8300 average score -6.0\n",
            "episode 8350 average score -4.6\n",
            "episode 8400 average score -4.1\n",
            "episode 8450 average score -4.0\n",
            "episode 8500 average score -3.2\n",
            "episode 8550 average score -6.1\n",
            "episode 8600 average score -4.7\n",
            "episode 8650 average score -3.3\n",
            "episode 8700 average score -5.6\n",
            "episode 8750 average score -6.9\n",
            "episode 8800 average score -8.1\n",
            "episode 8850 average score -6.2\n",
            "episode 8900 average score -3.2\n",
            "episode 8950 average score -3.2\n",
            "episode 9000 average score -2.2\n",
            "episode 9050 average score 2.0\n",
            "episode 9100 average score 2.5\n",
            "episode 9150 average score 1.8\n",
            "episode 9200 average score 2.4\n",
            "episode 9250 average score 1.5\n",
            "episode 9300 average score 2.2\n",
            "episode 9350 average score 3.1\n",
            "episode 9400 average score 3.3\n",
            "episode 9450 average score 2.3\n",
            "episode 9500 average score 2.6\n",
            "episode 9550 average score 4.1\n",
            "episode 9600 average score 4.0\n",
            "episode 9650 average score 3.6\n",
            "episode 9700 average score 3.4\n",
            "episode 9750 average score 4.0\n",
            "episode 9800 average score 3.2\n",
            "episode 9850 average score 1.0\n",
            "episode 9900 average score 0.8\n",
            "episode 9950 average score 2.4\n",
            "episode 10000 average score 3.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "np.shape(score_history)\n",
        "x = [e for e in range(10000)]\n",
        "yr = score_history\n",
        "plt.plot(x,yr)\n",
        "plt.xlabel('Episodes')\n",
        "plt.ylabel('Rewards')\n",
        "plt.title('MADDPG on Simple Adversary')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 312
        },
        "id": "A8IXckd0VJBu",
        "outputId": "17263565-5933-4b35-9bcc-63ee1c1ab895"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'MADDPG on Simple Adversary')"
            ]
          },
          "metadata": {},
          "execution_count": 88
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3wUZf7A8c83Cb136aF3BAUFDxAEleKdvZ69nR6e5Tx/ghXPht6pJx72XpBDsSNFQRAREFCkSSdA6D3UGJLn98fMJrub2d2ZzW52k3zfr9fCZmZ29pmd3ec7T5nnEWMMSimllFspiU6AUkqpkkUDh1JKKU80cCillPJEA4dSSilPNHAopZTyRAOHUkopTzRwKBVHIvJnEZkWp32/LSKPxWPfDu81U0RuLI73UslPA4cqREQyROR3EakbtPwXETEikh60fJS9/NSg5deKSK6IHLIfG0TkLRFp67dNuv1a3zY7ROQrETnTIU1H/bZ5W0Sq+q0/U0S+E5GDIrJHRBaLyL0iUjGWn40TEekjIj+KyAER2Ssic0SkJ4Ax5gNjzFnxTkM0xLJeRFYkOi2qZNHAoULZAFzu+0NEugCVgzcSEQGuBvba/weba4ypCtQABgFHgUUi0jlou5r2dicC3wCfisi1Qdv80d7mJKAH8ICdhouBj4FxQHNjTB3gUqAJ0NTDMXsmItWBr4AXgNpAY+ARIDue7xsj/YD6QEtfoCtOIpKWjPtSkWngUKG8R2AguAZ412G7vkBD4HbgMhEp77QzY0yuMWadMeavwCxgVIjtthtjnrfXPyUihb6jxpgtwGSgsx24ngX+aYx5zRiz195mlTHmb8aYNU7vIyI1RORdEdklIhtF5AHfe9klpR9E5N8iss8uKQ1x2g/Q1n6/D+1jPGqMmWaMWeK/L7/3NSLyVxFZY5eOHhWRVnaJJUtEJvg+QxHpLyKZInKfiOy2S11/DpEOROQcu6S1395f11Db2q4BPge+tp/77+tMEVlpl6L+C4i9vIK9/85+29azS4P1I6XDPoZ7RWQJcFhE0uy/t9ifxyoRGWhve4qIzLX3s01E/uv//bI/y+EisgZYIyJjReSZoOP4QkTuivA5KK+MMfrQR8ADyMAqHawCOgCpQCbQHDBAut+2bwATgHLAHuBCv3XXAj847P96YIf9PN3eZ1rQNi3t5R3802Q/bwosBx4F2genyeUxvouVaVaz07AauMEv3TnATfax3wpsBcRhP9Xt434HGALUClof8BnYaf3cfl0nrJLJdPt4awArgGvsbfsDx7ECYwXgdOAw0M5e/zbwmP28O7ATONVO8zX2Z1YhxPFXBrKAocCFwG6gvL2uLnAQuMg+r3fZ6bjRXv8m8LjfvoYDU9ykw36+2D6HlYB2wGagkd/3oZX9/GSgF5BmL/8NuDPos/wGq6RXCTjFPk8pfsdxBGiQ6N9UaXtoiUOF4yt1nIn1o93iv1JEKgMXA+OMMTlY1UVO1VXBtmL92CNtQ9B2n4nIfuAHrFLLE1iZA8B2v3SNt69Sj4jIVcE7FpFU4DJgpDHmoDEmA3gG8N92o7FKMLlYQaEh0CB4X8aYLKAPVib2GrDLvsottK2fp40xWcaY5cAyYJoxZr0x5gBWSap70PYPGmOyjTGzgEnAJQ77vBl4xRgz31gln3ewglKvEGm4wF4/zd5nOWCYvW4osNwY87F9Xv+D3+eLVSV4md/fV9jL3KZjjDFmszHmKJCLFRQ7ikg5Y0yGMWYdgDFmkTFmnjHmuH2OXsEKnv6eNMbsNVZJ7yfgADDQXncZMNMYsyPEZ6CipIFDhfMeVqZwLc7VVOdjXYl+bf/9ATBEROpF2G9jrDaRSNsQtN15xpiaxpjmxpi/2hnPHntdQ99GxpjLjDE1gZ+xrnqD1cXKKDf6Ldvo957gl1EaY47YT6viwBjzmzHmWmNME6Az0Agrsw3FPyM76vC3//vsM8YcDkpnI4d9NgfutgPmfjvANg2xLVglgQl2pnwMmEhBdVUjrFKA7/iM/9/Ad0BlETlVrI4S3YBPPaTDf99rgTuxqiZ32kG/EYCItBWro8R2Ecki8EKh0L5s7wBX2s+vxPoOqxjTwKFCMsZsxGokHwp84rDJNViZ3CYR2Q58hJUhXxFh1+cDs11ssxOruiycVVgloQsibOdvN1ZVVHO/Zc0IKlFFwxizEqsKKbjxP1q1RKSK39/NKCiN+duMVX1U0+9R2RjzYfCGItIEOAO40s6Ut2NVSw0VqyfdNvw6FdjtSPl/26WwCVidJy4HvjLGHPSQjoAhuY0x44wxfSioCn3KXvUSsBJoY4ypDtyH3dYSal/A+8C5InIiVjXrZw6flSoiDRwqkhuAM4KuehGRxlhVAudgXXF2w+oR9RQO1VUikioiLUTkBay6+0ec3kxEGojIbcDDWFVJeeESZ6+/G3hYRG4SkVpiaYND1ZL9Gl/G97iIVBOR5sDfsTIdT0SkvYjcbWfGiEhTrMx0ntd9hfGIiJQXkb5Yn/dHDtu8BtxilwJERKqIyDARqeaw7VVYbTrtKDh3bbHasS7HqrrqJCIXiNVb6XbghKB9jMPqufZnCqqpvKYDEWknImeISAXgGFaJy3fOq2G1wxwSkfZYbU1hGWMygQVYJY2JdqlUxZgGDhWWsXpCLXRYdRWw2Fg9iLb7HsAYoKtfr5veInIIKwOYidUo3NMYszRof/tF5DCwFKuEc7Ex5k2XafwfVr3/lVhXvLuxAsOrOGeyAH/Damhej9VmMg6r0derg1gNwfPt9M/Dare4O4p9OdkO7MMqZXwA3GKXagLY5+gm4L/29muxqhidXAO86H/e7HP3MlbD/G6stqvRWFWBbYA5Qe83H+vza4TVLhNNOsBq3xiNdc62Y3UPHmmv+wdW6fUgVkD6X5j9+HsH6IJWU8WNWNWXSqlkIyL9gfftthPlkoj0wyo9NjeawcWFljiUUqWGiJQD7gBe16ARPxo4lFKlgoh0APZj9bAL16tNFZFWVSmllPJESxxKKaU8KfUDg9WtW9ekp6cnOhlKKVViLFq0aLcxJuSNvKU+cKSnp7NwoVNvUqWUUk5EZGO49VpVpZRSyhMNHEoppTzRwKGUUsoTDRxKKaU80cChlFLKEw0cSimlPNHAoZRSyhMNHEoplaS+XbGDHVnHEp2MQjRwKKVUkrrx3YVc8OKPiU5GIRo4orBl/1Eydh+OvKFSqkRavvUA2cdzE50MwMpvko0Gjij8YfQM+v97ZqKTUSr8ti2L9BGTWL71QKKTohQA2w8cY9iYH3jws2UJS8Pvx/P4/XjYWZMTSgOHSqipy7fb/+9IcEqUF8dycrnno1/ZebBw/fu2A0eZZp/XeDDGEM10EMu2HCB9xCQ27glfW3DgaA4AM1ft4r5Pl3IsJ7DkYYyJa6a+I+sYbR+YTNsHJkfeOEE0cISxdf9RNu05kuhklGolcTqY47lFyzSO5eRyKPt4jFLj3eLN+6PKePce/p30EZMYN38TXy/dxkeLMnli0m+Ftjtv7Bxufm9R1Bl8KJ8v3sLPm/bR/dFv6Pv0d55f//GiTACm/7Yz7HYGK807D2Yzbv4mJizcHLD+3bkbafvA5IBG6+zjuazecdBzmvzd9O5CJi/dxiuz1ke9j10Hs/lgftjxCWNCA0cYp42eQb9/WV/Qf365gue+WR2w/ruVO0kfMYmtdh3kgaM5pI+YxJRl24o9rSWVL1uRMNt8v3oXHR+aktDM1mfGyh20vn8yy7ZEX7U24N8z6fzw1BimytmxnFxy8wIz7inLtnPe2Dl8ZGeiXmzea11EjV+wyXH91v1Hycsz7MjKBuCNHzbQYuTXHDhiXcGv3XmII7+HPocrt2flX+07uWP8Yi548Uf2H8khc1/4ev/D2cfJOpbD1v1HSR8xic8XbwlYv/3AMdbuPARATm4eExdlsm7XIYwxDP7P7IBt8/IM2w4cpcuoqazZcZAvft0KFHweAA99tpyznvvesQQWyp5D2aSPmMQlL88F4JsVO7j1g59ZuT2r0LYzVu5g18Fsx/2s3Xkw/zwPH/cz93+6rEjfTzc0cLj05pwNPD99TcCyD3+yfkCnjZ4BwPpd1hfxpSJcMZQ59hWphIkcz3yzmiO/57LG5RXdz5v2setgNos376fdA5PZfcj5BxcN39XqL5v3Byz3UnWx7UBB5vLKrHUszXT3I8/LMxz2EDzbPziF28b9HLDMV03j9rP0+XHdbvLsc2VMwfnyhaUNuw9z2ugZvDBjbf5rxtm/j12HssnLMwx6dhYdH5rKeWPncDw3j5827OXnTfvytx/8n9mc+Mg03pqzgT5PzQioIlqxtXBmGk6Px76l66hprNpuHecnPxcEjmemraLXk9MZ9OwsAE765zfc/dGvDHxmFh/MLxwUDTB12XYOHjvOmc99z6KN+wpts2DjXgB2ZmXz3crAEs3mvUd49pvV+aWv5VsPsHJ7Fr9ssr5DP2XsZf+R3/O3z3Eo0V7/9kKueG1eoeVrdhxk0LPf5+dNew9b+znnhR9CfDKxoYEjBDdF7FBXwNk5oXtjLMjYyymPf8vBYzms2XGQsd+tDbltUWw/cMzxCxgLa3cezL+KBOuHEW1f84ISR7gyhzcXvPgjQ8fM5rXZ68k+nsfcdXsKv68xhequPTGGIc/P5t25GWzac4S2D0zmo6AqDTeenLySP/7X3Y/8P9+uptPDUwM++0gmL9vOQ58v45VZ6wBITbE+Zy9fje9W7uSK1+bzxg8b8pf5ztfni7dy8FgOmfusq+/nvi0ola/fZQWp8qkpHPProbR4837mrd/LJa/Mdexq+siXK8jcd5T2D05hyrLt/H48j6FjZhfazufjRZlc8OIcbnxnQf6yo0Hn1v/C5PDvBeuO5+Zx0O93/IBDg3heiKzAt88DR3Lyj/X+z5Zx3dsLWLk9i9lrdvGH0TPo+/R3jJm+hnW7DtHv6e8YNuaHQqWap6asyn++IKNwYALICGqbWbRxL8vtgLpo415+2rA3vxQVbxo4Qgiu13Tyo0OGBLBye+iruWemrWLnwWyWbjnAJa/M5V9TV4UtvkdyPDev0FXooezj9HpyOg99voztB47x47rdUe375ncX0uuJ6QHLjuXkMujZ7zn/xTn5y/o+/R2nBm0XyYGjOSz2u2oPV+Lwovs/pwGELNb7vDkng/YPTnEd8A5lH2fTniMB6fxtWxYPfb48v277no+XRJdo4LGvVpCXZzW6fvpLpuOFi6+KZK/f1am/l2auI33EJI7l5DL+p4Ir53fnbuTJySsBSLEP4M05G1wHTl930K+WWFWwS7ccYOaqgqvqm95dyJ5DzmkCGPjsTDo+FFg1d+Ub8/OfhzsHt7y/KGQj8c6sY0xYsJl/fPQrP2/az7e/7eTZaasCLph87RWLMvY5llrGL4j8OzfGEO4y8kT7OwcFtQ6D/zObq974KaAr7ds/ZrDJr3prjV8m/+FPztV//nJyDcdz80gfMYn0EZO48KW53Pm/xQAs2WzlJ8VFA0cI905c6vk1/l8upx9+Xp4J+CLl5FrbBNdDe3HL+4voFFRffsQOJN/+tpPBz3/PFa/Nd3opV70xnxMfmea4DmDaih1szzrG6Mkr89Pe/sEpAKzffZjcPBPQULxsy4FCdbwzVu5wDIxXvzGf88bOiXnj+D6XV+NfLbEyYf8McNQXyxn+wc+O21/+6jz6/es7tu23js//nK2JwVXe6z9s4OdN+xgzfQ13/e/X/N5m/iJ9VG/8YFWRXvfWAib+7NyG4StxgBU83Hjl+3WFln22eGv+83nr93LPx7+GfL3vex7KqU9Mp+fj37pKi7+dB7P5v4mBwXrMjLUBweD6t63ZPw9mH+enjL2F9uEmeH6zwrnH34UvzSV9xKSAZQePhb4IfH9eYHB4asrKiO8drGuI3+tBh9qPWFbRBtPAESdOseCFGWvZvNcKHG/+sAHfb7gogePbCD1E9ofJSGev2R22MdLn5VnrWJy5v9BNj/2e/o7W9xdcDZ7zwg8MfGZW/t9rdx7k+rcXMvITKwj7B5pf7Xp937WcYAXbp6asLFTc/tUumbw3byPpIyY59mrKPp7LJ0GZ5cptoevFfQHr3olLeWvOBjbuOczbP2Ywaalzx4aldmPjdLv++udNBaWlaDKAUHxX3+HOy6rtWWFLkXPXO5eEtx04GnA1viqoZHz/p0t5fXZg+9z89Xvyv7PhRAoOkUQqIToJVUpdHabEH+wxh15hweZv2Mv2JBn248jv7qtXezzmPRi7pYEjRl6ZtY4j2QUnNc/hUnqO34/929925mdeuXlFrG936VhObsS2myHPz+bhzwvX8zq9zOmO1oPHjjNszGx2HczOv/rKsLs0935yOp1HBZaOxn5nXc2u2JbFjqxsXpq5jqvfcC4h+Ro4H3RI37+nruLvEwKvetftCt1f37+a7JEvV3D6v2aG3DZav27eT7d/TstvsHRaHyxcm49vyS3v/+xYioxUeuv95IyAjPJzv1IDwAfzNxXKSGMZFGPtnR8zHJe/Ny/23VGL0kW2NNLA4VFw0dTnyckreeLrgh9d8I94R9YxftoQWFROsYsc4xdspv2DU/Kv6CcuysyvK3199vqAbn+hBNbrFnY4+zjtH5zCM9NWO6wt8Nu2LN6ZW/iHd+FLPzJr9a6I6QBYvjUr8OrfFPSLP5bj3Co7edl21u+2jjk3Qg744U+F66V3hrliXbk9iw27Dwf0XAlnxMQl7Mw6FvZudl9anSzevJ9py7fz0sx17D+Sw/z1e5i9ZhcTgurTzx07J8QeYNaawp+1l2v6UA2swf4+YXHIda9+vy4gwCabCQu9dylWsZGW6ASUJv7VC8Eljhv8enz4+OqbfY2Oa3ceIr1uFe7+6FfKpQqTbu/LY5N+Y9z8Tcz4R3+O5eRy/6fLuHdIO+pXqxiwr1dmreP0tvVpXrdyfkOl/zVr1jErbR8vyuS01nUKXY3f/+nSkL1HfF793ttVl3hs8Q6+ij6Wk8uoL5a7eu2SMF1ax363Lr9kkyIw+oKuYfc1fsHm/HryjNHDHLdZtiV0Ndh5dkAY3OkEwPrsb7XbTi7p2TTk6wwFFxyTlmxj7BVW9d2EhZsZ1rVR2DRH65Oft/DsJd0c1z3xdfKWNlRiaeCIIf+2ilXbD3Ji05r5fztlNL4qjNy8wlfhObmGs577Hiho+Pp66TYm/pxJnjE8d2ngj/3f01bz7zClCV/VR54xhTLoN37YUKj/uq/xOFqrdxzi1JZ1onqtL61fLdkWsdfLFa/NC9m7zUmegScmR67X9sk6lkP1iuVcb+9vit3A7aWjRXD/nYUb93HvxKWMm7+JvWF6LsVSTm4e5VK1MkKFpt+OGPKvYglXDREsXsNu+FffLLRvUHK6t+PRr1YUWnbbuF+K9N5Hc47n96b6NfMAt7y3yNPrf928nzFBN1w68RI0fMJ1GAgWqpdVPCzauC/gRjUoaAz9NfOAY88ZH2MMe0K0pXiVncSD66nkoIEjhoJ7hhzOPs6f/vuD4xACXk1euq1Q468XvkDgtrtqLPiXbKZ4HPTu3LFzAvq8J8rsNbs57Un396hMWhJ+uJlQDbpgDa0SLLiXk78ej33D78fzuOW9Rdz9UfTfjXs++jViupXyp1VVLuRF2V12ztrdLMk8wL/87gp14tv7l0u2MrBD/ULrBfLryMuCWN0MGCtbD7jvijl8XPjz9HCYNpvg0pMxhtlrQne73X3od975McNzUA720aLMgLGrkuzjV0lIA0cc+Rp2p68Mf6+FryH988VbOcehEdSpx9Ad43/JH+umpFngcCOWKixc6cTn8a/dt9e4dTQnN/8Oc6WclLjAISKDgeeBVOB1Y8zoBCcpJC9Xqj6+u3/D+fSXLRG3SbSvl4a+Cr745fBDI7jJsjbtOVIsw0cn0qgvC7c9FYd3f8yI2MNOlW0lKnCISCowFjgTyAQWiMgXxpi4/sJyHHo9xcu89Xo17oZvuHsVe2NmxGfgTVV6lLTG8VOAtcaY9caY34HxwLnxftPfthVtgpZIslwM+6GUUsmipAWOxoB/x/5Me1kAEblZRBaKyMJdu9zd7RzO9N/iO63p7mLqn6+UUrFQ0gKHK8aYV40xPYwxPerVqxeD/cUgUcq1aNqGlFLFp6QFji2A/5gNTexlcRV+NH6llCpbSlrgWAC0EZEWIlIeuAz4It5vqiUOpZQqUKJ6VRljjovIbcBUrO64bxpj3I2CV5T3jfcbKKVUCVKiAgeAMeZr4OtEp0MppcqqklZVpZRSKsE0cLigbRxKqZJmwl96x23fGjhc0F5VSqmS5pQWteO2bw0cbmjcUEolgfrVKiQ6CYAGDnd0oFClVBKYeU9/ekc5s2YsaeBw4dCx0DOvKaVUcalcPo0Pb+5F+QRP7auBw4Xg+biVUiqh7FqQb//eLyFvr4FDKaVKmBQ7cDSuWTkh71/ibgBUSqmS6rKeTenUqDoPfl60AS8uOrkJ78/bRFpqQQPsW9f1ZMXWLAZ2qM+q7fGdCkIDh1JKFZO/nN6KxjUrFTlwPPKnzowY0oFyfm0dA9rVZ0C7+gC0P6F6kfYfiVZVKaVUMTExups4NUWoWiFx1/0aOJRSqhiJX/f+NLuxok39qmSMHpagFHmngUMppRKkUrnURCchKho4lFIqjh4Y1oEWdas4rptyVz8uOrkJY/98kuP6jNHDWP3YkHgmLyraOK6UUsXIV1MlAo1rVuLfF58YdvvyaSlMvqMvSzL3xz9xLmmJQymlikmkpvFrT0t3XN6hYXUu7dks5umJlgYOpZSKM7fD3Y36U6cS0UiuVVVKKVVM4jG3T41K5ejSuEbsdxyGBg6llIo3vyKHSOTyx2mt6vDzpn2udv3rw2dFm6qoaeBQSqkECBc+xt3Uq9jSEQ1t41BKqWJjHJ6VPBo4lFIqznyli3i0cSSCBg6llEqAkjyxqAYOpZRSnmjgUEqpOPP1pDIUlDQuPKlJwtJTVNqrSiml4qxmpXKANRx6SoqwZNRZVC6hAxyCBg6llMrXM70WCzLc3T/hxYt/Pokvl2yjpT3YYfWK5WL+HsVJq6qUUspB3zZ1Y7av+tUrckOfFq5u/isJNHAopZSD0pLJx4MGDqWUctAyxBwaSgOHUkrlGzGkQ/7ze85uxwc3nprA1CQvDRxKKWXr3Lg65VKtKqq0VOEPrWPXzlGaaOBQSimb+N3PLQ73dv/xxEbFmZykpYFDKaUcGIdhCF+4vHsCUpJ8NHAopZQfp5KGG6e1qkPl8iX3pj4vNHAopZTNqQfuKS1qu3ptg+oVS/TAhV5o4FBKlVnN61QOuS5/CHSXQ6Hf0KdF0RNUQmjgUEq58uwlJyY6CTHnWEKIstjQuZjn/U4kDRxKKVcuKMGjuYbiWJgoJZMtxVPSBQ4RGSUiW0Rksf0Y6rdupIisFZFVInJ2ItOplCobKpRzn03e2LclAO9cfwr1q1WIV5ISLukCh+05Y0w3+/E1gIh0BC4DOgGDgRdFpGx0YVBKxUXNyuUD/pb8fwo8fl4X1/u768y2ZIwexult63FO19J7z0eyBg4n5wLjjTHZxpgNwFrglASnSSlVTG4/o3XM93nbgMj7bBamAb2sStbAcZuILBGRN0Wklr2sMbDZb5tMe1khInKziCwUkYW7du2Kd1qVUsUgHk0P5dMCs0AdEdedhAQOEflWRJY5PM4FXgJaAd2AbcAzXvdvjHnVGNPDGNOjXr16MU69Uiqe0lISm3lr6IgsITMAGmMGudlORF4DvrL/3AI09VvdxF6mlFJR6dyoesDfsYxZN/ZtwaJN+zi/u2PFSImWdFVVItLQ78/zgWX28y+Ay0Skgoi0ANoAPxV3+pRSsde0dqWEvG+dqhWY+Y/++X+LCLWCGsyj1ahmJT4f/gfqVC19vauScc7xp0WkG1aVZgbwFwBjzHIRmQCsAI4Dw40xuQlLpVIqZmpVLs/mvUfDbmOK6f6KCX/pzfdrdlGxXPhOm5XLp3Lk97KZBSVd4DDGXBVm3ePA48WYHKVUMSiuoOBGszqVubJO84jbrfjnYNJHTCqGFCWfpKuqUkolry9v60OdKkWvyhkxpL2r7V67ukeR3ysS7UjlnQYOpZRrXZrUYNGDZxZ5P7ec3srVdmd2bBDw99rHh3B178ilgWhUqxB9BUxZCz4aOJRSnt05qE3M9vXGNT3CZrwdGlo9nwyGtNQUUiLk0h0bVg+7PpivmqxWDEpSZYUGDqWUZ3cOahuzfQ3s0CDs+qGdT/C0v/S60d3pXZRSQxkrcGjgUErFzxvXxL+NIhk8e0m3RCehWLkKHCJyh4hUF8sbIvKziJwV78QppUq2SNVKbjSuZd3j0bimc0miQ1DVVLRTvxbFeaXwJr9w3LYGXW+Med4eyrwWcBXwHjAtbilTSpUZ4brjnt+9MbWrlOf0ts7DB5kwL65ZuRz7j+QUNXkqiNuqKl8IHwq8Z4xZTimv1pt1T/9EJ0GpMu3SHtYIQyJC/3b1XQ9AaPyGQ5z9fwPikrayzm3gWCQi07ACx1QRqQbkxS9Zide8TpVEJ0Gpks/l5aV/THBbuzW40wl8dEvvsNtUq1jO3c6UJ26rqm7AGq12vTHmiIjUAa6LX7KUKv16ptdiQca+RCcjovYnVKNZ7eSbk+KUFrXpmV47Ie89sH39hLxvsggbOETkpKBFLXW8euWkT+u6/LB2d6KTUeya16nMxj1HonptIhpxozHlzn7F+n5uhx9JVFa08tHBlEst2x1SIx39M/ZjLDAPeBV4DZhvL1MKgDPK6BXYF8P7JDoJSaNVvYLq3Xeutybn7NK4huf9uB22KlSAqVMl9Gi0E289jfduKNrEoRXLpZJqj7/+0S29+fuZsbunpaQIGziMMQOMMQOwJlQ62Z4c6WSgOzoXhiqiG/u0SHQSiqxG5ejr0CuUK11XrdPv7p///PS29cgYPYy6RRhSPNoSxX1DO4Rcd3LzWvRtU4//3dyLf13UNcqUFeiZXpvbB8buLvqSwu03t50xZqnvD2PMMiD02VExde9gdwPCxVr/du5nT4xmcNOUBM/0lmg9msemfr5K+fDDfyeTcqlC3zZ1Cy3/a393Yz4WjK8AABsnSURBVFf5CxVYKrn4PE5tWYeL7V5bSTQwb4nhNnAsFZHXRaS//XgNWBLPhKkCxsVXOx6Nda9eFd+7fjtHUY1RmgTPdx2tBjUqxmQ/Xj10Tkcm3e6tqm7N40N574ZTAbjn7Hb0s+/NGNy5IY+e2wlw388/1kOxl+3LGG/cfnOvBZYDd9iPFWivqrjq3qxmopMQ98bH7k0Tf4yJdNHJTWKzowRdMl/fpwWdGtXgkh5NOMtvFNv2J1Rz9frhA1rz7vWF2xt87QdpqZqVJ6uI3XFFJBWYbLd1PBf/JCmA1vWq8sum/a63H9a1IdNX7oxjilSslY9Rz5y+beqyfvfhmOwrGk9fdGL+85WPDs7P+H0m/KU3l7wy1/X+zuvWmOqV0rjtDHdtB04lj2juGNcqK/ciBg5jTK6I5IlIDWPMgeJIlPKuTX13V3nJJDiDceO+oe2pkJbKw18sj0OKCjhNC1ouVejXpl5+gK5XrYhzScfogrp/u/q8M3djkfdzVscGTFuxo0j7cJpu9ZQWtfnh3gGuu7CWSxPuH9axSOmYdc8Ajvx+3NW2Wq7xzu0lzyGsdo43RGSM7xHPhJV1wYPDPXRO+B9SPKqVwu2yKFVpZ3dqwKPndY4q4725XytOLIYqLqeOAZXLp/HGtT2ZO/KMuL9/cTihemDbSIUIc2wXRZNalWlQPXxbTCyv+GtUKkfDGpViuEflz23g+AR4EPgeWOT3UFGKdLEd3DPEN0JoOG7rlmPhnrPbRf3aE6pX5KpezZP2Su+M9vXDDpOdakdpt42z53ZrFItkeZIxeljEbVrWS85hdbzeGFm3WtEmYNIqKu9cBQ5jzDtOj3gnTlmMcZdJjb+5l6v91SzCvQc+p7Uq3KUyWsnWK7dprUqOVS7R6tG8Vsz25caUO/uGXBeua7fTaahcArr6XntabO4HSrKvYVJzOx9HGxH5WERWiMh63yPeiVMFmkQocbSqV5WalZ2vvP52RuuAv5+95ETH7YrLCUFVCJGGsWlR1/nKOLiqJdaWjkrclDPBc227dfeZbWl/QuipU28Nc79ESR1NKC3ZrjzKALdVVW8BLwHHgQHAu8D78UqUsrx5rXUfRa+WdcLe8zD+5l6ubnryqVrBXYnDy7hk4eZECHZT38JXiMH3NNzlYmrSBtWja5xuU78qr18d+R6VkCOr5n8shY+5Tf2qrtMR7uNt16AaL18ZPFSc5Zu7Qo8d9bci3MXs1FlBs2TlxG3gqGSMmQ6IMWajMWYUELkStYx7/rJuPHZeZ8d1Q7s0jPj6M9o3YO3jQzjZQ1XH2CtOYlDQHM7J8OP3vypMc+hdE5zGWlUiBzevddO+O6z7ta3HoKArev8S3fABgSW0YKHq4O85ux1f/i12Y1cN7uz8HWnToBobnhyaf/NcqBtE3QbWPq3rcv0fWvCgQ0+m4hzUtJFdEk0PUcJUycNt4MgWkRRgjYjcJiLnA+4vrcqoc7s15o8nOjeMVinvbkR7/0y2T2vndgX/Rs5hXRvyegLmefbPYAZ1KHwXu6+X2M39WuYvi7ZR0j8ra1zTymweGBZ5BJzaVUM3ovqSn5Yi1I9QBVatonXuruqVHrB8+IDWMW0bCUdEIrYNzR0xMOJ8FUtHncVb1/XkoT92pFaVwp+Pl5JkUQ3q2IBxN53KdaelF9t7gtXNGqBOEcbVKmvcBo47gMrA7cDJwJXANfFKVKkSw9/dW9f1dFxev1qEuv6gq8ZwF5Et6lYpcmP1gw5dhw2GjNHDAgag8wWTLo1rRGzDCWXOiDNY/sjZ3OBiwEQ3vXXCnS5fJlqxXCoZo4dxx6DkGNwu1HGlpEjE+SqqVSwX9v6K4p5G4bRWdT2PYRZpSJ7nLg3fptekVmWeurALL195sqf3LcvcBo69xphDxphMY8x1xpgLjTHz4pqyJBTLweSi+T1GOwdAvH76S/wajyNdmTqtTk0RJt7am3euOyXssOyh0u/bZ5UKaa4yuOAMZtpd/Xj4jx3t94hjBumXtvph7l1Z/diQ/Lad4MOZfvfpIV8XKeP88CZ3ve0A0usETtgULr2JMrjzCQD0blXH1fbnd7eGdmlaO/TFyaU9mxX9hs4yxG1O9KaIrBOR8SIyXES6xDVVSWr0hVEMwxxlfhTcThFL4fL4UMl1Gjq6eojGYy+1Gyc3r+04NPlZHU9wv5Mw2tSvytW9mwcs86WvbYNq+e0EBeviWzXz0/2D8p8Hf9bl01JCXlC0cJjK2O1XK1QG6+biZUASzrPSq2UdMkYPo0PDwr3Hbj/DuX1q3siBfH176G7Kyhu393GcjjWM+gtATWCSiOyNZ8JKih7Nazn2EookUv7Ux2Ho6Wh5Ld1cYg837f8yL5PVOB3aXRFe719iGHN5d05wMeJrqOOa+Y/+PHpeZ9Y9MZSpfrPXhStVxLNGpl6YtpVgXuKWlxAX7f0YydCxwq0B7erx97Ocb0w9oUZFnX88hly10IpIH6Cv/agJfAXMjmO6Sow+bepSo1Jsv5Buuoq6dd/Q9oUG0xOBpy/syszVO5m8bHtgZiXw+PldeOCcjkWaL6Nl3Sr5A+/1bVM3Yk+lcFrVr+ppEL/0ulUCeuaEy4yb167MmR0bcNuA1oxfsJkLTmocdTp9Ljq5CR8vysz/++xOsSk9QfQB7od7z+BwduDYTU6BNLjKr6Te2xFrU+/sVyJuhiwu7rr2wEysIUaeBL42xvwetxSVQGGrfqL44QV3FS2Km/u14u05Gwotv6RnUy7p2ZR1uw4x8JlZAetSU4SqFayvRrlUISfXOsDzuzfm018iT/xYsVwKnw7/Ayu3ZXHpq/O4/JRmRTqG5y7tRueHpxZaHvy5T7y1d+SOAkHSUlN4zQ7UsRoD64nzu7Akcz+rdxwCQjcwF7Xh2cura1cpT22HXlMR36MERA7fja/NHarzYqVdMQ7nUxK4beOoC/wT6A1MEZFvReTR+CUrOYnAKRF6qRSXT/56Wkz206peVU73q+cPziZ+HDEwv2G2o0Odso/v7u6zOzWgYY1K1KhUjlPtumg396yEK7X5glh+GkPkZSc3r03T2pULLY9V3veX093NUlc+LYVpd4VuzPZJDZOw4DVuqqW83m0+6k+RR6BN/rABJzWrxVvX9mTk0MTMlFkWuW3j2A+sBzZgzT/eCgh9+2opNuGW3lzSI3ACHi8ZU8/0Wvx038Aip+OkZu5vChzapSENqlegVhRjVNWrVoFW9SLfsjOwQwO+vK1P1F0ab+rbstCyPq3rMnyA9ylFQ+nd0mokdrrPJJKM0cOKVN0GhUfcrVQ+lf9e0d3TPkJ91dY9MZRXPH72rd0MxV8SIgdWI36FNK1KKi5ux6paDzwD1MYaeqSd3WBeJvnXDQ8JcXdvKPWqVXC8wSxUb5Bgt0WRedWvXpH59w2iZYgA4H9X9yN/cr7THQoC5CktnEtdXZrUiLpqo3xaCud0Dfws37/xVO45O3ZXkR0bVSdj9DBOC3EjZTxljB7G29cVnu3unK6NuLp38/yg4maaYCepKRKXOdzj2k1ZlVhu2zhaG2Py4pqSEsT/x93uhGr8uG63530E5693ndmWMTPWRnzdP85uxz+iHNI8VBbw5AVdeHHmOh4Y1sFxOJBgXcr4XOGx9s9zHYJ10BckRazqPKfRbePZg7gENHGoBHAdOETkJaCBMaaziHQF/mSMeSyOaSsxivrbalSjYkIbIetXr8ioP3VK2PuXNX/p19LzzWYiwq8Px3+03uBvYTkdeVY5cNs4/howEsgBMMYsAS6LV6KSnZfiezL97OJ1YXpSEWYDdKuhw30dXqt1inPcpXBGDu3AjQ5tOkD+DYmnty08A2EosbzmKNR7SIscyoHbEkdlY8xPQVfF7ib0VUknllnBogcGUaWC26+RO04Z/KTb+7Ij6xjgvd69JGV9JzWr5Wr2Pn+xjIfBJaFwPelU2eX2F79bRFphX7SKyEVYvavKFF+GFXylG1zN9PXtfdl1KNv1fpPjOtg9/4wqliOKhquui/Y+BCh5n28y8Y0LpZQ/t1VVw4FXgPYisgW4E7gl2jcVkYtFZLmI5IlIj6B1I0VkrYisEpGz/ZYPtpetFZER0b53cejYqHp+VUOhO3ETeP3rGz46WWsf3FYl+dKfmhLdoI+ljdfz+dA5HZNuul5VsrgqcRhj1gODRKQKVrA5gtXGsTHK910GXIAVjPKJSEd7v52ARsC3IuIb5GgscCaQCSwQkS+MMSuifP+oRNNVMpl+n89d2o2352TQvWnxzoHtVaSOAh0bVucv/VpyZa/mYbfL318sEpXEvFZVXd+nBde7GIZeqVDCXrKJSHW7BPBfETkTK2BcA6wFLon2TY0xvxljVjmsOhcYb4zJNsZssN/nFPux1hiz3h7uZLy9bVLw5XODOtRnWtC0nlUqpHHXoLbcETS6bN82BY2fXqZ9LYqGNSoxcmiHIvf3T3SJJSVFGDm0g+Nd4rH29nU9Xd9jU9zi0ROvtAdZFRuRShzvAfuAucBNwP1Y363zjTGL45CexoD/PB+Z9jKAzUHLTw21ExG5GbgZoFmz6MdJalKrEpn7jhbef4ifV8MalWjboPDduHcMasNXS7YGLBvWtSH92p7F+/M2Fbrxza3/XNqN9g2LfwydeHdOinXvpxr2WEbRNOL3b1ef/u2Sb2jxeNFZ8JQbkX5JLY0xXQBE5HWsBvFmxphjkXYsIt8CTi1r9xtjPvecUg+MMa8CrwL06NEj6lzo3sHt+duHv0TczjeVqNchPapVLMet/aMfUuO87kUfyTWZxOteluEDWlGnSnkuOKlJ5I1j6Inzu/DNiu3F+p5FdWv/Vjz7zepEJ0MluUiBI8f3xBiTKyKZboKGvf2gyFsVsgVo6vd3E3sZYZbHTZrLKp1zT2zM4excLu5RvBmTcqdCWirXFPM81gBXnNqMK04t2sjAxS3aWSZV2RLpW3KiiGTZj4NAV99zEcmKQ3q+AC4TkQoi0gJoA/wELADaiEgLESmP1YD+RRzePyxfFVVwm0RKinBlr+ZJNcjauJtOZdY9/WO6T99scOd2axTT/aqiSZL7GlUZErbEYYyJS04oIudjzSZYD2s2wcXGmLONMctFZAKwAusGw+HGmFz7NbcBU4FU4E1jzPJ4pM1fqDH4Lz+lGW//mEEjF7PUJcpprWI/kF+relU935xWFtxyeiuO5eTGZd9/PLERw1wMS69UcYrtLb8uGWM+BT4Nse5x4HGH5V8DX8c5aQFCjiZr3w9RsVzylDBKg5FD2nM8Ny9m840XlxFD4jcPxAuXRx52PdG93FTZk5DAoZSTRjUr8VKU83mUZVpVpYqbtoQVE/1xK6VKCw0cxU2rFZRSJZwGDqVKKL0GUYmigcMDXyNkzUrWjX5l6Y5ipZTy0cbxKNSpWoH59w2kTpTDfCsVC71a1mH6yp3FMmaXUv40cESpQfXkvYdDlQ039m3B0K4NaVyzUqKTosoYraoqJmd2bMCwrg25f2iHRCdFlRIiokFDJYSWOIpJxXKpjL3ipEQnQymlikxLHB7ovRhKKaWBI6J/nNU28kZKKVWGaFVVBOl1q+Q/j/eYQG9e24MVW+Mx6LBSSsWOBo4kckb7BpzRvkGik6GUUmFpVVUE2q6hlFKBNHAopQL0bRP7uVxU6aJVVUqpAO/dcGqik6CSnJY4lFJKeaKBIwKdXU0ppQJp4IhAG8eVUiqQBg4PtPChlFIaOJRSSnmkgUMppZQnGjiUUkp5ooFDKaWUJxo4ItBOVUopFUgDhwdtGlRLdBKUUirhNHC4NLB9fVrXr5roZCilVMJp4HCpUvnURCdBKaWSggYOpZRSnmjgiMDomCNKKRVAA4dLoqMdKqUUoIHDNS15KKWURQNHBFrSUEqpQBo4lFJKeaKBQymllCcaOCLQtg2llAqkgcMlbetQSimLBg6llFKeaOBQSinlSUICh4hcLCLLRSRPRHr4LU8XkaMisth+vOy37mQRWSoia0VkjGjdkVJKJUSiShzLgAuA7x3WrTPGdLMft/gtfwm4CWhjPwbHP5lKKaWCJSRwGGN+M8ascru9iDQEqhtj5hmrm9O7wHlxS6BTGorzzZRSKoklYxtHCxH5RURmiUhfe1ljINNvm0x7mVJKqWKWFq8di8i3wAkOq+43xnwe4mXbgGbGmD0icjLwmYh0iuK9bwZuBmjWrJnXlyullAojboHDGDMoitdkA9n280Uisg5oC2wBmvht2sReFmo/rwKvAvTo0UPv4FNKqRhKqqoqEaknIqn285ZYjeDrjTHbgCwR6WX3proaCFVqUUopFUeJ6o57vohkAr2BSSIy1V7VD1giIouBj4FbjDF77XV/BV4H1gLrgMnFnGyllFLEsaoqHGPMp8CnDssnAhNDvGYh0DnOSVNKKRVBUlVVKaWUSn4aOJRSSnmigUMppZQnGjgi0Ok4lFIqkAaOCAxW5NAhFZVSyqKBIwJfiUPjhlJKWTRwRJAfOLTIoZRSgAaOiHxNHBo2lFLKooEjAqN1VUopFUADRwS+EkeKVlUppRSggSMiX4lDw4ZSSlk0cLikBQ6llLJo4IhAbwBUSqlAGjgiKOhVpUUOpZQCDRwRFdzHkdh0KKVUstDAEYEOOaKUUoE0cERQ0MahkUMppUADR0T5bRwaN5RSCtDAEZnex6GUUgE0cESgJQ6llAqkgSOCgqGqNHIopRRo4Igof8gRjRtKKQVo4IhIh1VXSqlAGjgi0ImclFIqkAaOCLRxXCmlAmngiCAtxYoY5dP0o1JKKYC0RCcg2V12SlO27j/K7We0SXRSlFIqKWjgiKBCWiojh3ZIdDKUUippaP2LUkopTzRwKKWU8kQDh1JKKU80cCillPJEA4dSSilPNHAopZTyRAOHUkopTzRwKKWU8kRMwaTapZKI7AI2RvnyusDuGCanJNBjLv3K2vGCHrNXzY0x9UKtLPWBoyhEZKExpkei01Gc9JhLv7J2vKDHHGtaVaWUUsoTDRxKKaU80cAR3quJTkAC6DGXfmXteEGPOaa0jUMppZQnWuJQSinliQYOpZRSnmjgcCAig0VklYisFZERiU5PUYhIUxH5TkRWiMhyEbnDXl5bRL4RkTX2/7Xs5SIiY+xjXyIiJ/nt6xp7+zUick2ijskNEUkVkV9E5Cv77xYiMt8+rv+JSHl7eQX777X2+nS/fYy0l68SkbMTcyTuiUhNEflYRFaKyG8i0rs0n2cRucv+Ti8TkQ9FpGJpPM8i8qaI7BSRZX7LYnZeReRkEVlqv2aMiEjERBlj9OH3AFKBdUBLoDzwK9Ax0ekqwvE0BE6yn1cDVgMdgaeBEfbyEcBT9vOhwGRAgF7AfHt5bWC9/X8t+3mtRB9fmOP+OzAO+Mr+ewJwmf38ZeBW+/lfgZft55cB/7Ofd7TPfQWghf2dSE30cUU45neAG+3n5YGapfU8A42BDUAlv/N7bWk8z0A/4CRgmd+ymJ1X4Cd7W7FfOyRimhL9oSTbA+gNTPX7eyQwMtHpiuHxfQ6cCawCGtrLGgKr7OevAJf7bb/KXn858Irf8oDtkukBNAGmA2cAX9k/iN1AWvA5BqYCve3nafZ2Enze/bdLxgdQw85IJWh5qTzPduDYbGeEafZ5Pru0nmcgPShwxOS82utW+i0P2C7UQ6uqCvN9IX0y7WUlnl087w7MBxoYY7bZq7YDDeznoY6/JH0u/wH+D8iz/64D7DfGHLf/9k97/nHZ6w/Y25ek4wXrankX8JZdRfe6iFShlJ5nY8wW4N/AJmAb1nlbROk/zz6xOq+N7efBy8PSwFFGiEhVYCJwpzEmy3+dsS41SkW/bBE5B9hpjFmU6LQUszSs6oyXjDHdgcNYVRj5Stl5rgWcixUwGwFVgMEJTVSCJOK8auAobAvQ1O/vJvayEktEymEFjQ+MMZ/Yi3eISEN7fUNgp7081PGXlM/lD8CfRCQDGI9VXfU8UFNE0uxt/NOef1z2+hrAHkrO8fpkApnGmPn23x9jBZLSep4HARuMMbuMMTnAJ1jnvrSfZ59Yndct9vPg5WFp4ChsAdDG7p1RHqsh7YsEpylqdg+JN4DfjDHP+q36AvD1rLgGq+3Dt/xqu3dGL+CAXSSeCpwlIrXsq72z7GVJxRgz0hjTxBiTjnXuZhhj/gx8B1xkbxZ8vL7P4SJ7e2Mvv8zujdMCaIPViJiUjDHbgc0i0s5eNBBYQSk9z1hVVL1EpLL9Hfcdb6k+z35icl7tdVki0sv+HK/221doiW70ScYHVs+E1Vg9LO5PdHqKeCx9sIqxS4DF9mMoVv3udGAN8C1Q295egLH2sS8Fevjt63pgrf24LtHH5uLY+1PQq6olVoawFvgIqGAvr2j/vdZe39Lv9ffbn8MqXPQ0SfQD6AYstM/1Z1i9Z0rteQYeAVYCy4D3sHpGlbrzDHyI1Y6Tg1WyvCGW5xXoYX+G64D/EtTBwumhQ44opZTyRKuqlFJKeaKBQymllCcaOJRSSnmigUMppZQnGjiUUkp5ooFDKRdEJFdEFvs9wo6aLCK3iMjVMXjfDBGpW9T9KBVL2h1XKRdE5JAxpmoC3jcDqy/+7uJ+b6VC0RKHUkVglwietucz+ElEWtvLR4nIP+znt4s1H8oSERlvL6stIp/Zy+aJSFd7eR0RmSbWPBOvY93Q5XuvK+33WCwir4g150iqiLwt1pwUS0XkrgR8DKqM0cChlDuVgqqqLvVbd8AY0wXrrtv/OLx2BNDdGNMVuMVe9gjwi73sPuBde/nDwA/GmE7Ap0AzABHpAFwK/MEY0w3IBf6Mdbd4Y2NMZzsNb8XwmJVylBZ5E6UUcNTOsJ186Pf/cw7rlwAfiMhnWEOBgDUUzIUAxpgZdkmjOtakPRfYyyeJyD57+4HAycACe4K2SlgD230JtBSRF4BJwLToD1Epd7TEoVTRmRDPfYZhjR90ElbGH80FmwDvGGO62Y92xphRxph9wInATKzSzOtR7FspTzRwKFV0l/r9P9d/hYikAE2NMd8B92IN510VmI1V1YSI9Ad2G2uelO+BK+zlQ7AGKgRrQLuLRKS+va62iDS3e1ylGGMmAg9gBSel4kqrqpRyp5KILPb7e4oxxtclt5aILAGysabe9JcKvC8iNbBKDWOMMftFZBTwpv26IxQMkf0I8KGILAd+xBo+HGPMChF5AJhmB6McYDhwFGvWP99F4MjYHbJSzrQ7rlJFoN1lVVmkVVVKKaU80RKHUkopT7TEoZRSyhMNHEoppTzRwKGUUsoTDRxKKaU80cChlFLKk/8HfwCz449P3zYAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#2. Multiagent Grid world"
      ],
      "metadata": {
        "id": "Rx0JF_jyVJSH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class MAalgo_MAGridWorld(gym.Env):\n",
        "    \n",
        "    metadata = { 'render.modes' : ['human', 'rgb_array'] }\n",
        "        \n",
        "    def __init__(self, size, n_agents):\n",
        "        self.size = size\n",
        "        self._grid_size = (self.size, self.size)\n",
        "        self.world = np.zeros((self._grid_size))\n",
        "        self._timestep = None\n",
        "        self._max_timestep = int(self.size*4)\n",
        "        self.n_agents = n_agents\n",
        "        self.dist = int((self.size-1)/(n_agents-1))\n",
        "        self._step_cost = -0.1\n",
        "        self.observation_space = spaces.Box(low=0, high=self.size, shape=(self.size, self.size))\n",
        "        self.action_space = spaces.Discrete(5)\n",
        "\n",
        "    def reset(self):\n",
        "        self.agent_pos = []\n",
        "        self.goal_pos = []\n",
        "        self._timestep = 0\n",
        "        self.done = [False for i in range(self.n_agents)]\n",
        "        for i in range(self.n_agents):\n",
        "            self.agent_pos.append([self.size-1, i*self.dist])\n",
        "            self.goal_pos.append([0, i*self.dist])\n",
        "        self.goal_pos.reverse()\n",
        "        observation = self.agent_pos\n",
        "        return observation\n",
        "\n",
        "\n",
        "    def _compute_distance(self, x, y):\n",
        "        dist = np.abs(x[0]-y[0]) + np.abs(x[1]-y[1])\n",
        "        return dist\n",
        "\n",
        "\n",
        "    def _is_inside_grid(self, pos):\n",
        "        return (0 <= pos[0] < self._grid_size[0]) and (0 <= pos[1] < self._grid_size[1])\n",
        "\n",
        "\n",
        "    def _is_collision(self, new_pos, agent):\n",
        "        for i in range(self.n_agents):\n",
        "            if i == agent:\n",
        "                continue\n",
        "            elif new_pos == self.agent_pos[i]:\n",
        "                return True\n",
        "            elif new_pos == self.goal_pos[i]:\n",
        "                return True\n",
        "\n",
        "    def step(self, actions):\n",
        "\n",
        "        self._timestep += 1\n",
        "        old_pos = self.agent_pos.copy()\n",
        "        rewards = [0 for _ in range(self.n_agents)]\n",
        "\n",
        "        for i in range(self.n_agents):\n",
        "            self.update_agent_pos(i, actions[i])\n",
        "            old_dist = self._compute_distance(old_pos[i], self.goal_pos[i])\n",
        "            new_dist = self._compute_distance(self.agent_pos[i], self.goal_pos[i])\n",
        "            \n",
        "            if new_dist < old_dist:\n",
        "              rewards[i] = 5 #-1 worked\n",
        "            elif new_dist > old_dist:\n",
        "              rewards[i] = -2 #-2 worked\n",
        "\n",
        "            elif (new_dist == old_dist) and not self.done[i]:\n",
        "              rewards[i] = -2 #-2 worked\n",
        "\n",
        "            if new_dist < 1.5:\n",
        "              rewards[i] += 25\n",
        "            \n",
        "            if (self.agent_pos[i] == self.goal_pos[i]):\n",
        "              self.done[i] = True\n",
        "              rewards[i] += 250\n",
        "            else:\n",
        "              self.done[i] = False\n",
        "              rewards[i] += -5\n",
        "\n",
        "        if self._timestep >= self._max_timestep:\n",
        "          self.done = [True for i in range(self.n_agents)]\n",
        "            \n",
        "        observation = self.agent_pos\n",
        "        info = {}\n",
        "        return observation, rewards, self.done, info #all(done)\n",
        "\n",
        "\n",
        "\n",
        "    def update_agent_pos(self, agent, action):\n",
        "        next_pos = copy(self.agent_pos[agent])\n",
        "        if action == 0: # Down\n",
        "            next_pos[0] += 1\n",
        "        elif action == 1: # Up\n",
        "            next_pos[0] -= 1\n",
        "        elif action == 2: # Right\n",
        "            next_pos[1] += 1\n",
        "        elif action == 3: # Left\n",
        "            next_pos[1] -= 1\n",
        "        elif action == 4: # No move\n",
        "            pass\n",
        "        else:\n",
        "            raise Exception('Action is not valid')\n",
        "            \n",
        "        if self._is_inside_grid(next_pos) and not self._is_collision(next_pos, agent):\n",
        "            self.agent_pos[agent] = next_pos\n",
        "        else:\n",
        "            pass\n",
        "        \n",
        "\n",
        "    def render(self):\n",
        "        state = np.copy(self.world)\n",
        "        for i in range(self.n_agents):\n",
        "            state[tuple(self.agent_pos[i])] = 1 - (0.1 * i)\n",
        "            state[tuple(self.goal_pos[i])] = 1 - (0.1 * i)\n",
        "        return plt.imshow(state)"
      ],
      "metadata": {
        "id": "E7iRMtzXVJSH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class RandomAgent:\n",
        "    \n",
        "    def __init__(self, env):\n",
        "        self.env = env\n",
        "        self.observation_space = env.observation_space\n",
        "        self.action_space = env.action_space\n",
        "\n",
        "    def step(self, observation):\n",
        "        return np.random.choice(self.action_space.n)"
      ],
      "metadata": {
        "id": "euXmgd2RVJSI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#3. MADDPG on Grid world"
      ],
      "metadata": {
        "id": "JY9ODifXBlfY"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Networks: GW"
      ],
      "metadata": {
        "id": "j5thjY-EJuji"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "class MultiAgentReplayBuffer:\n",
        "    def __init__(self, max_size, critic_dims, actor_dims, \n",
        "            n_actions, n_agents, batch_size):\n",
        "        self.mem_size = max_size\n",
        "        self.mem_cntr = 0\n",
        "        self.n_agents = n_agents\n",
        "        self.actor_dims = actor_dims\n",
        "        self.batch_size = batch_size\n",
        "        self.n_actions = n_actions\n",
        "\n",
        "        self.state_memory = np.zeros((self.mem_size, critic_dims))\n",
        "        self.new_state_memory = np.zeros((self.mem_size, critic_dims))\n",
        "        self.reward_memory = np.zeros((self.mem_size, n_agents))\n",
        "        self.terminal_memory = np.zeros((self.mem_size, n_agents), dtype=bool)\n",
        "\n",
        "        self.init_actor_memory()\n",
        "\n",
        "    def init_actor_memory(self):\n",
        "        self.actor_state_memory = []\n",
        "        self.actor_new_state_memory = []\n",
        "        self.actor_action_memory = []\n",
        "\n",
        "        for i in range(self.n_agents):\n",
        "            self.actor_state_memory.append(\n",
        "                            np.zeros((self.mem_size, self.actor_dims[i])))\n",
        "            self.actor_new_state_memory.append(\n",
        "                            np.zeros((self.mem_size, self.actor_dims[i])))\n",
        "            self.actor_action_memory.append(\n",
        "                            np.zeros((self.mem_size, self.n_actions)))\n",
        "\n",
        "\n",
        "    def store_transition(self, raw_obs, state, action, reward, \n",
        "                               raw_obs_, state_, done):\n",
        "        # this introduces a bug: if we fill up the memory capacity and then\n",
        "        # zero out our actor memory, the critic will still have memories to access\n",
        "        # while the actor will have nothing but zeros to sample. Obviously\n",
        "        # not what we intend.\n",
        "        # In reality, there's no problem with just using the same index\n",
        "        # for both the actor and critic states. I'm not sure why I thought\n",
        "        # this was necessary in the first place. Sorry for the confusion!\n",
        "\n",
        "        #if self.mem_cntr % self.mem_size == 0 and self.mem_cntr > 0:\n",
        "        #    self.init_actor_memory()\n",
        "        \n",
        "        index = self.mem_cntr % self.mem_size\n",
        "\n",
        "        for agent_idx in range(self.n_agents):\n",
        "            self.actor_state_memory[agent_idx][index] = raw_obs[agent_idx]\n",
        "            self.actor_new_state_memory[agent_idx][index] = raw_obs_[agent_idx]\n",
        "            self.actor_action_memory[agent_idx][index] = action[agent_idx]\n",
        "\n",
        "        self.state_memory[index] = state\n",
        "        self.new_state_memory[index] = state_\n",
        "        self.reward_memory[index] = reward\n",
        "        self.terminal_memory[index] = done\n",
        "        self.mem_cntr += 1\n",
        "\n",
        "    def sample_buffer(self):\n",
        "        max_mem = min(self.mem_cntr, self.mem_size)\n",
        "\n",
        "        batch = np.random.choice(max_mem, self.batch_size, replace=False)\n",
        "\n",
        "        states = self.state_memory[batch]\n",
        "        rewards = self.reward_memory[batch]\n",
        "        states_ = self.new_state_memory[batch]\n",
        "        terminal = self.terminal_memory[batch]\n",
        "\n",
        "        actor_states = []\n",
        "        actor_new_states = []\n",
        "        actions = []\n",
        "        for agent_idx in range(self.n_agents):\n",
        "            actor_states.append(self.actor_state_memory[agent_idx][batch])\n",
        "            actor_new_states.append(self.actor_new_state_memory[agent_idx][batch])\n",
        "            actions.append(self.actor_action_memory[agent_idx][batch])\n",
        "\n",
        "        return actor_states, states, actions, rewards, \\\n",
        "               actor_new_states, states_, terminal\n",
        "\n",
        "    def ready(self):\n",
        "        if self.mem_cntr >= self.batch_size:\n",
        "            return True\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "jojFTJS3Jujj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "class CriticNetwork(nn.Module):\n",
        "    def __init__(self, beta, input_dims, fc1_dims, fc2_dims, \n",
        "                    n_agents, n_actions, name, chkpt_dir):\n",
        "        super(CriticNetwork, self).__init__()\n",
        "\n",
        "        self.chkpt_file = os.path.join(chkpt_dir, name)\n",
        "\n",
        "        self.fc1 = nn.Linear(input_dims+n_agents*n_actions, fc1_dims)\n",
        "        self.fc2 = nn.Linear(fc1_dims, fc2_dims)\n",
        "        self.q = nn.Linear(fc2_dims, 1)\n",
        "\n",
        "        self.optimizer = optim.Adam(self.parameters(), lr=beta)\n",
        "        self.device = T.device('cuda:0' if T.cuda.is_available() else 'cpu')\n",
        " \n",
        "        self.to(self.device)\n",
        "\n",
        "    def forward(self, state, action):\n",
        "        x = F.relu(self.fc1(T.cat([state, action], dim=1)))\n",
        "        x = F.relu(self.fc2(x))\n",
        "        q = self.q(x)\n",
        "\n",
        "        return q\n",
        "\n",
        "    # def save_checkpoint(self):\n",
        "    #     T.save(self.state_dict(), self.chkpt_file)\n",
        "\n",
        "    # def load_checkpoint(self):\n",
        "    #     self.load_state_dict(T.load(self.chkpt_file))\n",
        "\n",
        "\n",
        "class ActorNetwork(nn.Module):\n",
        "    def __init__(self, alpha, input_dims, fc1_dims, fc2_dims, \n",
        "                 n_actions, name, chkpt_dir):\n",
        "        super(ActorNetwork, self).__init__()\n",
        "\n",
        "        self.chkpt_file = os.path.join(chkpt_dir, name)\n",
        "\n",
        "        self.fc1 = nn.Linear(input_dims, fc1_dims)\n",
        "        self.fc2 = nn.Linear(fc1_dims, fc2_dims)\n",
        "        self.pi = nn.Linear(fc2_dims, n_actions)\n",
        "\n",
        "        self.optimizer = optim.Adam(self.parameters(), lr=alpha)\n",
        "        self.device = T.device('cuda:0' if T.cuda.is_available() else 'cpu')\n",
        " \n",
        "        self.to(self.device)\n",
        "\n",
        "    def forward(self, state):\n",
        "        x = F.relu(self.fc1(state), inplace=False)\n",
        "        x = F.relu(self.fc2(x), inplace = False)\n",
        "        pi = T.softmax(self.pi(x), dim=1)\n",
        "\n",
        "        # print(pi)\n",
        "\n",
        "        return pi\n",
        "\n",
        "    # def save_checkpoint(self):\n",
        "    #     T.save(self.state_dict(), self.chkpt_file)\n",
        "\n",
        "    # def load_checkpoint(self):\n",
        "    #     self.load_state_dict(T.load(self.chkpt_file))"
      ],
      "metadata": {
        "id": "0QSdZmVtJujo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "class Agent:\n",
        "    def __init__(self, actor_dims, critic_dims, n_actions, n_agents, agent_idx, chkpt_dir,\n",
        "                    alpha=0.01, beta=0.01, fc1=64, \n",
        "                    fc2=64, gamma=0.95, tau=0.01):\n",
        "        self.gamma = gamma\n",
        "        self.tau = tau\n",
        "        self.n_actions = n_actions\n",
        "        self.agent_name = 'agent_%s' % agent_idx\n",
        "        self.actor = ActorNetwork(alpha, actor_dims, fc1, fc2, n_actions, \n",
        "                                  chkpt_dir=chkpt_dir,  name=self.agent_name+'_actor')\n",
        "        self.critic = CriticNetwork(beta, critic_dims, \n",
        "                            fc1, fc2, n_agents, n_actions, \n",
        "                            chkpt_dir=chkpt_dir, name=self.agent_name+'_critic')\n",
        "        self.target_actor = ActorNetwork(alpha, actor_dims, fc1, fc2, n_actions,\n",
        "                                        chkpt_dir=chkpt_dir, \n",
        "                                        name=self.agent_name+'_target_actor')\n",
        "        self.target_critic = CriticNetwork(beta, critic_dims, \n",
        "                                            fc1, fc2, n_agents, n_actions,\n",
        "                                            chkpt_dir=chkpt_dir,\n",
        "                                            name=self.agent_name+'_target_critic')\n",
        "\n",
        "        self.update_network_parameters(tau=1)\n",
        "\n",
        "    def choose_action(self, observation):\n",
        "        state = T.tensor([observation], dtype=T.float).to(self.actor.device)\n",
        "        actions = self.actor.forward(state).clone()\n",
        "        noise = T.rand(self.n_actions).to(self.actor.device)\n",
        "        action = actions + noise\n",
        "\n",
        "        return action.detach().cpu().numpy()[0]\n",
        "\n",
        "    def update_network_parameters(self, tau=None):\n",
        "        if tau is None:\n",
        "            tau = self.tau\n",
        "\n",
        "        target_actor_params = self.target_actor.named_parameters()\n",
        "        actor_params = self.actor.named_parameters()\n",
        "\n",
        "        target_actor_state_dict = dict(target_actor_params)\n",
        "        actor_state_dict = dict(actor_params)\n",
        "        for name in actor_state_dict:\n",
        "            actor_state_dict[name] = tau*actor_state_dict[name].clone() + \\\n",
        "                    (1-tau)*target_actor_state_dict[name].clone()\n",
        "\n",
        "        self.target_actor.load_state_dict(actor_state_dict)\n",
        "\n",
        "        target_critic_params = self.target_critic.named_parameters()\n",
        "        critic_params = self.critic.named_parameters()\n",
        "\n",
        "        target_critic_state_dict = dict(target_critic_params)\n",
        "        critic_state_dict = dict(critic_params)\n",
        "        for name in critic_state_dict:\n",
        "            critic_state_dict[name] = tau*critic_state_dict[name].clone() + \\\n",
        "                    (1-tau)*target_critic_state_dict[name].clone()\n",
        "\n",
        "        self.target_critic.load_state_dict(critic_state_dict)\n",
        "\n",
        "    # def save_models(self):\n",
        "    #     self.actor.save_checkpoint()\n",
        "    #     self.target_actor.save_checkpoint()\n",
        "    #     self.critic.save_checkpoint()\n",
        "    #     self.target_critic.save_checkpoint()\n",
        "\n",
        "    # def load_models(self):\n",
        "    #     self.actor.load_checkpoint()\n",
        "    #     self.target_actor.load_checkpoint()\n",
        "    #     self.critic.load_checkpoint()\n",
        "    #     self.target_critic.load_checkpoint()"
      ],
      "metadata": {
        "id": "KawpsAT-Jujp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Grid_MADDPG:\n",
        "    def __init__(self, actor_dims, critic_dims, n_agents, n_actions, \n",
        "                 scenario='simple',  alpha=0.01, beta=0.01, fc1=64, \n",
        "                 fc2=64, gamma=0.99, tau=0.01, chkpt_dir='tmp/maddpg/'):\n",
        "        self.agents = []\n",
        "        self.n_agents = n_agents\n",
        "        self.n_actions = n_actions\n",
        "        chkpt_dir += scenario \n",
        "        for agent_idx in range(self.n_agents):\n",
        "            self.agents.append(Agent(actor_dims[agent_idx], critic_dims,  \n",
        "                            n_actions, n_agents, agent_idx, alpha=alpha, beta=beta,\n",
        "                            chkpt_dir=chkpt_dir))\n",
        "\n",
        "\n",
        "    # def save_checkpoint(self):\n",
        "    #     print('... saving checkpoint ...')\n",
        "    #     for agent in self.agents:\n",
        "    #         agent.save_models()\n",
        "\n",
        "    # def load_checkpoint(self):\n",
        "    #     print('... loading checkpoint ...')\n",
        "    #     for agent in self.agents:\n",
        "    #         agent.load_models()\n",
        "\n",
        "    def choose_action(self, raw_obs):\n",
        "        actions = []\n",
        "        for agent_idx, agent in enumerate(self.agents):\n",
        "            action = agent.choose_action(raw_obs[agent_idx])\n",
        "            actions.append(action)\n",
        "        return actions\n",
        "\n",
        "    def learn(self, memory):\n",
        "        if not memory.ready():\n",
        "            return\n",
        "        torch.autograd.set_detect_anomaly(True)\n",
        "        actor_states, states, actions, rewards, \\\n",
        "        actor_new_states, states_, dones = memory.sample_buffer()\n",
        "\n",
        "        device = self.agents[0].actor.device\n",
        "\n",
        "        states = T.tensor(states, dtype=T.float).to(device)\n",
        "        actions = T.tensor(actions, dtype=T.float).to(device)\n",
        "        rewards = T.tensor(rewards, dtype=T.float).to(device)\n",
        "        states_ = T.tensor(states_, dtype=T.float).to(device)\n",
        "        dones = T.tensor(dones).to(device)\n",
        "\n",
        "        all_agents_new_actions = []\n",
        "        all_agents_new_mu_actions = []\n",
        "        old_agents_actions = []\n",
        "\n",
        "        for agent_idx, agent in enumerate(self.agents):\n",
        "            new_states = T.tensor(actor_new_states[agent_idx], \n",
        "                                 dtype=T.float).to(device)\n",
        "\n",
        "            new_pi = agent.target_actor.forward(new_states)\n",
        "\n",
        "            all_agents_new_actions.append(new_pi)\n",
        "            mu_states = T.tensor(actor_states[agent_idx], \n",
        "                                 dtype=T.float).to(device)\n",
        "            pi = agent.actor.forward(mu_states)\n",
        "            all_agents_new_mu_actions.append(pi)\n",
        "            old_agents_actions.append(actions[agent_idx])\n",
        "\n",
        "        new_actions = T.cat([acts for acts in all_agents_new_actions], dim=1)\n",
        "        mu = T.cat([acts for acts in all_agents_new_mu_actions], dim=1)\n",
        "        old_actions = T.cat([acts for acts in old_agents_actions],dim=1)\n",
        "        \n",
        "        \n",
        "        for agent_idx, agent in enumerate(self.agents):\n",
        "            critic_value_ = agent.target_critic.forward(states_, new_actions).flatten()\n",
        "            critic_value_[dones[:,0]] = 0.0\n",
        "            critic_value = agent.critic.forward(states, old_actions).flatten()\n",
        "\n",
        "            target = rewards[:,agent_idx] + agent.gamma*critic_value_\n",
        "            critic_loss = F.mse_loss(target, critic_value)\n",
        "            agent.critic.optimizer.zero_grad()\n",
        "            critic_loss.backward(retain_graph=True)\n",
        "            agent.critic.optimizer.step()\n",
        "\n",
        "            \n",
        "\n",
        "            actor_loss = agent.critic.forward(states, mu).flatten()\n",
        "            actor_loss = -T.mean(actor_loss)\n",
        "\n",
        "            agent.actor.optimizer.zero_grad()\n",
        "            actor_loss.backward(retain_graph=True)\n",
        "            agent.actor.optimizer.step()\n",
        "\n",
        "            agent.update_network_parameters()"
      ],
      "metadata": {
        "id": "LT5K0PyIJujq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "nAJcRyENJrwe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Grid main"
      ],
      "metadata": {
        "id": "r_qCgpK_Jske"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.keras.utils import to_categorical\n",
        "def obs_list_to_state_vector(observation):\n",
        "    state = np.array([])\n",
        "    for obs in observation:\n",
        "        state = np.concatenate([state, obs])\n",
        "    return state\n",
        "\n",
        "states_map = {(0,0): 0, (0,1): 1, (0,2): 2, (0,3): 3,\n",
        "                  (1,0): 4, (1,1): 5, (1,2): 6, (1,3): 7,\n",
        "                  (2,0): 8, (2,1): 9, (2,2): 10, (2,3): 11,\n",
        "                  (3,0): 12, (3,1): 13, (3,2): 14, (3,3): 15}\n",
        "\n",
        "\n",
        "env = MAalgo_MAGridWorld(4, 2)\n",
        "obs = env.reset()\n",
        "print(\"Observation: \", env.observation_space)\n",
        "print(\"Actions: \", env.action_space)\n",
        "\n",
        "\n",
        "n_agents = 2\n",
        "actor_dims = []\n",
        "for i in range(n_agents):\n",
        "  actor_dims.append(16)\n",
        "critic_dims = sum(actor_dims)\n",
        "\n",
        "# action space is a list of arrays, assume each agent has same action space\n",
        "n_actions = 5\n",
        "maddpg_agents = Grid_MADDPG(actor_dims, critic_dims, n_agents, n_actions, \n",
        "                      fc1=64, fc2=64,  \n",
        "                      alpha=0.01, beta=0.01,\n",
        "                      chkpt_dir='tmp/maddpg_grid/')\n",
        "\n",
        "memory = MultiAgentReplayBuffer(10000, critic_dims, actor_dims, \n",
        "                  n_actions, n_agents, batch_size=64)\n",
        "\n",
        "total_episodes = 100\n",
        "epsilon = 1\n",
        "epsilon_decay = (0.001/1)**(1/total_episodes)\n",
        "rewards_val = []\n",
        "agent1_route = [[3,0]]\n",
        "agent2_route = [[3,3]]\n",
        "\n",
        "for episode in range(total_episodes):\n",
        "  obs = env.reset()\n",
        "  obs_new = []\n",
        "  total_steps = 0\n",
        "  for state in obs:\n",
        "    state = states_map[tuple(state)]\n",
        "    one_hot = to_categorical(state, 16)\n",
        "    obs_new.append(one_hot)\n",
        "  obs = obs_new\n",
        "  done = [False]*n_agents\n",
        "  total_rewards = [0, 0]\n",
        "\n",
        "  while not any(done) and total_steps < 20:\n",
        "      \n",
        "      #time.sleep(0.1) # to slow down the action for the video\n",
        "      # print(\"Grid w obs: \", obs)\n",
        "      \n",
        "      actions = maddpg_agents.choose_action(obs)\n",
        "      # print(\"action prob: \", actions)\n",
        "      actions_new = []\n",
        "\n",
        "      for action in actions:\n",
        "        rand_num = np.random.random()\n",
        "        if epsilon > rand_num:\n",
        "          # actions_new.append(np.random.choice(5))\n",
        "          actions_new.append(np.argmax(action))\n",
        "        else:\n",
        "          actions_new.append(np.argmax(action))\n",
        "\n",
        "        # actions_new.append(np.random.choice(5,1,p = action)[0])\n",
        "\n",
        "      # print(\"Actions_new\", actions_new)\n",
        "\n",
        "      obs_, reward, done, info = env.step(actions_new)\n",
        "\n",
        "      if episode == total_episodes - 1:\n",
        "        agent1_route.append(obs_[0])\n",
        "        agent2_route.append(obs_[1])\n",
        "        # print(\"Actions: \", actions)\n",
        "      \n",
        "      obs_new_ = []\n",
        "      for state in obs_:\n",
        "        state = states_map[tuple(state)]\n",
        "        one_hot = to_categorical(state, 16)\n",
        "        obs_new_.append(one_hot)\n",
        "      obs_ = obs_new_\n",
        "\n",
        "      state = obs_list_to_state_vector(obs)\n",
        "      state_ = obs_list_to_state_vector(obs_)\n",
        "\n",
        "      # if total_steps >= MAX_STEPS:\n",
        "      #     done = [True]*n_agents\n",
        "\n",
        "\n",
        "      memory.store_transition(obs, state, actions, reward, obs_, state_, done)\n",
        "\n",
        "      if total_steps % 2 == 0:\n",
        "        maddpg_agents.learn(memory)\n",
        "\n",
        "      obs = obs_\n",
        "      total_steps += 1\n",
        "\n",
        "      for agent in range(2):\n",
        "        total_rewards[agent] += reward[agent]\n",
        "\n",
        "  \n",
        "  if epsilon > 0.001:\n",
        "    epsilon *= epsilon_decay\n",
        "  else:\n",
        "    epsilon = 0.001\n",
        "      \n",
        "  rewards_val.append(total_rewards)\n",
        "    \n",
        "  #After every episode\n",
        "  # avg_reward = np.mean(rewards_val[-10:])\n",
        "  \n",
        "  if episode % (total_episodes // 10) == 0 and episode > 0:\n",
        "      print('Episode: {}, Individual Rewards: {}'.format(episode, total_rewards))\n",
        "      \n",
        "\n",
        "\n",
        "x = [episode for episode in range(1, total_episodes+1)]\n",
        "yr1 = [rewards_val[episode][0] for episode in range(total_episodes)]\n",
        "yr2 = [rewards_val[episode][1] for episode in range(total_episodes)]\n",
        "\n",
        "#Plots showing episodes vs epsilon, episodes vs rewards\n",
        "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10,4))\n",
        "\n",
        "#episodes vs rewards\n",
        "ax1.plot(x,yr1)\n",
        "ax1.set_title(\"Agent1: Rewards per episode\")\n",
        "\n",
        "#episodes vs rewards\n",
        "ax2.plot(x,yr2)\n",
        "ax2.set_title(\"Agent2: Rewards per episode\")\n",
        "\n",
        "print(\"Agent1 Route: \", agent1_route)\n",
        "print(\"Agent2 Route: \", agent2_route)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 524
        },
        "id": "WuS2c6quBozu",
        "outputId": "b6b468ea-a6f9-4725-abc7-8c1b18e1aa51"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
            "Observation:  Box(4, 4)\n",
            "Actions:  Discrete(5)\n",
            "Episode: 10, Individual Rewards: [-120, -100]\n",
            "Episode: 20, Individual Rewards: [-120, -120]\n",
            "Episode: 30, Individual Rewards: [-120, -120]\n",
            "Episode: 40, Individual Rewards: [-120, -120]\n",
            "Episode: 50, Individual Rewards: [-120, -120]\n",
            "Episode: 60, Individual Rewards: [-120, -120]\n",
            "Episode: 70, Individual Rewards: [-120, -120]\n",
            "Episode: 80, Individual Rewards: [-120, -120]\n",
            "Episode: 90, Individual Rewards: [-120, -120]\n",
            "Agent1 Route:  [[3, 0], [2, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [1, 0]]\n",
            "Agent2 Route:  [[3, 3], [2, 3], [1, 3], [1, 3], [1, 3], [1, 3], [1, 3], [1, 3], [1, 3], [1, 3], [1, 3], [1, 3], [1, 3], [1, 3], [1, 3], [1, 3], [1, 3]]\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 720x288 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAl4AAAEICAYAAACOHvrxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3df5icdXnv8ff9zOxssgGSkATkVwgCWtG2qKk/Wqq0agXaivZUBa2ix5aeq3rUnvZUOT291FO5Lu1pte2RqmmrIBYRrQoH8RdWUU9FjK1SEKgpBBMIkoSQbH7s7vy4zx/P95l9dnZmdzY7OzPfZz+v69orOzPPznxnkr1zP9/v/b0fc3dEREREZOklgx6AiIiIyHKhxEtERESkT5R4iYiIiPSJEi8RERGRPlHiJSIiItInSrxERERE+kSJl0THzM43s52DHkcvmdkXzOyyHj/nO83s4718ThFZHMWvrp+zsPFLiVcPmdnXzWyfmY328TXdzM7K3a6Y2afNbHt47PwFPt/XzWzCzA6a2R4z+4yZndTzgcsM7n6hu18z6HHI8jUk8es5ZvYVM3vMzHab2acWEn8UvwZD8WthlHj1iJltAn4RcOAlAx0MfAv4LeCRo/z5N7n7McBZwDHAn/dqYAtlZuVBvfZczKw06DGI9MoQxa+1wBZgE3A6MA58dIHPofg1D8WvwVLi1TuvBW4HrgZmTLma2Toz+79mdsDMvmtm7zazb+Ue/6ncWd59ZvaK3GNXm9lVZvZ5Mxs3s++Y2ZnhsW+Ew34QzvBe6e5T7v6X7v4toL6YN+TujwOfA86db6xmdoaZPW5mSbj9t2b2aO7nrjWzt4bvX29m94T3c7+Z/W7uuPPNbKeZvc3MHgE+amYrw+ewz8x+CPxcy+f7NjN7KDzffWb2gnbvJzzHh8L4x83sNjM7fb73lvvZD5rZLWZ2CPilNs+/2sz+3sx2hfG8OwtwZvY6M/t/ZvYBM9tvZvfmxxnO1H87fH9WGNv+cNb+ydxxPx/+De0Pf/587rEzws+Nm9lXgPUt43uOmf1z+Hv6gS1wNlQKbVji1xfc/VPufsDdDwMfAH7haN6Q4pfi19Byd3314AvYBvwe8EygCpyYe+z68DUGnAPsAL4VHlsVbr8eKANPB/YA54THrwb2As8Kj/8DcH3uuR04q8OYdgLnt9z3KuDOOd7H14HfDt+vA24FbuxyrD8Gnhm+vw+4H3hK7rGnh+9/FTgTMOD5wGHgGeGx84Ea8F5gFFgJvAf4JnA8cBpwF7AzHP/kMKaTw+1NwJkd3tvVpGfQzwvP/VcL/HvYT/qfQAKsaPP8nwU+HJ7rBOAO4HfDY68L7+v3gRHgleH5jm/zuX8C+OPsdYDzwv3HA/uA14QxXhpurwuPfxt4X3hvzwvv9ePhsVNI/x1dFJ73ReH2hkH/7uhr8F8MYfwKj78VuD13W/FL8Sv6+DXwARThCziPNFitD7fvBX4/fF8Kjz05d/y7c78wrwS+2fJ8HwbeEb6/Gvi73GMXAffmbi8o8erivXydNJDsD8/9fWBjl2O9FvhvwBNIA9efAf8FOAN4HEg6vObngLeE788HpvKBgTQAXpC7fTnTgess4FHghcDIPO/tamYG/WNIZwVP6/Lv4WNzPPeJwCSwMnffpcDXwvevAx4GLPf4HcBrcp97Frg+RrrccmrLa7wGuKPlvm+H595IGhhX5R67junA9Tbg2paf/RJw2aB/f/Q12C+GN379DPAY8IsLeC9fR/Gr09+D4teQfGmpsTcuA77s7nvC7euYnq7fQJrd78gdn//+dODZYfr0cTN7HHg16S9/Jl+rdZj0F24pvdndV5MGvrXAqV2O9TbSwPM84Bukv4zPD1/fdPcGgJldaGa3hynxx0mDcX5aebe7T+Run8zMz+zB7Bt330Z6VvxO4FEzu97MTp7jvTWfx90Pkgb2k7t4bzN+to3TSc8Ed+V+/sOkZ46ZhzxEjNz7aDfWPyI9m77DzO42s/8c7j+Z3HvPPccp4bF97n6o5bH8+F7e8v7OA1R4LEMXvywtuP8CaULzzYW8GRS/FL+G3FAW/sXEzFYCrwBKYU0f0qnSNWb2s6TTyjXSX/5/D4+flnuKHcBt7v6iPg25a+7+b2b2buAqM3sG84/1NuB/k8603UZa5P8hYCLcxtIdU/9IWlNyo7tXzexzpL+ozZdued5dpJ/Z3eH2xpZxXgdcZ2bHkQaL95KeXbXT/OzN7BjS6e+Hu3hv7caVt4P0jHG9u9c6HHOKmVkueG0Ebpr1Iu6PAL8TxngecKul9TAPkwagvI3AF0k/o7VmtioXvDbmxryD9Izxd+Z4D7LMDGP8CnVLtwJ/6u7XHu3zKH7Novg1JDTjtXgvJZ3uPYe0iPNc4Cmka/qvdfc68BngnWY2ZmY/RfpLm7kZeJKZvcbMRsLXz5nZU7p8/Z8AT8zfYWajZrYi3KyY2Qozs9k/2pVrSKehXzLfWN39R8AR0h2Vt7n7gTC+/0QIXECFNLDvBmpmdiHwK/OM4QbgCjNba2anAv81916fbGa/HALiRHj9xhzPdZGZnWdmFeBPSetHdsz33ubj7ruALwN/YWbHmVliZmea2fNzh50AvDk898tJ/53c0vpcZvby8D4hrYHw8J5uCWN8lZmVzeyVpP/ubnb3B4GtwLssbSlyHvDruaf9OPDrZvZiMyuFfxPn515Hlqehil9mdgrwT8AH3P1Di31zKH4pfg0hJV6LdxnwUXf/sbs/kn2R7sZ5taXbid8ErCadcr+WtPhwEsDdx0l/cS8hPSN4hOnCzG68E7gmTL9mu1juI/0FPoV0HfwI4UzDzF5tZne3e6J23H2KtIjzT7oc623A3hAMstsG/Evu/b6ZNBjtIy2WnXXW1OJdpNPOD5AGh/xZ8Chp8eqeMJ4TgCvmeK7rgHeQTtE/kzTI9uLvAdL/kCrAD0nf26eZORX+HeDsMNYrgd90971tnufngO+Y2UHSz+Yt7n5/OPbXgD8gLSz9I+DXcktErwKeHd7bO0hrLQjvbwdwMfA/SP/T2AH8dxQDlrthi1+/TZqIvdPSnY4Hw+8BoPiF4lch4pfNXLKVfjCz9wJPcPfL5j1YesbMriYtav2fA3jt15EWn57X79cW6SXFr8FQ/CqOKLPF2FjaX+VnLPUs4A2kW3dFRIaa4pdIb6m4vj+OJZ2eP5m0ZuAvgBsHOiIRke4ofon0kJYaRURERPpES40iIiIifRLFUuP69et906ZNgx6GiPTR9773vT3uvmHQ4+gFxTCR5WWu+BVF4rVp0ya2bt066GGISB+ZWWuX62gphoksL3PFLy01ioiIiPSJEi8RERGRPlHiJSIiItInSrxERERE+kSJl4iIiEifKPESERER6RMlXiIiIiJ9suwTr8/960McnKwNehgiIl276QcPs/9IddDDEJGjsKwTr537DvPWT36fL9/9yKCHIiLSlT0HJ3nzJ/6Vm+98eNBDEZGjsKwTr4lqHYCpWmPAIxER6U4WtyarilsiMVrWiddUzQGoNXzAIxER6U617uFPJV4iMVrWiVcWuBquxEtE4pDFLZ0wisRJiRdQqyuAiUgcstIIzXiJxGlZJ15ZAKvrzFFEIqETRpG4Le/EKwSwupYaRSQSzRqvhma8RGJU2MRr1/4jXPvt7XMekwUwzXiJSCyyGa9qTXFLJEaFTbw+f+cu/uTGuzkw0bnJoJYaRSQ2U83ies14icSosInXVBd1ENodJCKxqTaL6xW3RGJU2MQrS7jmOits1njpzFFEIpElXDXtahSJUoETr9Cja47YVG0mXv0YkYjI4k3V0871mqkXiVNxE69GFzNeNc14iUhcsqJ69fESiVPhE6+5Cuc14yUisclKJJR4icSpsIlXN4Xz0+0kFMBElisz+4iZPWpmd+XuO97MvmJmPwp/rg33m5n9tZltM7M7zewZ/R6vGqiKxK2wiVcWlBpzJF6TNTVQFRGuBi5oue/twFfd/Wzgq+E2wIXA2eHrcuCDfRpjU7OPl2q8RKJU3MSrWePVzVKjApjIcuXu3wAea7n7YuCa8P01wEtz93/MU7cDa8zspP6MNKVdjSJxK27i1UVSlfXD0ZS9iLQ40d13he8fAU4M358C7MgdtzPcN4uZXW5mW81s6+7du3s2sCnFLZGoFTfxWsiMl5YaRaQDd3dgwUHC3be4+2Z337xhw4aejWd6qVEzXiIxKnziNdeM15SWGkWkvZ9kS4jhz0fD/Q8Bp+WOOzXc1zdV7WoUiVpxE68ukqqpmi6SLSJt3QRcFr6/DLgxd/9rw+7G5wD7c0uSfTFd46W4JRKjniRew7gdu9rFJYNUXC8iZvYJ4NvAk81sp5m9AXgP8CIz+xHwwnAb4BbgfmAb8LfA7/V7vOrjJRK3co+e52rgA8DHcvdl27HfY2ZvD7ffxszt2M8m3Y797B6NoylLuOae8dJFskWWO3e/tMNDL2hzrANvXNoRza2quCUStZ7MeA3jduz6Aorr5+r1JSIyTNRAVSRuS1njtajt2Ivdit1NUjXVRXd7EZFhkpVRaKlRJE59Ka4/mu3Yi92KXasvYMZL7SREJBKq8RKJ21ImXgPdjt1VOwk1IhSRyGipUSRuS5l4DXQ7dlZc391FshXARCQOaqAqEree7GoM27HPB9ab2U7gHaTbr28IW7MfBF4RDr8FuIh0O/Zh4PW9GEOrbi6Src71IhKbak19vERi1pPEaxi3Y1e7KJxXOwkRiU1+U5C7Y2YDHpGILERhO9fXmzVenafjp9ROQkQiky+q10mjSHwKm3hVF7CrUcFLRGKRT7y0s1EkPoVNvLLi+rlrvOavAxMRGSbVXG1XVXVeItEpbOLVTef66RovnTWKSByyuAVQ04yXSHQKm3h10ypiShfJFpHIqMZLJG6FTbxq89RvubvaSYhIdKr1BitGkub3IhKXwiZe1Xk616dbsdPv66qTEJFIVOvOWCXtBKReXiLxKWziVZ8n8cqfKWrGS0RiMVVvsHKkBGjGSyRGhUy83H3e4vqs+zOoxktE4pCVSKwazRIvxS6R2BQy8coHo04NVLPC+sRUoCoicaiHEonmUqN2ZItEp5CJV34Gq1NSlSVeK0dKmvESkShkJ5VjFc14icSqkIlXNXcW2Kk5ajX0wllZUeIlInHIenhliZf6eInEp5CJV36nT8carxCwVmjGS0Qi0Zypby41KnaJxKagiVdux2KHwDRZ01KjiMQlO2EcC7sapzTjJRKdYiZeXdR4VevTS406axSRGDQTr9FsqVGxSyQ2xUy8csGoY41XOGZFOHPUhbJFZNg1Ey/VeIlEq5CJV764ft4Zr5B4adZLRIbdVC3b1ZjWeFUVt0SiU8jEK1+z1al+K9sdlF3zrKHu9SIy5GadMGrGSyQ6hUy88pfR6KaP11zHiYgMi9lLjYpbIrEpZOJV66Jzfb64Pj1OAUxEhttUS9zSrkaR+BQz8WrM304i38drruNERIZFtiloVdbHS4mXSHSKmXjVu6/xml5qVAATkeFWbe1crxNGkegUM/Hq6lqN6f0rm+0kln5cIiKL0VoioWs1isSnkIlXtYvO9flrNYJmvERk+E01i+u11CgSq0ImXlmyNVpOOi81ttR4Ke8SkWFXbZmpVx8vkfiUBz2ApZAFp9Fy0rmBam1m4qUZLxFpZWbbgXGgDtTcfbOZHQ98EtgEbAde4e77+jGebDa/Uk4YKdmM2X0RiUMhZ7yyJGrFHBfArtYbmKUBDLSrUUQ6+iV3P9fdN4fbbwe+6u5nA18Nt/siS7RGSkY5SbTUKBKhYiZe2YzXSOcZr6m6UykllBMDoK7O9SLSnYuBa8L31wAv7dcLZ7uxR8oJ5ZKpuF4kQsVMvEKytaJc6njx66lag0opoRQSL3WAFpE2HPiymX3PzC4P953o7rvC948AJ7b7QTO73My2mtnW3bt392QwWaJVKSWMlBKVSIhEqJA1XrVc4fxcF8keKSeULE28dK1GEWnjPHd/yMxOAL5iZvfmH3R3N7O2wcPdtwBbADZv3tyTADO91JjO1uuEUSQ+Sz7jZWbbzezfzOz7ZrY13He8mX3FzH4U/lzby9esztjV2PmSQZVSQqkUZrxU4yUiLdz9ofDno8BngWcBPzGzkwDCn4/2azzVeoPEoJQYI6VES40iEerXUmNfi1Pr4axwzhqvWoORsjVrvDotSYrI8mRmq8zs2Ox74FeAu4CbgMvCYZcBN/ZrTFP1BiOlNGxrV6NInAa11HgxcH74/hrg68DbevXk+Rqvufp4jZSmlxo14yUiLU4EPmtpjCgD17n7F83su8ANZvYG4EHgFf0aULWWbgoCKKvGSyRK/Ui8suJUBz4c6h7mLU4NhayXA2zcuHFBL1jN7Wqcq51Evrhe7SREJM/d7wd+ts39e4EX9H9E07WpAOVEuxpFYtSPxOuoilMXU5iaFdePzjHjVa07lbISLxGJR3bCCGmBvfp4icRnyWu8BlGc2lxqnK/GSzNeIhKRqXpamwpQLplKJEQitKSJ16CKU2uNBqUk7ew8d41Xegwo8RKR4Vet+3RxfZKouF4kQku91DiQ4tRa3SknRimxOWu8jl0xQsi7dOYoIkOvWsstNZaNiaoSL5HYLGniNaji1Fpj/sQr7VyvGS8RiUc1104ivVZjbcAjEpGFKuYlg+oNylln5zkaqKY1XultXatRRIZdViIBWR8vxS2R2BQy8ao2nJHSfEuNHhKvbMZLU/YiMtxmzXgpbolEp5CJV62eFtfPu9RYTpqd61WjKiLDLmuDA2FXo2a8RKJTzMSr4ZSTtFVEw9tfDijrXJ80Ey9lXiIy3LI2OJD28aoqbolEp5iJV91Dq4iQVLWp30obEU4fo12NIjLsqq01XjXFLZHYFDPxaqTF9aU5dixWw1JjYrpItojEIX+RbF2rUSROxUy8Qh+v8hxd6bMAphkvEYnFjEsG6VqNIlEqZuLVcMola9ZvtSZV7j69q7GkSwaJSByqNZ8546VdQSLRKWTiVa03KCdJxxmv7CyxUk4omRIvEYlDteVajVXFLZHoFDLxyl8yCJhVB5Fd36ySv0i2GqiKyJDL13iNJJrxEolRIROvelhq7DTjNVVLg1XWZBWgrloJERlyM2q8SgkN12y9SGwKmXhVG609ulqXGkPilVtqVHG9iAy7rDYV0qXG9D7NeonEpJCJ13y7GqdyS41JYiQGDS01isgQqzecesNzDVR10igSo2ImXg2nlCS5Gq/2S43ZpTdKiSl4ichQm56pD8X1oU+h6rxE4lLMxCt0dy53aKCa7WrMzhxLiamBqogMtfymIJie8VIvL5G4FDPxanjoXJ/e7ljjlSVephkvERlurSeM5fCnarxE4lLIxKtabzCSWMdLBjVrvHJLjdoZJCLDrPWEMfuzphkvkagUMvGqN5xS0vkC2Pl2EpCeOSrxEpFh1hq3mkuNul6jSFQKmXhV69lSY7arsXMDVYBES40iMuSqLTP108X1il0iMSlk4lVrNGY2R205IZwdwFRcLyLDbXaNl/p4icSokIlXvZ4uNXa6ZNBUbfauRs14icgw67SrUbFLJC6FTLyyzvXzNVDNJ16ty5EiIsNkKnfFDZheatSMl0hcCpl4zb5Idks7idrMM8dSYqhMQkSGWXVWcb0SL5EYFS7xcvdmH69mA9V6+z5eM9tJKHiJyPDKarxmLTXqrFEkKoVLvLJlxXJiJFkDVe+01GjNY9VOQkSGWWsfr6yBamsNq4gMt8IlXtmyYnmOSwY1++GUp9tJKPESkWHWWpua1bDqkkEicSlc4tU8K5zjItkT1ToAK8olIE3SlHiJyDCbLpGYWeOlpUaRuBQu8coSqHzn+tb6rfGJGqPlpFnjpQaqIrIQZnaBmd1nZtvM7O39eM3ZS43q4yUSo8IlXtNNBjs3UD0wUePYFSPN26rxEpFumVkJuAq4EDgHuNTMzlnq16229B+saFejSJQKl3hlhaZzXTJofKLKcSvKzdu6SLaILMCzgG3ufr+7TwHXAxcv9YvOqvFSA1WRKBUv8apP72rsdJHs8YkaxyrxEpGjcwqwI3d7Z7hvBjO73My2mtnW3bt3L/pFHzs0BdCMXdPXatSMl0hMBpZ4LVWNRH5XY6lD5/rxiSrHrZxeakwbqCrxEpHecfct7r7Z3Tdv2LBh0c+3fe8hTlq9ghUj6aagkZJ2NYrEaCCJ11LWSGRnf+Uk6Zh4HdCMl4gcvYeA03K3Tw33Lantew5x+rqx5m318RKJ06BmvJasRqJ9cf3sGa9jR2cW12tLtoh06bvA2WZ2hplVgEuAm5b6RR/ce5gz1q9q3taMl0icBpV4zVsjcbT1EdPtJKYvGdRNjVdDS40i0gV3rwFvAr4E3APc4O53L+VrHpiosvfQFKevyyVeuki2SJTK8x8yGO6+BdgCsHnz5q6zompzV2P7Ga9avcHhqfqMdhKlRH28RKR77n4LcEu/Xu/Hew8DsCm31JgkRmJqoCoSm0HNeC1ZjUQWhGZ0rs8FpoOTNYCWGa+EhhIvERlSD+w5BDBjxgvSOq+qarxEojKoxGvJaiSaxfWl9GwQZl4ke3yiTeJl6oUjIsPla/c+yqFwovjg3izxGptxzIjqU0WiM5DEaylrJJrtJBLDzEJX+ukzwgMTVYCWpcZEuxpFZGg8dmiK11/9XbZ8434Atu89zInHjTJWmVkdUi4l6uMlEpmB9fFy91vc/Unufqa7X9mr5813rofZ9VvZjFe+c70uGSQiw+Txw2mz1K//e7qx6MG9h2YtM0LaxX5KM14iUSlc5/pqrnN99me93m6pcXrGK1FxvYgMkSxO3bnzcR47NMUDew7PKKzPjJRMM14ikSlc4pXNXGXXM0tautKPN5caZ854qZ2EiAyLbBOQO3zxrkfYc3CSTetnz3iVSzppFIlN4RKvrKdNKT/j1WapsbWPl84aRWRYZCeIAB/79nYANrVbakwS9fESiUzhEq9arnM9pIXzM2u82hXXGzppFJFhkZ0gPmPjGu59ZByYvaMRwoyXarxEolK8xKuluL61xuvARI3RckKlPP3Wy4npemciMjSyxOtXf+bk5n3tiuvLSaLYJRKZAiZeM4vrS21qvI5bOTLjZxLtahSRIZLVeP3qT58EwPpjRjlmdPaFRkbK2tUoEpuhvWTQ0arV2yRejZkzXvn6ruxYJV4iMizGJ6qsGEl4wuoVnHvaGlaNltoeN6L6VJHoFC7xqtZnLzW29vHK13cBJJbWeLk7Zta/wYqItHFwcjpObXnNMzseVyknTNWUeInEpHCJ13Q7ifyM13RgGp+ozmieCtOzY/WGUy4p8RKRwTowUePYsLR4wnErOh5XKSfNZUkRiUNha7xKuaXGWr11xmtm4lUKyZb64YjIMDjYJk61UylpxkskNoVLvLKlxpFk+pJBjZbi+mNHZy41lmx6xktEZNDGJ6oc00XiNTpSYlKJl0hUCpd41epOYulORehU49Uy45UtNap7vYgMgYOTtVkniO1oxkskPsVLvBpOOZl+W/ldjbV6g8NT9VnF9c3ES9uyRWQIjE/UuprxqpQTzXiJRKZ4iVe9MaNAPl/jlRWhtmsnAZrxEpHh0G2N12g5YapW78OIRKRXipd4NbyZSMHMBqrtrtMI08uSqvESkUFrNJyDU9O7GueiGS+R+BQw8WowUspfDihpJlQH2lynMT1GiZeIDIdDUzXcZ8epdkbLCVP1Bq7ZepFoFC/xqnuzZgvCUmOWeB1JZ7xa+3iVQk2YEi8RGbRsZr6rGq9Sgrta4YjEpHCJV7XuLTNe0w1UxzvMeGWHK3iJyKB1qkVtp1JOg5d2NorEo3CJV60xs7g+vQB2+n2nGi/NeInIsMhOENtdFLuVEi+R+BQw8Zq51Nh+xqsl8VIDVREZEp1OENsZLacXz1aBvUg8ipd41RvNrvUws8ZrOqB16OOlxEtEBqxTnGpHM14i8Slg4jXzQtflXAPV8ckaK0aSZrDKHwNKvERk8LIarwUtNdbVy0skFsVLvBpOOVdcn+QaqI5PVNueRWYzXrWGzhpFZLAOLmCpsRJinZYaReJRwMSrMaOBajl3kewDHbpBZ4lXQ71wRGTAxieqmMGqSjcXyVbiJRKbwiVe1Xpr5/pkRo3XnDNeulajiAzY+GSNYyrl5hU15jJaUo2XSGwKl3jV6q2d63M1XhPVWc1TIVdcrxkvERmwbi+QDSquF4lRd7/dEbn8eU+k3LqrMTTyevxwlVPWrJz1MyquF5Fh0e0FskGJl0iMCpd4XfC0k2bcLiVGlk/tOzzF2rHKrJ9JmsX1SrxEZLDGJ6td7WgE9fESiVHhlhpblROj1mjQaDj7j1RZOza7xiub8Woo8RIRwMzeaWYPmdn3w9dFuceuMLNtZnafmb241699sEMtajtqJyESn8LNeLUqhRqvAxNV3GF1uxkv04yXiMzyfnf/8/wdZnYOcAnwVOBk4FYze5K79yzzGZ+ocerxY10dq6VGkfgs2YzXIM8Y87LO9fsOp5cLajvjVdKMl4h05WLgenefdPcHgG3As3r5AuOTtbabgNqpaFejSHSWeqnx/e5+bvi6BWadMV4A/I2ZlZZqAKXEcIfHDk0BtK3xKqvGS0Rme5OZ3WlmHzGzteG+U4AduWN2hvtmMbPLzWyrmW3dvXt31y86PrGAGi/18RKJziBqvJb8jDEvS6r2HJwEYE2bGa9sqVENVEWWDzO71czuavN1MfBB4EzgXGAX8BcLfX533+Lum91984YNG7r6mWq9wUS10X2NlzrXi0RnqWu83mRmrwW2An/g7vtIzw5vzx3T8YyxF0qhtUSWeLWf8UqPUQNVkeXD3V/YzXFm9rfAzeHmQ8BpuYdPDff1RHa5oG5nvLTUKBKfRc14LeUZ49FO07dqzniNd15qzNp+qY+XiACYWb4vzcuAu8L3NwGXmNmomZ0BnA3c0avXzS6Q3W0fryQxRkrGVF2Jl0gsFjXjtZRnjO6+BdgCsHnz5qPOiLIeXXsPTZJY+4CWzXipc72IBH9mZucCDmwHfhfA3e82sxuAHwI14I293NF4YCLdBNRt4gXprNdkVYmXSCyWbCQGd0sAAA47SURBVKnRzE5y913hZusZ43Vm9j7S7dg9PWNslc147T04xeqVI22vf1ZScb2I5Lj7a+Z47ErgyqV43emlxu5qvABGR0rq4yUSkaWs8RrIGWOrLKnafXCy7TJj/hi1kxCRQRqfWNhSI6QzXqrxEonHkiVegzpjbJXf1dhuRyNoxktEhkNW49XtRbIhbaKqxEskHoW/ZFCSW2qcb8ar3lDwEpHBGT+aGq9youJ6kYgU/pJB2YzX/iNV1nRIvMrNxKtvwxIRmeXFT30CZ244puNJYjujZRXXi8Sk8IlXKVdM3+5yQfljNOMlIoN0wnErOOG4FQv6Gc14icSl8EuNWasIaN+1HqBkmvESkThVSok614tEpPCJVyn3DjstNSaJYaYZLxGJj4rrReKyDBKv6bc4V91EyUy7GkUkOqPlkma8RCJS+MSr3EWNF6R1XupcLyKxGS0nTNXUQFUkFoVPvPLF9Z2WGrPj6rpItohERsX1InFZVonX2lWa8RKRYlHnepG4LKvEa83KzjNe5cSoq8ZLRCIzOqJdjSIxKXzildV4jZYTVlZKHY8rKfESkQhpxkskLoVPvLIZr/k6QSvxEpEYqZ2ESFwKn3hlDVQ7NU/NqJ2EiMSoUk6oNVwnjiKRKHzilbXxmnfGq2Q0FLhEJDKVchrkNOslEofCJ17ZjNdcOxqz4zTjJSKxGS2ntatKvETiUPjEK6vxWj3HjkaAxFA7CRGJTjbjNVlXE1WRGJQHPYClVm4W188/46UGqiISm9FSb5car/raNm695ycAnH78GO9/5bmY2Tw/JSLdWjYzXvPVeCWJ8R+7D/Kh2/6Dz9+5qx9DExFZtOaMV48Sr4/f/iCP7J/gwJEqn/v+wxw4UuvJ84pIqvCJ1/GrKpy0egVPO2X1nMc9cf0qfvToQd7zhXt543X/wqFJBRsRGX6jPSyun6jW2bV/gkuftZHfO/8sAB4/MrXo5xWRaYVPvFaNlvn2FS/guWeum/O4D7zq6dzzvy7g3S99GgCPHVKwEZHh18tdjT9+7DAAp68ba7bgefxwddHPKyLTCp94dcvMWFkpccKxo4CCjYjEoZl49eBC2dv3HAJg07pV04nXEcVCkV5S4tViTagF0/S6iMSgEorrJ6uLT7we3JvOeG1at6q5E/zxw4qFIr1U+F2NC5XtftynGS8RicDoSOjj1YN2Eg/sPcSasRFWj41QbaSJ3H7NeIn0lGa8WjRnvHSWJyIRqPSwncSDew+xad0qAFavDCehh5R4ifSSEq8WWV2Dgo2IxKCX7SS27znMpnVjAIyUEo4ZLavsQqTHlHi1GCklHDtaZp9mvEQkAr1qJzFZq/Pw/iOcHma8IJ312q+yC5GeUuLVxppVI1pqFJEojPZoxmvHY0dwh03rx5r3rRkb0a5GkR5T4tXG2rGKiutFJAq96uOVtZLIz3itHavoJFSkx5R4tbF6pc7yRCQOverjtX1vmnidkV9q1IyXSM8p8WpDZ3kiEote7Wp8cO9hjltRbm4wAlijGi+RnlPi1cbasRH26ZJBIoVnZi83s7vNrGFmm1seu8LMtpnZfWb24tz9F4T7tpnZ2/s/6pnKpYTE0uL4xdi+9xCb1q/CzJr3ZTVe7r7YYYpIsKjEqwhBq501YxUOTNSo9eASHCIy1O4CfgP4Rv5OMzsHuAR4KnAB8DdmVjKzEnAVcCFwDnBpOHagRsulnsx45eu7ANasrFBvOAcna4t6bhGZttjO9VnQ+nD+zpagdTJwq5k9KTx8FfAiYCfwXTO7yd1/uMhx9FTWvX7/kSrrjhkd8GhEZKm4+z3AjFme4GLgenefBB4ws23As8Jj29z9/vBz14djBxrDKuWkmXj9y4/3sffgwmbs3Z2HHj/CxeeePOP+1bkLZR+7YqTdj4rIAi0q8SpK0Gq1dlXavX7fYSVeIsvUKcDtuds7w30AO1ruf3a7JzCzy4HLATZu3LgEQ5xWKSdM1Rs8sn+C3/ibfz7q53nKScfNuL1m5fRJ6GmLGqGIZJbqWo1RBa1Wq5vBRnVeIrEzs1uBJ7R56I/d/calel133wJsAdi8efOSFklVSgmT1QbbHj0IwHt+46d52imrF/Yc5YSzTzhmxn3Tl1BTgb1Ir8ybeC2HoNVqbQg2umyQSPzc/YVH8WMPwYxJnlPDfcxx/8CMjiRM1hvNlhDPf/IGTlq9ctHPm+1w1GWDRHpn3sRrOQStVs3ESy0lRJarm4DrzOx9pHWqZwN3AAacbWZnkMauS4BXDWyUQaWU1ng9uPcQo+WEE49d0ZPnbV67VjNeIj2zVO0kbgIuMbPREKCyoPVdQtAyswpp0LppicZw1Nasmi4oFZHiMrOXmdlO4LnA583sSwDufjdwA2n96ReBN7p73d1rwJuALwH3ADeEYwdqNBTXb997mNPXjZEks+puj0qz7EInoSI9s6gaLzN7GfB/gA2kQev77v5id7/bzLKgVSMErfAzWdAqAR8ZhqDV6tjRMuXENOMlUnDu/lngsx0euxK4ss39twC3LPHQFqRSTpis1dnVcpHrxRotlxirlHQSKtJDi93VWIig1crMWDM2oul1EYnCaLnEoakaD+49zPOftKGnz71Gl1AT6Sl1ru9gjS4bJCKRqJQTdjx2mMlag03rezfjBbB6rKIZL5EeUuLVwZqVIwo2IhKFSilhT2iauqmHS40QrteoXY0iPaPEq4M1YxXVeIlIFCrl6VB++rqxnj73mjGdhIr0khKvDtYq2IhIJEZD4lUpJT3p35WXXShbRHpDiVcHa1dpxktE4pDNeG1cN0apR60kMqtXVth/uIp7X/tYixSWEq8O1oyNMFlrcGSqPuihiIjMKUu8NvV4mRHS2f+peoMjVcVCkV5Q4tWButeLSCyyxKuXPbwyzcsGqfRCpCeUeHWwZqWCjYjEYbS0dDNeq1fqQtkivaTEq4M1Y1mw0YyXiAy30ZESsMQzXmopIdITi+pcX2Rrw/Ua//BTP2DVqD4mkV5410ueys+ftX7QwyicbFfjGT1ungrTidcf3qBYKMvbeWev5x2//tRFP49+izo4c8Mx/NZzNvLYIZ3lifSK/uNeGr9yzhM4OFnj1LW9bSUBaSx89bM3qt5Vlr2Te9SqRVGwg5FSwrtf+tODHoaIyLw2rhvjrS980pI890gp4cqXKRaK9IpqvERERET6RImXiIiISJ8o8RIRERHpEyVeIiIiIn2ixEtERESkT5R4iYiIiPSJEi8RERGRPlHiJSIiItIn5u6DHsO8zGw38GAXh64H9izxcJZCrOOGeMce67hh+Yz9dHffsJSD6ZcuY9hy+XsdJrGOG+Ide6zjhh7FrygSr26Z2VZ33zzocSxUrOOGeMce67hBYy+qmD+bWMce67gh3rHHOm7o3di11CgiIiLSJ0q8RERERPqkaInXlkEP4CjFOm6Id+yxjhs09qKK+bOJdeyxjhviHXus44Yejb1QNV4iIiIiw6xoM14iIiIiQ0uJl4iIiEifFCLxMrMLzOw+M9tmZm8f9HjmYmanmdnXzOyHZna3mb0l3H+8mX3FzH4U/lw76LG2Y2YlM/tXM7s53D7DzL4TPvtPmlll0GNsx8zWmNmnzexeM7vHzJ4bw2duZr8f/p3cZWafMLMVw/qZm9lHzOxRM7srd1/bz9hSfx3ew51m9ozBjXzwYolhil+DEWv8AsWwdqJPvMysBFwFXAicA1xqZucMdlRzqgF/4O7nAM8B3hjG+3bgq+5+NvDVcHsYvQW4J3f7vcD73f0sYB/whoGMan5/BXzR3X8K+FnS9zDUn7mZnQK8Gdjs7k8DSsAlDO9nfjVwQct9nT7jC4Gzw9flwAf7NMahE1kMU/wajOjiFyiGdeTuUX8BzwW+lLt9BXDFoMe1gPHfCLwIuA84Kdx3EnDfoMfWZqynhn94vwzcDBhpF99yu7+LYfkCVgMPEDaT5O4f6s8cOAXYARwPlMNn/uJh/syBTcBd833GwIeBS9sdt9y+Yo5hil99GXeU8SuMSzGszVf0M15M/8Vmdob7hp6ZbQKeDnwHONHdd4WHHgFOHNCw5vKXwB8BjXB7HfC4u9fC7WH97M8AdgMfDcsMf2dmqxjyz9zdHwL+HPgxsAvYD3yPOD7zTKfPONrf2yUQ5Weh+NU3UcYvUAzrpAiJV5TM7BjgH4G3uvuB/GOeps9D1efDzH4NeNTdvzfosRyFMvAM4IPu/nTgEC3T8kP6ma8FLiYNvCcDq5g9DR6NYfyM5egofvVVlPELFMM6KULi9RBwWu72qeG+oWVmI6RB6x/c/TPh7p+Y2Unh8ZOARwc1vg5+AXiJmW0Hriedrv8rYI2ZlcMxw/rZ7wR2uvt3wu1PkwayYf/MXwg84O673b0KfIb07yGGzzzT6TOO7vd2CUX1WSh+9V2s8QsUw9oqQuL1XeDssEuiQlq4d9OAx9SRmRnw98A97v6+3EM3AZeF7y8jrZ0YGu5+hbuf6u6bSD/jf3L3VwNfA34zHDZ04wZw90eAHWb25HDXC4AfMuSfOen0/HPMbCz8u8nGPfSfeU6nz/gm4LVhZ9BzgP256fzlJpoYpvjVfxHHL1AMa2/QhWw9Koa7CPh34D+APx70eOYZ63mkU5V3At8PXxeR1ht8FfgRcCtw/KDHOsd7OB+4OXz/ROAOYBvwKWB00OPrMOZzga3hc/8csDaGzxx4F3AvcBdwLTA6rJ858AnSOo4q6Vn6Gzp9xqSFzVeF39l/I931NPD3MMDPLooYpvg1sDFHGb/C2BXDWr50ySARERGRPinCUqOIiIhIFJR4iYiIiPSJEi8RERGRPlHiJSIiItInSrxERERE+kSJl4iIiEifKPESERER6ZP/Dx8ZqdLLPqCkAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "env.render()"
      ],
      "metadata": {
        "id": "quxjvqEcQxUZ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 286
        },
        "outputId": "74baa175-e61a-4b31-fa27-597fafa91a14"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7ff7524304d0>"
            ]
          },
          "metadata": {},
          "execution_count": 10
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQcAAAD8CAYAAAB6iWHJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAM5ElEQVR4nO3df6zddX3H8efLUkEGys+ErlRQYWROGUjTYUgWApIBMXSJmMEfCgbSxcjEZUumW8Iy/wH3hyYGw9IAGRijGHCsMxiCAaJmg1FI6aAM7PiH1mZggWID4sre++N86y7Xz21pz/d8z7nc5yM5ud8fn3ven5M2r3vO9/s933eqCkma7x3TnoCk2WQ4SGoyHCQ1GQ6SmgwHSU2Gg6SmscIhyTFJ7kvy0+7n0QuMeyPJpu6xYZyakoaRca5zSPL3wItVdUOSLwJHV9VfNcbtrqojxpinpIGNGw5PA+dW1Y4kK4AHq+q0xjjDQVpkxg2Hl6vqqG45wEt71+eN2wNsAvYAN1TV3Qs83zpgHcBhh+es977/nQc9t1m144nDpz0FHaDfOf3VaU9hYh7d/PrPq+r41r79hkOSHwInNHb9DXDb3DBI8lJV/cZxhyQrq2p7kvcD9wPnV9V/7avuaR8+rG7acNI+57YYXf+B06c9BR2ge3+2adpTmJhlK7Y+WlWrW/sO2d8vV9XHFtqX5L+TrJjzseL5BZ5je/fz2SQPAmcC+wwHSdM17qnMDcAV3fIVwD/PH5Dk6CSHdsvHAecAW8asK2nCxg2HG4ALkvwU+Fi3TpLVSW7uxvwusDHJ48ADjI45GA7SjNvvx4p9qaqdwPmN7RuBq7vlfwU+PE4dScPzCklJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNRkOkpoMB0lNhoOkpl7CIcmFSZ5OsrXrfDV//6FJ7uj2P5zk5D7qSpqcscMhyTLgG8BFwAeBy5N8cN6wqxg1vDkF+BrwlXHrSpqsPt45rAG2VtWzVfUr4DvA2nlj1gK3dct3Aud3HbIkzag+wmEl8Nyc9W3dtuaYqtoD7AKO7aG2pAmZqQOSSdYl2Zhk48svvjHt6UhLWh/hsB1YNWf9xG5bc0ySQ4D3ADvnP1FVra+q1VW1+qhjlvUwNUkHq49weAQ4Ncn7krwTuIxRm7y55rbNuxS4v8Zp7y1p4sbqeAWjYwhJrgHuBZYBt1bVk0m+DGysqg3ALcA3k2wFXmQUIJJm2NjhAFBV9wD3zNt23ZzlXwKf7KOWpGHM1AFJSbPDcJDUZDhIajIcJDUZDpKaDAdJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqGqpX5pVJXkiyqXtc3UddSZMz9g1m5/TKvIBRt6tHkmyoqi3zht5RVdeMW0/SMPq4+/Sve2UCJNnbK3N+OByQI98B577rf3uY3my5ftoT0AF78LWl+el7qF6ZAJ9IsjnJnUlWNfa/qR3eCztthydN01CR+C/AyVV1OnAf/99x+03mtsM7/ljb4UnTNEivzKraWVWvd6s3A2f1UFfSBA3SKzPJijmrlwBP9VBX0gQN1Svz80kuAfYw6pV55bh1JU3WUL0yvwR8qY9akoaxNM/RSNovw0FSk+EgqclwkNRkOEhqMhwkNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUZDhIajIcJDUZDpKaDAdJTYaDpCbDQVJTX+3wbk3yfJInFtifJF/v2uVtTvKRPupKmpy+3jn8I3DhPvZfBJzaPdYBN/VUV9KE9BIOVfUjRneVXsha4PYaeQg4at7t6iXNmKGOObyllnm2w5Nmx0wdkLQdnjQ7hgqH/bbMkzRbhgqHDcCnu7MWZwO7qmrHQLUlHYReOl4l+TZwLnBckm3A3wLLAarqHxh1w7oY2Aq8Cnymj7qSJqevdniX72d/AZ/ro5akYczUAUlJs8NwkNRkOEhqMhwkNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUZDhIajIcJDUZDpKaDAdJTYaDpCbDQVKT4SCpyXCQ1DRUO7xzk+xKsql7XNdHXUmT08s9JBm1w7sRuH0fY35cVR/vqZ6kCRuqHZ6kRaavdw5vxUeTPA78DPjLqnpy/oAk6xg12uUwDuePfvuMAacntV3/gdOnPYUJembBPUOFw2PASVW1O8nFwN2MOm6/SVWtB9YDvDvH1EBzk9QwyNmKqnqlqnZ3y/cAy5McN0RtSQdnkHBIckKSdMtruro7h6gt6eAM1Q7vUuCzSfYArwGXdV2wJM2oodrh3cjoVKekRcIrJCU1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUZDhIajIcJDUZDpKaxg6HJKuSPJBkS5Ink1zbGJMkX0+yNcnmJB8Zt66kyerjHpJ7gL+oqseSHAk8muS+qtoyZ8xFjPpUnAr8AXBT91PSjBr7nUNV7aiqx7rlXwBPASvnDVsL3F4jDwFHJVkxbm1Jk9PrMYckJwNnAg/P27USeG7O+jZ+M0BIsi7JxiQb/4fX+5yapAPUWzgkOQK4C/hCVb1yMM9RVeuranVVrV7OoX1NTdJB6CUckixnFAzfqqrvNYZsB1bNWT+x2yZpRvVxtiLALcBTVfXVBYZtAD7dnbU4G9hVVTvGrS1pcvo4W3EO8CngP5Js6rb9NfBe+HU7vHuAi4GtwKvAZ3qoK2mCxg6HqvoJkP2MKeBz49aSNByvkJTUZDhIajIcJDUZDpKaDAdJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqGqod3rlJdiXZ1D2uG7eupMkaqh0ewI+r6uM91JM0gKHa4UlaZIZqhwfw0SSPJ/lBkt9b4PdthyfNiD4+VgD7bYf3GHBSVe1OcjFwN6OO229SVeuB9QDvzjHV19wkHbhB2uFV1StVtbtbvgdYnuS4PmpLmoxB2uElOaEbR5I1Xd2d49aWNDlDtcO7FPhskj3Aa8BlXRcsSTNqqHZ4NwI3jltL0nC8QlJSk+EgqclwkNRkOEhqMhwkNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUZDhIajIcJDUZDpKaDAdJTYaDpCbDQVKT4SCpqY8bzB6W5N+7nhRPJvm7xphDk9yRZGuSh7v+FpJmWB/vHF4Hzquq3wfOAC5Mcva8MVcBL1XVKcDXgK/0UFfSBPXRDq/29qQAlneP+XeWXgvc1i3fCZy/91b1kmZTX01tlnW3pX8euK+q5rfDWwk8B1BVe4BdwLF91JY0Gb2EQ1W9UVVnACcCa5J86GCex16Z0uzo9WxFVb0MPABcOG/XdmAVQJJDgPfQ6HhVVeuranVVrV7OoX1OTdIB6uNsxfFJjuqW3wVcAPznvGEbgCu65UuB++14Jc22PtrhrQBuS7KMUdh8t6q+n+TLwMaq2sCol+Y3k2wFXgQu66GupAnqox3eZuDMxvbr5iz/EvjkuLUkDccrJCU1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUZDhIajIcJDUZDpKaDAdJTUP1yrwyyQtJNnWPq8etK2my+rj79N5embuTLAd+kuQHVfXQvHF3VNU1PdSTNIA+7j5dwP56ZUpaZPp450DXs+JR4BTgG41emQCfSPKHwDPAn1fVc43nWQes61Z3/7DufLqP+b1FxwE/H7DeUHxdi8+Qr+2khXakz8ZTXeerfwL+rKqemLP9WGB3Vb2e5E+BP6mq83or3IMkG6tq9bTn0Tdf1+IzK69tkF6ZVbWzqvZ2xr0ZOKvPupL6N0ivzCQr5qxeAjw1bl1JkzVUr8zPJ7kE2MOoV+aVPdTt2/ppT2BCfF2Lz0y8tl6POUh6+/AKSUlNhoOkpiUfDkkuTPJ0kq1Jvjjt+fQlya1Jnk/yxP5HLx5JViV5IMmW7nL9a6c9pz68la8hDD6npXzMoTuI+gyjMyzbgEeAy6tqy1Qn1oPugrPdwO1V9aFpz6cv3ZmvFVX1WJIjGV1898eL/d8sSYDfmvs1BODaxtcQBrPU3zmsAbZW1bNV9SvgO8DaKc+pF1X1I0Znht5WqmpHVT3WLf+C0WnxldOd1fhqZKa+hrDUw2ElMPcy7m28Df6jLRVJTgbOBFqX6y86SZYl2QQ8D9y3wNcQBrPUw0GLVJIjgLuAL1TVK9OeTx+q6o2qOgM4EViTZKofB5d6OGwHVs1ZP7HbphnWfSa/C/hWVX1v2vPp20JfQxjaUg+HR4BTk7wvyTuBy4ANU56T9qE7cHcL8FRVfXXa8+nLW/kawtCWdDhU1R7gGuBeRge2vltVT053Vv1I8m3g34DTkmxLctW059STc4BPAefNubPYxdOeVA9WAA8k2czoj9Z9VfX9aU5oSZ/KlLSwJf3OQdLCDAdJTYaDpCbDQVKT4SCpyXCQ1GQ4SGr6Pxl6Bb+PLCXwAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Improved MADDPG"
      ],
      "metadata": {
        "id": "2XXTI-PCanVZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.keras.utils import to_categorical\n",
        "def obs_list_to_state_vector(observation):\n",
        "    state = np.array([])\n",
        "    for obs in observation:\n",
        "        state = np.concatenate([state, obs])\n",
        "    return state\n",
        "\n",
        "states_map = {(0,0): 0, (0,1): 1, (0,2): 2, (0,3): 3,\n",
        "                  (1,0): 4, (1,1): 5, (1,2): 6, (1,3): 7,\n",
        "                  (2,0): 8, (2,1): 9, (2,2): 10, (2,3): 11,\n",
        "                  (3,0): 12, (3,1): 13, (3,2): 14, (3,3): 15}\n",
        "\n",
        "\n",
        "env = MAalgo_MAGridWorld(4, 2)\n",
        "obs = env.reset()\n",
        "print(\"Observation: \", env.observation_space)\n",
        "print(\"Actions: \", env.action_space)\n",
        "\n",
        "\n",
        "n_agents = 2\n",
        "actor_dims = []\n",
        "for i in range(n_agents):\n",
        "  actor_dims.append(16)\n",
        "critic_dims = sum(actor_dims)\n",
        "\n",
        "# action space is a list of arrays, assume each agent has same action space\n",
        "n_actions = 5\n",
        "maddpg_agents = Grid_MADDPG(actor_dims, critic_dims, n_agents, n_actions, \n",
        "                      fc1=64, fc2=64,  \n",
        "                      alpha=0.001, beta=0.001, gamma = 0.99,\n",
        "                      chkpt_dir='tmp/maddpg_grid/')\n",
        "\n",
        "memory = MultiAgentReplayBuffer(10000, critic_dims, actor_dims, \n",
        "                  n_actions, n_agents, batch_size=128)\n",
        "\n",
        "total_episodes = 1000\n",
        "epsilon = 1\n",
        "epsilon_decay = (0.001/1)**(1/total_episodes)\n",
        "rewards_val = []\n",
        "agent1_route = [[3,0]]\n",
        "agent2_route = [[3,3]]\n",
        "\n",
        "for episode in range(total_episodes):\n",
        "  obs = env.reset()\n",
        "  obs_new = []\n",
        "  total_steps = 0\n",
        "  for state in obs:\n",
        "    state = states_map[tuple(state)]\n",
        "    one_hot = to_categorical(state, 16)\n",
        "    obs_new.append(one_hot)\n",
        "  obs = obs_new\n",
        "  done = [False]*n_agents\n",
        "  total_rewards = [0, 0]\n",
        "\n",
        "  while not any(done) and total_steps < 20:\n",
        "      \n",
        "      #time.sleep(0.1) # to slow down the action for the video\n",
        "      # print(\"Grid w obs: \", obs)\n",
        "      \n",
        "      actions = maddpg_agents.choose_action(obs)\n",
        "      # print(\"action prob: \", actions)\n",
        "      actions_new = []\n",
        "\n",
        "      for action in actions:\n",
        "        rand_num = np.random.random()\n",
        "        if epsilon > rand_num:\n",
        "          actions_new.append(np.random.choice(5))\n",
        "          # print(actions_new)\n",
        "        else:\n",
        "          actions_new.append(np.argmax(action))\n",
        "\n",
        "        # actions_new.append(np.random.choice(5,1,p = action)[0])\n",
        "\n",
        "      # print(\"Actions_new\", actions_new)\n",
        "\n",
        "      obs_, reward, done, info = env.step(actions_new)\n",
        "\n",
        "      if episode == total_episodes - 1:\n",
        "        agent1_route.append(obs_[0])\n",
        "        agent2_route.append(obs_[1])\n",
        "        # print(\"Actions: \", actions)\n",
        "      \n",
        "      obs_new_ = []\n",
        "      for state in obs_:\n",
        "        state = states_map[tuple(state)]\n",
        "        one_hot = to_categorical(state, 16)\n",
        "        obs_new_.append(one_hot)\n",
        "      obs_ = obs_new_\n",
        "\n",
        "      state = obs_list_to_state_vector(obs)\n",
        "      state_ = obs_list_to_state_vector(obs_)\n",
        "\n",
        "      # if total_steps >= MAX_STEPS:\n",
        "      #     done = [True]*n_agents\n",
        "\n",
        "      # print(reward)\n",
        "\n",
        "      # if reward[0] > 10:\n",
        "      #   reward[0] += 50\n",
        "\n",
        "\n",
        "      memory.store_transition(obs, state, actions, reward, obs_, state_, done)\n",
        "\n",
        "      if total_steps % 2 == 0:\n",
        "        maddpg_agents.learn(memory)\n",
        "\n",
        "      obs = obs_\n",
        "      total_steps += 1\n",
        "\n",
        "      for agent in range(2):\n",
        "        total_rewards[agent] += reward[agent]\n",
        "\n",
        "  \n",
        "  if epsilon > 0.01:\n",
        "    epsilon *= epsilon_decay\n",
        "  else:\n",
        "    epsilon = 0.01\n",
        "      \n",
        "  rewards_val.append(total_rewards)\n",
        "    \n",
        "  #After every episode\n",
        "  # avg_reward = np.mean(rewards_val[-10:])\n",
        "  \n",
        "  if episode % (total_episodes // 10) == 0 and episode > 0:\n",
        "      print('Episode: {}, Individual Rewards: {}'.format(episode, total_rewards))\n",
        "      \n",
        "\n",
        "\n",
        "x = [episode for episode in range(1, total_episodes+1)]\n",
        "yr1 = [rewards_val[episode][0] for episode in range(total_episodes)]\n",
        "yr2 = [rewards_val[episode][1] for episode in range(total_episodes)]\n",
        "\n",
        "#Plots showing episodes vs epsilon, episodes vs rewards\n",
        "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10,4))\n",
        "\n",
        "#episodes vs rewards\n",
        "ax1.plot(x,yr1)\n",
        "ax1.set_title(\"Agent1: Rewards per episode\")\n",
        "\n",
        "#episodes vs rewards\n",
        "ax2.plot(x,yr2)\n",
        "ax2.set_title(\"Agent2: Rewards per episode\")\n",
        "\n",
        "print(\"Agent1 Route: \", agent1_route)\n",
        "print(\"Agent2 Route: \", agent2_route)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 524
        },
        "id": "WyxSNm2g1_E2",
        "outputId": "2d6bf35a-daba-4e23-ec09-e22f977c3dc8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
            "Observation:  Box(4, 4)\n",
            "Actions:  Discrete(5)\n",
            "Episode: 100, Individual Rewards: [-77, -77]\n",
            "Episode: 200, Individual Rewards: [133, 327]\n",
            "Episode: 300, Individual Rewards: [-70, 270]\n",
            "Episode: 400, Individual Rewards: [-21, 305]\n",
            "Episode: 500, Individual Rewards: [298, 298]\n",
            "Episode: 600, Individual Rewards: [305, 305]\n",
            "Episode: 700, Individual Rewards: [298, 298]\n",
            "Episode: 800, Individual Rewards: [305, 18]\n",
            "Episode: 900, Individual Rewards: [305, 305]\n",
            "Agent1 Route:  [[3, 0], [2, 0], [2, 1], [1, 1], [0, 1], [0, 2], [0, 3]]\n",
            "Agent2 Route:  [[3, 3], [3, 2], [3, 1], [2, 1], [1, 1], [0, 1], [0, 0]]\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 720x288 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAl8AAAEICAYAAABh3JHPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydd5wW1dXHf2c7W2AXdum7LAgoCFJcAQWxUETQqIndKBqjyZv4qjHFEhNN1IQkthRfS+xGY4wxscfesIBYQAUpUgREem/L7t73j5nn2XmeZ8qdXp7z/Xz2s1Pu3HtmnpkzZ84991wSQoBhGIZhGIYJhoKwBWAYhmEYhskn2PhiGIZhGIYJEDa+GIZhGIZhAoSNL4ZhGIZhmABh44thGIZhGCZA2PhiGIZhGIYJEDa+mNhBREcS0aqw5fASInqeiKZ7XOe1RPQ3L+tkGMYdrL+k60y0/mLjy0OI6HUi2kxEpQG2KYiov2a9hIgeJ6Ll6r4jbdb3OhHtIaIdRLSBiJ4goh6eC85kIIQ4VgjxQNhyMPlLRPTXGCJ6iYg2EdF6IvqnHf3D+iscWH/Zh40vjyCiRgCHAxAAvhGqMMBMAN8G8LXD4y8SQlQC6A+gEsCNXglmFyIqCqttM4ioMGwZGMYrIqS/agDcBaARQB8A2wHcZ7MO1l8WsP4KHza+vOMcAO8BuB9AhvuViLoQ0dNEtI2I3iei64lopmb/AZqvvYVEdKpm3/1EdBsRPUtE24loFhHtp+57Uy02V/3SO00I0SyEuFUIMRNAq5sTEkJsAfAfAMOtZCWivkS0hYgK1PW/EtE6zXEPEdGl6vJ5RLRAPZ+lRPQ9TbkjiWgVEV1ORF8DuI+IOqjXYTMRzQdwSNb1vZyIVqv1LSSiCXrno9Zxhyr/diJ6g4j6WJ2b5tjbieg5ItoJ4Cid+jsR0T1EtEaV5/qUkiOic4nobSL6CxFtJaLPtXKqX+zfVZf7q7JtVb/e/6Epd5h6D21V/x+m2ddXPW47Eb0EoDZLvjFE9I76O80lm15RJtFERX89L4T4pxBimxBiF4C/ABjr5IRYf7H+ijRCCP7z4A/AEgA/AHAwgH0Aumn2Par+lQMYDGAlgJnqvgp1/TwARQBGANgAYLC6/34AGwGMUvc/DOBRTd0CQH8DmVYBODJr25kA5pmcx+sAvqsudwHwMoAnJWX9EsDB6vJCAEsBDNLsG6EuTwOwHwACcASAXQBGqvuOBNAC4HcASgF0ADADwFsAOgOoB/ApgFVq+f1VmXqq640A9jM4t/uhfEmPV+v+o83fYSuUF0EBgDKd+v8N4E61rq4AZgP4nrrvXPW8fgSgGMBpan2dda773wH8PNUOgHHq9s4ANgM4W5XxDHW9i7r/XQA3q+c2Xj3Xv6n7ekG5j6aq9U5S1+vCfnb4L/w/RFB/qfsvBfCeZp31F+uvROiv0AVIwh+AcVAUVq26/jmAH6nLheq+/TXlr9c8NKcBeCurvjsBXKMu3w/gbs2+qQA+16zbMr4kzuV1KMpkq1r3xwAaJGV9CMBlALpDUV6/B/B9AH0BbAFQYNDmfwBcoi4fCaBZqxygKMEpmvUL0a68+gNYB2AigGKLc7sfmYq/Eop3sF7yd3jQpO5uAPYC6KDZdgaA19TlcwF8BYA0+2cDOFtz3VPK60EoXS+9s9o4G8DsrG3vqnU3QFGOFZp9j6BdeV0O4KGsY18AMD3s54f/wv1DdPXXQQA2ATjcxrm8DtZfRr8D668I/XG3ozdMB/CiEGKDuv4I2l33dVCs/JWa8trlPgBGq67ULUS0BcBZUBRACm3s1i4oD52fXCyE6ARF+dUA6C0p6xtQlM94AG9CeSCPUP/eEkK0AQARHUtE76nu8S1QFLLWxbxeCLFHs94TmddsRWpBCLEEytfxtQDWEdGjRNTT5NzS9QghdkBR7j0lzi3jWB36QPkiXKM5/k4oX5ApVgtVa2jOQ0/Wn0H5qp5NRJ8R0XfU7T2hOXdNHb3UfZuFEDuz9mnlOyXr/MYB4GBkJnL6i5Qg/OehGDVv2TkZsP5i/RUDIhkMGCeIqAOAUwEUqn38gOI2rSaiYVBczC1QFMAidX+9poqVAN4QQkwKSGRphBCfENH1AG4jopGwlvUNAH+A4nF7A0rg/x0A9qjrIGUk1b+gxJg8KYTYR0T/gfKwppvOqncNlGv2mbrekCXnIwAeIaKOUBTG76B8ZemRvvZEVAnFFf6VxLnpyaVlJZQvx1ohRItBmV5ERBoF1gDgqZxGhPgawAWqjOMAvExKfMxXUJSQlgYA/4VyjWqIqEKjwBo0Mq+E8uV4gck5MHlGFPWXGsf0MoDrhBAPOa2H9VcOrL8iBHu+3HMiFNfvYCiBncMBDILSx3+OEKIVwBMAriWiciI6AMqDm+IZAAOJ6GwiKlb/DiGiQZLtrwXQT7uBiEqJqExdLSGiMiKi3EOleACKS/obVrIKIRYD2A1lpOUbQohtqnzfgqq8AJRAUe7rAbQQ0bEAJlvI8BiAK4mohoh6A/hfzbnuT0RHq0pxj9p+m0ldU4loHBGVALgOSjzJSqtzs0IIsQbAiwBuIqKORFRARPsR0RGaYl0BXKzWfQqU++S57LqI6BT1PAElJkKo5/ScKuOZRFRERKdBue+eEUKsADAHwK9ISTcyDsDxmmr/BuB4IjqGiArVe+JITTtMfhIp/UVEvQC8CuAvQog73J4cWH+x/ooobHy5ZzqA+4QQXwohvk79QRmlcxYpQ40vAtAJivv9ISgBiXsBQAixHcrDezqUL4Ov0R6sKcO1AB5QXbGp0S0LoTzEvaD0i++G+sVBRGcR0Wd6FekhhGiGEtj5C0lZ3wCwUVUIqXUC8KHmfC+GopA2Qwmgzfl6yuJXUFzQy6AoCO3XcCmUgNYNqjxdAVxpUtcjAK6B4q4/GIqi9eJ3AJSXUgmA+VDO7XFkusVnARigynoDgJOFEBt16jkEwCwi2gHl2lwihFiqlj0OwI+hBJv+DMBxmu6iMwGMVs/tGiixF1DPbyWAEwBcBeXFsRLAT8E6IN+Jmv76LhRj7FpSRkDuUJ8DAKy/wPorMfqLMrtwmSAgot8B6C6EmG5ZmPEMIrofSqDr1SG0fS6UgNRxQbfNMF7C+iscWH8li9hajXGClPwrB5HCKADnQxnWyzAME2lYfzGM93DAfTBUQXHV94QSQ3ATgCdDlYhhGEYO1l8M4zHc7cgwDMMwDBMg3O3IMAzDMAwTILHodqytrRWNjY1hi8EwTIB88MEHG4QQdWHL4QWswxgmv7DSX7EwvhobGzFnzpywxWAYJkCIKDsbdmxhHcYw+YWV/uJuR4ZhGIZhmABh44thGIZhGCZA2PhiGIZhGIYJEDa+GIZhGIZhAoSNL4ZhGIZhmABh44thGIZhGCZA2PhiGIZhGIYJEDa+HLB0/Q68s2RD2GIwDMN4ztbd+/DU3K/CFoNhEk0skqxGjaNvegMAsHzGtJAlYRiG8ZYfPzYXLy9Yi8E9OqJ/18qwxWGYRMKeL4+45slPccOz88MWg2EYxhVrtu4GAOxubg1ZEoZJLmx8ecQD767AX99aFrYYDMMwriBS/guIcAVhmATDxhfDMAyThqBYX4JtL4bxDTa+GIZhmDTtni+GYfyCjS+GYRgmDVHK88XmF8P4BRtfDMMwTBrV8cWeL4bxETa+GIZhmDTpbke2vhjGN9j4YhiGYdKkPV9sfTGMb7DxxTAMw6RJxXwxDOMfbHwxDMMwaTjmi2H8h40vhmEYJk3K8dXWxuYXw/gFG18MwzAAiKiQiD4iomfU9b5ENIuIlhDRP4ioRN1eqq4vUfc3him316RTTYQsB8MkGTa+GIZhFC4BsECz/jsAtwgh+gPYDOB8dfv5ADar229RyyWG9oD7UMVgmETDxhfDMHkPEfUGMA3A3eo6ATgawONqkQcAnKgun6CuQ90/gRIYpc5zOzKMf7DxxTAMA9wK4GcA2tT1LgC2CCFa1PVVAHqpy70ArAQAdf9WtXwGRHQhEc0hojnr16/3U3ZPKeD5hRjGd9j4YhgmryGi4wCsE0J84GW9Qoi7hBBNQoimuro6L6v2Fba9GMZ/isIWgGEYJmTGAvgGEU0FUAagI4A/AqgmoiLVu9UbwGq1/GoA9QBWEVERgE4ANgYvtj9whnuG8R/2fEmyt6UVs5dtClsMhmE8RghxpRCitxCiEcDpAF4VQpwF4DUAJ6vFpgN4Ul1+Sl2Huv9VkaB08KSG3Lcl55QYJnKw8SXJr56ej1PvfBeL124PWxSGYYLhcgCXEdESKDFd96jb7wHQRd1+GYArQpLPF7jbkWH8h40vSRZ+rRhdW3fv87Wd65+Zjw9WbHZVxx9fXow3F63H3W8txfOfrNEt86dXFuONRfEJAk6xccde/OgfH2Pn3hbrwgxjEyHE60KI49TlpUKIUUKI/kKIU4QQe9Xte9T1/ur+peFK7Q8JcuYxTOTgmK+IcffMZbh75jIsnzHNcR23vLwoY12vrptfWmS4L8rc+vJi/Puj1RheX43phzWGLQ7DJA5Ossow/sOeLyZWcO4hhvGXdMIyftQYxjfY+DLgb++twJRb38zZftljcz1va8GabRj8y/9i7bY9ruuatTQxg64YhgmBVMzXefe/j5bWNvPCDMM4grsdDbj6P5/qbv9y0y7P23rgneXY1dyKVxasc13XTS8tsi4UY1IjsZKXT5xhokGB5uFqbm1DUSF/ozOM1/BTlTBKWFEyDOMC7XcNx9wzjD/wmzpCeBHPVFTILiGGYZzDXmWG8R/X3Y5EVAbgTQClan2PCyGuIaK+AB6Fkh/nAwBnCyGaiagUwIMADoaSFfo0IcRyt3K4YfWW3diwfS+G1VcblnGS/mHr7n34dPVWjO1fm7OvpbUNr36+DgO7VeHR91fm7H9lwVpMGNQNn67eii/W78DIhho89N4K1Nd0QFVZMSYf2A2PzPoS2/a04JIJA7BjbwteX7hOV86W1ja88vk6TB7cDdnz/9784kLUVpWiV3UHbNrZjNF9u6C8tBBL1+/E/t2r8MmqrWhqrMHfZ3+JTTub0a+uAhu2N2NncwtWbd6NEQ3V2LW3FVVlRdiyex8KiVBTUYKGzuWYtXQjOnYoxttLNqCytAjV5SVYuXkXDuhehQ079qKushTrdzSjuaUN67bvQWlRAarKilFVVoTte1rQvVMZNu1oRkOXcpQWKd8JD723AgBwz8xlWLR2O3btbcWgHh2xYM027N+9CkTA7GWb0L9rFWorS9C9UxnmrdqK0qIC7G5uBRGweN0O1FaWorayFP1qK7B22x4UFBCWrNuB4fXV+GrLblSVFaFndQe888VGFBcWYFdzC3Y1t2J3cytG9+2MitIiFBUS1m7bg4rSIgzq3hHrt+/F/DXbUF1ejLqqUrS2CexrFSgpKsC8lVvQuaIEpcWFqCwtRG1lKT5euQWbdjajS0UJ6juXY8m6HenyPTqVYdXmXRACOLBnRwgAzS1t2NWsXOsUe/a1goiwctMudO9UhtY2gfXb9+KrrXtQVlSAXjUdsGFHM4b17pTxu2/ZtQ/V5cVYvnEnvty0G/3rKrGvtQ19upRj4dfbUVtVip17W/DVlt3oWd0BgOIRaRUCqzbvxkG9qzH/q63Yr2slSosKsXNvC3Y1t6BPlwos37AT/btW4vOvt6OtTaCspBD9aisAAJMGd0OfLhXyDxETAmx9MYzfeBHztRfA0UKIHURUDGAmET0PJfngLUKIR4noDgDnA7hd/b9ZCNGfiE4H8DsAp3kgh7zALa3YubcVnStKAABjZ7wKwDjtwpqtux21c+GDczBr2SbMvWYyOnUozth3++tf5MRnaV385z8wB89ePA7H/XmmZTt1VaV4bt4avGsQbH/nm0vxhxcW4o5vH4wpQ7pn7PvTq0tyytd37oCVm3ZjbP8ueHvJRkw5sDv++9nXunU//sEqS/myeWn+WtvHZLNi4y6s2PilsvLR6pz9LzuMn/u3Tl3ZzFyywVHdTvmnBzMOPj33K9P9b9rM+fb6QqW83evc0Lmcja+Iw54vhvEf192OQmGHulqs/gkARwN4XN3+AIAT1eUT1HWo+ydQtjvGZy588AOMvO4l6fK7mlsdtbN4nXJZ9EYMrd5ibdBt2y2XSHTX3hYsXmeceX/VZqWtjTv3StW3cpNSftFaRf5FJnUHxbxrJ0uV66Ia1ADw02P290scz7n/vEOkyz5/yeGYd+1k/PfSw6WPue6EAzHv2smYd+1kDOrREQBw6cQBOeU6lvkzBufx7x+KeddOxlEHdPWlfsY72PZiGP/xRNMSUSGUrsX+AG4D8AWALeqEtACwCkAvdbkXgJUAIIRoIaKtULomN2TVeSGACwGgoaHBCzHx6eqtWLlpV2CZ3Y0yRH+1ZbduV2N26Yse+VCqnd8+/7np/oVfb8tYf/Dd5VL1plix0fsRnnbpWFZsXQhAsWbAQVlxoV/ieE7/rpXSZRs6l6OitAjbSuVnWygrLkxfw1QX7mDVCNNSXV6CbXu8nz2gtrJU+jdkwoU9XwzjP54E3AshWoUQwwH0BjAKwAEe1HmXEKJJCNFUV1fnWkYAOO7PM/E/D8sZNH5y9j2zpMpt3NnsSXsffrklvbxi40788snPbB3f2haNIU9XHNt+Wx2t40G58ZRh+O23hgIAzhqdabAPr6/GxEHdLNuorSyxLOMH3TqWSZctL3FnVH7/iH7oV1uBYfXVqK0szdh386nDUN+5A045uDcK1JdwASkv5MMH5MYuntZUj59MHoia8kzD6pIJmV61Di5lZoKjgK0vhvEdT/sYhBBbiOg1AIcCqCaiItX71RtAKphmNYB6AKuIqAhAJyiB94nDqDd18y4Dj4XP47qFALb74NWww/IZ09B4xbMZ69mk9r/506Mw/g+vpbd//4j98P0j9gMAzF25Ba9+rsQb9exUhneunJBT570zlwEAzh7TB9edOAQ79rZgyDUvmMo35+pJ2Lm3BQdalJNlREM1PtIYv1pSAwuunjYIxYUFabm112dMv854b+mmjONS95X2/so+9t0rj0aPTh1w3n2z8drC9RllpwzpgSlDegAA5lw9EU9+vBqXPPoxDmmsQVNjZ7z1s6MBAH84ZViOzNc/Mx93q9dV+9sNr6/BtzUfFQf17pTxW7PxFR+c2l63vrwI9729HHOvkQsRYJh8xrXni4jqiKhaXe4AYBKABQBeA3CyWmw6gCfV5afUdaj7XxU+z+D6l1cX450v5IOkkzqh7NX/+RQvGgTORxGzl4B2X0GB3NtC9p3iZbqOLhWllmX8CHm0k+/Nj/ZTVab+d4hRFzDTjlYTbtrZjKbrX8Knq7fqlr315cXYulu+K5xh8hkvPF89ADygxn0VAHhMCPEMEc0H8CgRXQ/gIwD3qOXvAfAQES0BsAnA6R7IYMqNL8Yj63sQJp/e6MaocftZI7Fuu9zgAAC4Z7p+sLpTm6K4wLv0d2ZdhEbiPfGDw3DRwx/iq63m002ZnV5JUeY5yFwKshFqPTwrLYvRtf73D8bi+U/WZMTiMdHGyBh/e8kGbNjRjNtf/wK3nTUyYKkYJll4MdpxnhBihBDiICHEECHEr9XtS4UQo4QQ/YUQpwgh9qrb96jr/dX9S93K4DcJdYRFlmOH9sD0wxpNy2gNhf27V0nVm/1OaehcrlvOzJM2LitnW3V5MWZeflROufPGNuq2qStX1vrIhhpcfqy7sMls48trpg3tYbo/9fsMr6/GlVMH+SpLUlm3fQ++WL/DuqDHGN2yqVgwntyeYdzDn6M6+GFsnXrnu+kYpCDbzWdSLxEjA6hIsrtSS3FWl2R1B/MRfGYtmHX3yXQF6hVJpYrI9t6ZduFatmQNh2h7z+jfvIIJN70ReLtG915qcxvPtc0wrmHjSwIvQmJmL9uEXz8z331FDABnv0l2t5pMV9iD3xllekxpUaFrI8qKe6Y34fHvHypV9qmLxuF33xoqHQfnG2yNuSasjzGjny61nT1fDOMefzIqxhyvVIvdwP2oBfqHLY5bu8XK8JEJrD+4Tw16VXdIJ8XNNr7cyCjrjZpgkCJDL0arsbYCjbW5GeRlBy84ho2txGB0P6S2h60XGCYJsOcrQkRPp0VPIjdkv1SKJIPAtQlQsw02IvMw9Xy1SfL1vJOA9rfTfhBSOuaLYRi3sPElgdMvvYBnTUoc5t15HtQvJQPw5zNHpNezE1CmEpB63b5UoL6Nds1MRDujHP2sg4kGhjFf6n/2fDGMe9j40iGs7r+oKbWoyeM1ssaL2bQ4ltnAY2STeBnLwx8e8aVQEy+ovSPaf9Pc+ySMUZkME2fY+GIii3l3nv2Xe7Y94MQ8yD6GyLnXx9SzF6DVxnYSo2W/usx5RlvbBB6bsxJt6teY3kfZ03O/Mqxvz75WPP7BKsuP2i/W78C7X2ROdrJq8y68vnCdpOTmPDPvK04Cy0QGNr50CMvhEzVHU9TksYsXAcJWRpCVh8dPI8pOzXL5xpzL6oVhy0QD7UBZIYBHZq3Azx6fh7+9t0LZZnH87ubWjPX/e/0L/OSfc/H8p+aza0y46Q2c8df3MrZNuvlNnHvf+xnbtu7eh1lL7c1It3LTLlz0yEf4379/ZOs4hvELNr6YyOL1CL1s48JR15iOkcGeIybJpLxFG3c0AwA++nJzjhdLu/qTx+dm7Nu7TzHGlm3Yabvt3ftac7Zd8MAcnHbXe9ijs8+IlNduKXePMhGBja8IEbVUE4y1YWWVTstpclMpg84jo8+PTBNskMaPHz7yIc64671Mz5Zon/EhZcBs3rUPfa98Dmu27tat59l5a/DQu8vxxqL1GPHrF7FKTdOSMt7cMm+1MlH9vlb5bK9l6tyie1s4QywTDdj40sErGyjuxlTY8puO0HPitPJkhKS9VBMMExeenbcG7+p05xWmUkxkqYOZizekl7OfrTvfXIr/fvo1Nu/ah+Wqx2tXc4sncqbkaGmV108p8fba8JYxjJ+w8cUYEmXT0Y7JY5g00gM5Csi8IqfeLTnHl51r4HPajqxK2CRNBgIiPfqxLcv62qN6kYQQePGztRn7igsLsLdFMXRqyksA6HchOpJJFcPK8/X2kg3YuGNvxrY97PliIgIbXzp4NeQ+7sPtY+64y5Hfk+617JivmP/GDEBEZUQ0m4jmEtFnRPQrdXtfIppFREuI6B9EVKJuL1XXl6j7G8OU32uyn5tUOpVs4yvlRXrhs7WYv2Zbxr7iQsLefYqhU1GqdPllB+I7JSXHvjZjBbVtzz6cdfcsXPLoxwDaPySb2fhiIgJPL+QjRt12Rq/rqBk7oXc7+jwljpNEpjmxTXDTPerOG2UvyarzvVL15xilrqsMkr0AjhZC7CCiYgAzieh5AJcBuEUI8SgR3QHgfAC3q/83CyH6E9HpAH4H4LSwhPcTIdrzfmWrg+ufXYBTmurx8cotOcctWrsDi9Yqwe0vqF6xF+evReMVz1q2qVfm1Dvfxexlm9C1qhQtqtG1YuNOjJ3xqmldM5dsyKlPRgaG0ePYId1x+7cP9qQu9nxFiKhNWBstadzjR9eYZZJVJvIIhdQwuGL1TwA4GsDj6vYHAJyoLp+grkPdP4ES7ALNDrjX8uXGXbZGHTpl9rJNAIB129u7Ed/SxJwFQWmRt6/LM0c3eFpfmBwxsM7X+s8YVY+LJwzAxRMG4BvDenpad6/qDhnrF08YgHMPa8zYVlVahIsnDMDUoT08a5eNLx20OsaN9yf2+jjsibUd7vNWhuwA+/blQxpr8IvjBlvEbnnvFZMvIdeWH1M1xe3OJ6JCIvoYwDoALwH4AsAWIUQqSnwVgF7qci8AKwFA3b8VQJdgJfYP7UegAFCkGl+tOrrwmU++Cs1DfvvrXwTaXvdOZZ7Wd9FR/T2tL0y+ObKXdaEsaitLpMv+4Mj+uGzSQFw2aSDOHdsIAKgsze24sxp9rsfIPjUZ65dNGogfTRqYsW14QzUumzQQx3to+LHxFQJGqipy3Y5hC+AxXhsE//z+YRjcs6PHtTJhIIRoFUIMB9AbwCgAB7itk4guJKI5RDRn/fr1rmUMi9RoxzadcKk731iKB9Xkq0nnqqmDPK2viw3jIyxKCuVMhE4dijF5cDfTMoN7dERxoTMtXFdVml7u26UCZcUF+N74fvj2mEzv4Q0nDTWt50AdfV1XWZqzrWOZ/xFZbHxZEKRBFDVjJ+yYL9NRhH4kp5I6JPcgP0c0xpaYnpwQYguA1wAcCqCaiFJauDeA1eryagD1AKDu7wQgJ0eDEOIuIUSTEKKprs7fbhm/EEKkux2NywQkjA/8+oQDM9YfuWA0ls+YllNu+YxpOObA7lg+Yxq++M1ULJ8xDR/9YpJp3dedcCDuPFs/PmjS4G4oLSo0PDbVtfa98f1QUaKUe/i7o03bmzioK5b+ZiqW3HCs7v7s7jUrrjz2ACy4bkp6/YvfTM3Y//RF41CrGi4FRLjrnCYsnzENt5w2DIDSVafluhOH4PPrcmU7aYTiNRs/sA43nzosY1+fLuVYPmNaOk8bANRUlOCzX03B/04YgOtPHIrlM6Zh2W+n4ovfTMUZoxqwfMY0jO7bOaed5TOm4dmLD0dX1ZBr7FKuyp577kSEs8f00b8wHsHGl4/YDbiPGjHWqbqElRvMCXIB9zZSTZh1f0rXYiaLB5WEBBHVEVG1utwBwCQAC6AYYSerxaYDeFJdfkpdh7r/VRH6l4p/pJwfejFfTunhcReeU7JjNmXiQAtl+7aI0l5Du2gPS7UnU1NBAaHIwFtlN2atrqo041yzz7usWL++zhWKcVORZXxVlBbqXrvUlkICSiRlzK6HiKR/l9S916tGMUa1hp0RXSq891Ky8RUhoqa+w5bHXBHaV2q5Aff20dOlTmP74mywWBGzPF89ALxGRPMAvA/gJSHEMwAuB3AZES2BEtN1j1r+HgBd1O2XAbgiaIFf/XwtTr/rXV+80xkxrzBONeEG2ZesU/54+nCpckbP4N8vGIM/nHyQo2PT+yX2/fK4weaV2MLZM1ffuQMO7aeELI7t3wVXTxuEqUO744ThikfqvvMOwdMXjcs5Tmvkaa/F+AG1+NU3DsRV0zK7aRYONQEAACAASURBVMuLs7vyKEfsIidBWzb56zlN+P3JB+H/zjoY151wIA7OivnKpl9tBX594hDP5eBUEzpkKx+nxD3gPmqjL7UElmpCph6XbfjRrh05vLlP43uvCyHmARihs30plPiv7O17AJwSgGgAgCXrdqB/18qMbd9/6EM0t7ahubXNtPsqxbxVW9CzukO6m8gO7UlWbR9qiN8jJK1eqClyPF/q6qH7dcHS9favVTZWj9bwhmr946Q3anH2A/33kvG44bkFeHfpRkwd2gNnjc7sbjtq/666xxnFbxERph/WiNc+X5exPdfgVuTV/gaFBf77g7p2LMOpTfUAgLMPbcRrC9eZlj/70D7oWFbsuRzs+fIRu1+lUTN2Qvd8RfB97qVMMfMO2SKKv11cmXjzG67r+MZf3sZxf5pp+zgh2r0RXnrZ1m7ba13IBbIpYNyM0rV6fomce8e0H0RuPo60Bo+AfndvRWlROlj+kMbcWCktxxzYHlSvDcbXvRZZm6rL9Q0YbbEoDmDyS5Wx8WWBFwonWiZVMpDyDHniHZNI+eDD0+l1u36n7WBjK/p8vW2Po+Paux29lMY53xzZCw2dy03LSMdlmRC1ngsjY69fbUW6RDaf/3oKXr7siPT6u1dO0K3jyP27YvmMaRjYrcpUhjvPbkrHP1kNxEjtre/cISdoPqOcpppe1R3w1EVjTev1mrB+5bw2vowMq6h5oMIi7lfBH8+d3Uc1Wgo8KPLzrJPHuN+9im17lFRnXsZ8+Y2s3eRqujBLrxZJeMcMuu4k2wC0AePtv0/KLioooLTn0qv0CZ1UD1ZBhnfOuLzlNYi4tvDLCOeYrwgROd0Wdrej2T4PHgiZhz6nW0LXu+59wL1czJeN0Y6mMV/S1ZjIwiSFP7ywML28t6UNMxdHK0eZzH0v3+3o3yhgr9/ZevWdMaoBY/p1Ts9hmeKFS8fjgxWbASjpGn4+dVA6Qej/nTUS3TqW4lu3v+tIjge/MwovzV+LzhYjAFM62sqZkTuFW7DaJCwPJxtfOngVcK9XX5xgD6B7ItZzERhR67KJG1t37QtbhFgjneIhx/Ml34ZUWacjIiXluHTiAMxNz63ZftCAblUYoHYhEhEuGN8vvc/tFDm9a8px3ti+luLKXsqwVQV3O4ZAXI2ioAj7+pi9wIN6YKwm1jbeKFG3s8McVuDvFWNjy1tOuM1+cLxfpH7btogEfcncak4D7u2247Rur+ojIGsyqDhBWf/VtTxRJXltfMnghQESVw9SPKWWx6uH3I8s9kEqIH8GDHhfZz6xfOOusEWINVQA3GWQXT6jnIvcfx44vgyfk6jHQeWgF44hmRojgNRepoSlq/La+DKcY1G6pDl2vQFRS5Idtjx+fpUCzgxru+36n1+LYYIhWtrJnEIiFEskcjUfBezu+VRSTTgLNpdO5kztqUD8TlzrFJmUHFHGL/k45stHwjZe3BJl6YP6MpRK+eBLu96UkSnrxbXMGZjgukZGhkBVTEQUgsy9Jd3t6CrmK/xRfATCEQPr8L3x/TLiuoJDpOXIJrXNMuA+ZG1h1b5fzxgbXzpojSblxnF5c0j+eFGz1cKWJ4pfRHYVRdiKhWHyESLJbsGcQnLpE6TlsN1+1nGi/X1kVLaosABXTh2kvzMg9D11zo6Not73g2j6KQMi7p6pfCZKD6gf3YdO0mA4LuvFSyZPFWg+ERVtKXNvySZZdfNxZHmk16km9LbF4DmzzvNlr7xUmx6OWvXrGrs2voionoheI6L5RPQZEV2ibu9MRC8R0WL1f426nYjoT0S0hIjmEdFItzL4SZD2WVSUW1Tw22vk5KHyNubLfvsMExZx+lgtIHI0S0QQubn8PC4s3KWaCLvbMRy88Hy1APixEGIwgDEAfkhEgwFcAeAVIcQAAK+o6wBwLIAB6t+FAG73QAZP8Srg3u7RMdJticCr6+3LwysV8yXfst9pO3IN5Zi9PWJGkCOoo2Z0ySVZDUAOK49JAO0l8SmLm+HpFNfGlxBijRDiQ3V5O4AFAHoBOAHAA2qxBwCcqC6fAOBBofAegGoicpf1zSHRUilMHLDr+vd7xCaT3wRhF4ms/3GASM5vbpZqwnWeL5KZXshge0bsmcmHU0SUiNFITBlkB0f4RhJSTRBRI4ARAGYB6CaEWKPu+hpAajr0XgBWag5bpW7LrutCIppDRHPWrw92eotchebu15FVkHHNB+YbPnfZOet2jIayAzyM+fIAjvlKPhFzgFkiNWLYTf0ycZmW3jH5VBNx83PJho3kq+7wzPgiokoA/wJwqRBim3afUPzWth5dIcRdQogmIURTXV2dV2JmtSFd0lH9UXpRM/5hOj+cw3sgyDuH79P4EsQHW0pPRuXj0Mvb1WxibbfPRRBPVVSeXDeXKuxzCGtEuifGFxEVQzG8HhZCPKFuXpvqTlT/r1O3rwZQrzm8t7otkrj52jOKlTC6UeP2Zek3SU9Q6kUSRz/KekH8f51owylMrJEbMexitKMHnjXDbkcXqRuiguNUEwHf27HNcE/KW/AeAAuEEDdrdj0FYLq6PB3Ak5rt56ijHscA2KrpnowGHhtB2V+MbGS5J0p6yJ/peYI7wyhdS8YegcZ8RURv+er5Mlj2om4/iIpBFufRjmHhRZLVsQDOBvAJEX2sbrsKwAwAjxHR+QBWADhV3fccgKkAlgDYBeA8D2RwRFTc6CmiJU34RPGR9FPx2z7exhXyW7+Zdd8wySAu+mlY704AnHmmPH++rWK+DPfn7tA3cOL/nPn5GzhpPyhcG19CiJkwln+CTnkB4Idu2/UTrVHmSYZ76Ybjot7CJ+nvdq9Pz+zWSvq1TCL5nGrC6um47sQh8jX5PKgnX5Cei1L3YG9l8Qq/n7HEZ7j3RnG4zPMVNd0VE6LoPdGdw8yhAo/e2XlHks8tSgQ6tWOEFJnZizH1jEqOtdM9VupIiZGM1tnd5Uc7uknnEGWyU00EfUphvWcSb3yZIaNLzMp4rYyio9qiTxLc7WZ4PbG2aT1eTOeR8N+DiQ/p58LlMxTENDeWczvG5rGy91FqfmQ08FunJd74cmIfRegDL6+J4kOp//FpkmoikmfhPxF0WiYSmQ9Atx+JUQy49+q5chNv5DSBqlN0Y74i/JzJB9ybr/uNYfYB7naMLvLJUzPhVBPuCUvpBNWs10ab37EtUX4J5Duu9UpM9ZJcElSTDycvnguH++PmNXKX5yvcswir9cQbX070hshaZqOISWE3/06+GiVhK9R8IdCYrwDbMsPLO8tNXVHx0EQVp3m+omJScrdjApB1+0ct9UXY5GuwuixRUsZRkiUfsPNi8EqrxCbgnjL/O8V9ni+yDOaWmdvRqo0oYDBswMWx7rBzq3K3o0+YKQy5gHuT450IxHhDWN2OOk+qmShOxQxSp/rRVETeCYnFzovBfcyXUP8nj4KsN6C9mK9gSaw3OXu0Y0JPM5vEG19OyFZWQSmdCH1YRoK4KJuwvj6jdH2iJEs+EaTOiIp+sgq4t5NqIruejHUPPGfWqkE+1YT80cGj+1EqKVxB6CfBqSZih3R3osPAfMaYsF72dlt1apcFenahK79wIaJ6InqNiOYT0WdEdIm6vTMRvUREi9X/Nep2IqI/EdESIppHRCPDPQNzXMfbx0wxtXc7yuSacNOOdZeiZcC9VaoJi7JJ8BLlGsABtx/SNUy88WWmN2Rc9xxwHx5xUSxhiRml6xMlWRzQAuDHQojBAMYA+CERDQZwBYBXhBADALyirgPAsQAGqH8XArg9eJFVXOYqjCNefniZpZpgb648usai7LERvcwccB8CYemqpClJPwntgbXZruMuSY/Pz3R6IR+UTFQVqh5CiDVCiA/V5e0AFgDoBeAEAA+oxR4AcKK6fAKAB4XCewCqiahHwGIHRhT1klcB925CBqzTSJBEFnyD7akDLa59lA1E2WubawAHe05GrXHAvUvcKg7TDPfq/4++3OyuESbWxMnQYMwhokYAIwDMAtBNCLFG3fU1gG7qci8AKzWHrVK3Zdd1IRHNIaI569ev901mK9y+RKI2CtvTya9N6nY9WtJjvRBlPePnb5JUEm98mWFkWGVvt1I+J/3fO47aySkXMSUXZcJzfNlr2U85varbj2SSUf4iN4KIKgH8C8ClQoht2n1CCfC09YAKIe4SQjQJIZrq6uo8lFTThkzoRJ6pFXtzO7poR6oBMi3r1ssTaYNMtlzI52D0G3C3o0v8NGjYqPKXsB9KWfQe3vS9EZNz8Jq4/HYpiKgYiuH1sBDiCXXz2lR3ovp/nbp9NYB6zeG91W2JJMnGW87UNpoH1jpY3iLg3qFMxvVF96HSk81xxEVETpO7HX1E6tIKf7sus9ti5AgtvUNEFANg7xr4LXeUrotdSLmQ9wBYIIS4WbPrKQDT1eXpAJ7UbD9HHfU4BsBWTfdkoCTZMDKCYJFqwk7Ml88GjfOYL89FiSxhJ4oNq/WikNoNDEcTa3tsBeWhfvQdLx4YJ8+87VQTEf5a9ZOYvTzGAjgbwCdE9LG67SoAMwA8RkTnA1gB4FR133MApgJYAmAXgPOCFdceSTTQTAPudZaMyM4xlRnz5b5LMFUDQf89YCfDfZRTTejK5lD3Ba0zw7qGiTe+3GL2kMsaafLTCzFa/H4I4/5SiojeVYmWNHYQQsyE8QlM0CkvAPzQV6EkCeIWjtpj4qmnJKfb0Us0ZiBR/BWOQ6JiIDrFL/Hzu9vRMOI+u5z/sjD2COuBtttu3BWPU/LV4xdF8i3mNCpzOyoymAf/Gz0nsl6uKD9n6WwZVukyTLyPQWB1Df16etjzpaGtTfjyw0uHfLGVl0G+Gi6yROn6REkWJpPAYlZjSE52dS+dagE8E/zcxZe89nxlM+K6l3Dkja9nbBPC+MtRerRjgpVXWIQ3vVDytJ0fZ8QvhWDgDzY95FNNmN2nXqZgMU41YX5c5rboPlRurqPZeQVxezsdFOGWxHu+ZJKkpti6ex+27t7ng5tRrsbZyzZ53nKccXvTW72YAvky9bPuCFk40ZGEyca9PkuugacT1u5j3d5XEuXnTrbbMer4JT57viwI8r6Zu2prgK3FHA+0jhOlECF7xxZBK8CYXqbYIfOzJs07ZumpSMd8Wd+FZmXcepqIrKcXMj5WbltU0B2dKXn9zGK+onzObkm88WU6WlE2wz3HTIRClDw7TonDKfDtmWzc/r5x01+U9d+0rM/B3u3Z9uUD65XtsoZL9BVMWN16soQlX+KNrygQM90VC0Ib7RhOs5En+yUQg3dCIgjCMIqa/vIy9sm0pgiMlrSqLyqPmZeeOi8MyjjoHza+LDBz2cftizBumD0/cl+14T+BUQ6SZfKDfNNT6fQOEo9e7vRCXstiXrHxvIL5Q9hqOiwdnXjjy1TxGKb58jjDfZ4pv7jg6KEPW1M4xPfphSS2MCHB+seEbI+tNjGqy5q1sUvuqsqpz2xbGHjq+XInSmxIvPHlFgGT2DDZDPes/TwntLkd7ZbPF03ChEIQuiVqAfuyMTphe53l5pa0uyM+SAfcRzzVhF8k3vhy8tt5/YPn1peAJysA2HCJD2FnqWaM4Y8/Y8y6Hb28hb15HnT8yxF50DyNwwv4lNj4CoGglFLEPhwTQVgqx/b0Qv6IwTAKErrF9Whtd4d7jtUz5dX0Qm4hWKeasDOxdtxwOr1Q1OA8Xw5x6zL3IsM944yofNUx1uRM0xKSHIz3JFnPZd+nmTmmvBxVmewRfPrzTjqsK2DtwQH3EcJrXZNrvCVYmwVEeKkm7Lq+IqwxVaIvIWOEVJJV36UIFjfT1QQKWctiN/9XHHFzLlG4DpznyyGmgx1lXPYQnGQ1gsgo2KgFCkcVL65SbsxXBLQmAyD/ngM33Y5aveL2DiYJGSzndhQ62yKIbg4ySYHNcgRywL0FRHQvEa0jok812zoT0UtEtFj9X6NuJyL6ExEtIaJ5RDTSCxm8RFZZXfXvT9B4xbPG9RjuifJjxJjBMV9MvpFfpps/eDJJd4Q/aLwULeizDOuqeuX5uh/AlKxtVwB4RQgxAMAr6joAHAtggPp3IYDbPZLBNm6VypMff+WJHIx9IqyHGCYw5Lz3yUJ66h0Hr1Uv5xXUzu1oVJXh9kToN9nfKT/xxPgSQrwJYFPW5hMAPKAuPwDgRM32B4XCewCqiaiHF3Loy+a2AufKK3VT5ZnXP9HENc8X34P5i/uwCb55okBEVIkBemkwPKvKV2Ld7WhANyHEGnX5awDd1OVeAFZqyq1St2VARBcS0RwimrN+/XofxczF8zxfifv2ZBgmCuSjbgkq1YTbwH3S1GHorUtwqglZovKBGjSBBNwL5dPJlpYQQtwlhGgSQjTV1dW5aNy0DanDvUhXwYRHmMkSzZRokDEcvk8vlKcKNA7ko4HmFE9jlzx+JqL8jMU51URYPkU/ja+1qe5E9f86dftqAPWacr3VbYmFVZ/3yE2aqxTy8vp7aTBxdw4TCHl2m9mZWNu8Hi9ksdif4FQT8rF5ZnV4I0sU8dP4egrAdHV5OoAnNdvPUUc9jgGwVdM96TlR+OrjlyyTdKI8EivJBKFaIqe+LA0aF1V7nFiV0stG7Rkdq19fVPHymudLqokiLyohor8DOBJALRGtAnANgBkAHiOi8wGsAHCqWvw5AFMBLAGwC8B5XsjgBKkEhS4C7hn/iLIi0sI2CRM27sccsQYMDdYfvmN0if02/DwxvoQQZxjsmqBTVgD4oRftyuDkAnp10UXWf8Y7ZIwaPzyOiTam3GSi9k4KxgayH5BJwjJrfETuYyKNLPbi7XVPIsq6x43H0Gxy8yST+Az3ZnCGe8ZvYqVI+D5lmDTuR0vqL+cLcTlnI8PRb2M3r40vv2n/7fit5jVhPdh2uzuj/LXqJflynlFDbsR2skZr+zm3Y9D3seGLP1gxXKMnr+xtk31sVOJH/b7vE298Obl+XsU4RExnMQyTh0TNeIoymXM7euf6iopBweQS9+mFYomMkaUormR9OSYBL5SZ2+lH5Mrnh9KNywAIPeI4N22QOiVu+itOj5ydUZBRPi9Xeb6yRzu6lsYehiNOudvRHU6Crr1SNunphbypjvGYfBrFFbcXaMDcjxjOTQvk6dyOLvebHkv6y47q0nrRbL7g4/wxk8Jpt2NU4G7HkFEy3LusI2naLwKEF/PFJI0oz01rhB3DIIl5BuNwSt5nuI+u9vHSWPTiNO3cH2EZuok3vkx/g4Ae4CQqvzgRVZXF0wtFGldz0wIIdX5aLa4/HiPmO7O81zxKNeFp2gnD7cnNcC9LPp2rlsQbX07IVjVOVU+0VFayCOuBtR/z5Y8cTHA4mZtWPc6b+WlDhr8dneH1ox9lVeLpnJiRPlPvSLzx5b7L0L3mYd3lPaEF3CdZMUQkOWVEiMXctPloGFkmWfXoRvZ0qiE78wgZb44VsucQVZ3KAfc+YqS3vOomTAfc56GCZBSiqlgYSyIxN60XuNU/2/a0eCOIh8TBo6w1uLwQNw7nrMXxbRd4rjX97bGYXijJCOH8RzA6LG4PEdMO/3YGxPi6xHFu2iA/6Bas2RZcY5KYnb+76YW8M5hk5JAZBZl0nRP2+YXlHEm88WUWLBrURc+WgT1hwWLo4QygQ9hMsURuIEbExAmKKM9Na4VUrsKE/bDWGe79q9trjPN86cztGOEvHDd5vnIz3LuVxhu42zEEcgLuHb4k079dsnQfw+QQ5RdDErGXasI/OZKMp3M75uHjEffbjvN8uSXEOyDuN1/SCcJgyEOdywRIPhpWlpkmXFg6wWdXt2hRE/YSZQPO1TXPOjTCp+kpyTe+TDByx3ut0PJQPyaWKCc6DBO+LNElifoniPvN7bOeebgXo7Pzg6B1bFi6K/HGl1vFI4Q3dWjhF1X+EJXfOh89JEnGzu8ZudhCDzANuHdRr7cvfonphQzl8FCMADAT1zonrvcn6+3P6M+PkXjjKwokLeA1SpQUOb+FK0oLbR8TM53IJJy81Cxxs0y8Iqanbfcejdxp+vTxktfGl/E19WZ0Yj7m+Zp+aJ/A2vrNSUPx3MWHW5YzephnfPMg/PSY/T2R5dbThuPpi8bptG2sSiI3vRAnWU0keaR+ALhNNeEdrlJNpCSJyYMVZ3s4rPdzYo2v5Rt2YsGabaHOayay/ntBRYl9b02QnDm6D8b06xxQWw3o37XS8fE1FSX44VH9bR1jpGROHNELQ3t3ciwLwzhBpksxaR9/YXRjucXu3I5uy0YJu1JHzpDzSaDE5vk68sbXAQDvXZmTqieNcYZ7b2VJYsyFEZF7cDwm4afHMHmNt3MU5g9xNQzDJLGerxRu462UgPvgjaepQ7vrbo/jaLsXfzRed/srPz4iYEmAJ384NtD2YvhzMTFCTjMl6+PP8pnyKMO9WzKmF5LIZG9dn0uBYkK+GHKJN76c8Ne3lqaXW9racMcbS01KW6NVfXv2tebsb+pTk7Ntvzr97rSo35Z68hmdi9F2PxlWXx14mwzjFVt2NeMPL3yOVhve9DxyvDMhEWfDkFNN+ISZ4jHqDnxszqr08j/eX4k3F613KUT74h1vfJGze0iv3FihGN/LTFzhl3Tk+dXT83Hba1+wQWWCq5epT92OXnhz8uWdEGdDzg6JN77csq/VvZbTdlvuas71fOnebHYTw0SEfHlwGCYMtu7el7EuY4QlzU7Ll26pFHHQqTEQMXIkNuA+hVvF4yaPVFoGjRAPvLMce1vaMvYX6E6imhzCPpcwXz5mMXr5NBCD8Ya9Lbkfb1bwbSaPpwH3pL/stL04xvvGAU41EQIyF72k0Pkl0ntUsg0vo3Lxfc5iKzjDRJ7mHP2Rf5ZVXHSjjIcuMV68hJxGkOS18SWDG89XOs+XhX7UUyZGD2Uc7/G4KEs/iNWpx0rY/ETv480KnmEjfLx4tOL2eHYuLwEATB3aw9ZxQb8vjNob278WADCkZ0df2k1+t6NLn2KpF92OFvt1ux3j9qQlgBtOGuJZXT84aj98uWkXvnVwb/z6mfme1RsHjh/WEw2dy8MWI5Fke75k1FubfXuN8QA3Ge6TQKfyYsy9ZjKqSuNpZkwd2gNzfzkZncqLfak/nlfFQ5Zt2Gm6v9hFt2MKSwNQ1/NlUDTiT6uuFy9kmWVbP/ng3p612bWqDPeee4hpmbCvix90rijBn88YEbYYiSW329GatoQFfcXxqUnisy5Dpw72DZcodcX6ZXgBedDtaKZ3Xl+4DkepmfDDRO9my9NnNfIUq57QwgL+gZjgSZYZxbAWyV/y2vO1cO12yzJefDVa1UCkxJZpv2rj+qUUT6nlObWpHm0CGNM3mPkrGcYMGe2UOM9XTJRMXOSMGvly3ULzfBHRFCJaSERLiOgKv9r5etsew316sVbZeKG3rOooIGDqEP3phLLJlxszDGTc3cWFBTh7TB8M6FYVgEQM4562ZNlesUGrT4xTTSRDoUepq9AueZVqgogKAdwG4FgAgwGcQUSD/WjrlDveNdwnY3wF8dVIoBwFGdMcq4lRJgwTB6QC7gN6u0wa3C2QduKiY6rKrDuW4nEmwZIv1yQsz9coAEuEEEuFEM0AHgVwQtBCyBlfXrRkXglRbok4f0nElZjodP9gD0kiCerLvqy4MJiGYkKlZpRf3uuWCJNvczv2ArBSs75K3ZaGiC4kojlENGf9epdzKxogEzPtScyXxGDHpMRlcNZ2hgkOmRxeSXwmN+5oDlsESyplPF9W7yCRfzMUxMWz6ZbIjnYUQtwlhGgSQjTV1dX50obMiLU2D1xf1gH3lKMg43r/JcWIzEties8x5gQV83XJhP7BNATg6AO6Gu5LzUrSpaI0KHF0KS4sQI2aaHTSoMyY3lQCz2wGdqv0Xa6oMEWNcx7UQ0limromWjXkVVd21yrje6FjWXs6iWG9O3nSngxhjXZcDaBes95b3RYoBTLGVwAB90S5iRDjmuerJcbRvdG+sky+I4RAi4OMqUF4vh7+7mj075o5COX2s0bifx7+EABwzqF98OC7KwAAvzhuMM4a3YADfvFfx+3dfNowXHfiEIy87qWM7R9cPTHd/VlXVYrrThyCX/znU0we3A3XnzgEo37ziuM2zTh8QC3eWrwhvf7RLyYBALpUluL9n09EVVkR7n17GQoIeO/KCahTjQEiwoe/mISK0kJs3b0P1R1KsHNvCx59X+0YMlBK7105AR1i2s376a+OAUFJYH7pxIGorSzF7Kvar4mWq6YOctXWn88YgSP2r0NxgbGfqVN5MeZcPREFRCgvCe6ahmV8vQ9gABH1hWJ0nQ7gzKCFCKrb0Qol4F7O8xV1A6E1xsYXw0SZ3z7/OVZu2p2xTS7g3ieBNOgl0yzXxDxVa/b371rpKj6MCCgtKkRpUW4dXSozX+B1lSXpY7p2LMspn31pnKr7bMOhpqIkY9++1najOVuOzmrZrlXK+ZQUlcCK7p1yzyUuaGPhatXfS3tNtO8+t/kUu1SWZHi2jKitDN5LGorxJYRoIaKLALwAoBDAvUKIz4KWQy7VhBfdjuZ1FFCugoxrwH2cpzKJuleRyW8eeGe5o+PCivky9N771N4905tytlmd+rDe1enlCw7vi44ScVpBw2opmYR2pwkhngPwXFjtA8GNdpSbWDsZMV+tHPPF5AFENAXAH6F8PN4thJjhd5t6XgCZxy1pT6TRh+mEQfbig84c3ZBxTX8+zZdsRwCi32MRJfLlIzh6Zn6ABJXnSybgPim9dTLdjldNPUDas3fuYY04pNFZNvmTRvbGSwvW4n+OzAwEnvHNoVi9ZbfBUfr8fOogqZFlepx8cG8cd1APnHvf++ltVWVFuP+8Q3DVvz/Bt0Y6n1PyO2P7YnhDteH+Xxw3GK1iPg7tpx/ga5cLx/fDgq+34ZSmTJlrykswbWgPnDe20ZN2oowmT+EkKCO13yeip4QQvs6gXujwpRRI6ITuBwRrHQAAGIVJREFUnK76K3nybmUYU/Lc+LIu443ny7qS3JgvfeGirrhkFP2F4/eTru/abxzoWJZOHYrx8HfH5Gw/fVSDbnmzS3vB+H6O5bjxlGE52z659hgA0JXPDr883vxrvV9dJR78zihXbWjp2rFMV+aCAsJtZ430rJ2Ik85TCABElMpT6KvxJTNASI+UDou67kgy+eLNYeSJbKqJIJBRZkHESxToeL7i+qgKgdiOwokSReronCASVxapzwEnyZTGMk8h4H2uQt1uRwlvbOqDyK1Oqa00DgTX82QbebfdxrOyHcMkgbz2fMlg5snpVd3BdveVHkS5Rl7cFExVWREmD+6Opj41+P3Jw/DQeyswdWh3zFq6CQBw9zlNkT8nrXyPXDAaGxwkcvzzGSPSo5eyeeSC0Xjs/ZWYdlBPy3qa+tTg4gkDcPaYPult957bhFYfBjSMbKjBJRMG4Kwx+h5BxhlCiLsA3AUATU1Nrr/iZMIk9AVx23IKFw8wx4IyTAZ5bXzJxCeZFZH1ijnJcB9xOyWHq6cNwmmHKC/vuqpSXDZpIADggO5KAr2JAc375hWH7ecsRur4YcaG1WH71UrXW1BA6WuY4ugD/LmGBQWEH2W1xZgSSp7CQp1+CjtzOzo23lTMOgosY7407G1pdSWHV7pxvzrvEpoe3KcGT3yo3AJ6iTpTMh9zYPecfXoM6qHkTDuoVzU2bG/Gvz5chdLi/OqomjjIOJFuEshr40smPsnMQNPb9cSHq/DNrABqq64BgfYUDaVFBdjb0oZCPU0bYZLwYctxGYwkoeQpdGo8eRXz5ep4zcEbdxp7lF++7AhMvPkN6Wpn/3wCRt1gnTg1u6vz2YvHYf9uVQal7XPmqAaMH1CH0uICVJXm5pUqKCC8d+UE1FRY55wCgCP374q3fnYU6juXY9LgbvjJMQNRXpI/r+tZV03QzR2XJOL1hpfkhc++lionM3WQmXdLL63Cba8t0anDWpaUgXbdiUNwzfGD0dSnxvogD7nltGE4Y1R9zvbqcuMH4JrjB2OK+iWXlNGaDGOFEKIFQCpP4QIAjwWRp9Cp8SXSMV8uY61Mjtf1fBmU3WRifNV37mBLpq5VzpKNHtizE4o8/MAlItR3LkfXqjJ0MMiS3r1TmW5iWCPqO5cDAEqKCtCjk73rEne6dSxLfAxqIk3p7z30gVQ5GYPBrIie8eYkI68Q7bI0dC7HmH5dsH77Xqljh9VXY+7KLbbbzOakEb3R1Kcz/j67PY54/25VaGqswcOzvtQ95ryxfbFo7Q4AcoG/DJMUwshTWFTozvPltr/OTLXZMQxPPyTzI+/C8f2w8OvtOHaIXJdcdlMPnT8Kr33ufkCDE348aSAaaysCa+/GU4Zh4w65d0M+89tvHoQbX1yIpj7O0hQFQSKNL1lkDAazbkc9z5eeErLyfAkIDOnZCbOXbUqPKKox8Tj5RfboT5mEqalD2PPFMN6wZ59+TJTTPF/Co9GOZt3yRXqWGQGXThyAW19enLG5ujxzQIp2/j4n8WCHD6jD4QPqbB/nBf87YUCg7Z18sPOcgPlE39oK3HZmtFPfJLLbURaZkWNGXh9A3vMlY5dcOfUAPHXR2PTktEWFBXjiB4el9/940kDMvPwoiZqck63cZbplU4eENYUJwySN95dv0t2ulxonKnM7plKjaON0rLrYJhzQFd8ckZmhQ8bAtNN92qtG6a4bqhMEL0v/rt4F5ieNMf26AADK8mwwgBfkt+fLpcGgp9RSxtearbuxZdc+AMCWXeYpC4QAigsLcFDvzEzlIxva477OGtNHN4VBc4t3uQeydbvMgISUp0/GUGMYxpoWg2fJsedL/fxz+31UYPJ+LVS7RN/46ZEY/uuXAAAj6qvx1mK1O1AIzLl6YkZPwj3nHpJTT1Fhga63zCkH9a7G85cc7ji4/s2fHiUdJJ+P3HjKMFw6cSCqJCavZjLJa3PV7bQbel2SKWPk0N++mt52/bMLXLUDGCvecoPgTidkf1lPGdIj7bUryQpOHaVO+ZM2vtj2YhhvMHiW9FSAgMDu5lb89vkFht2VqWezxeWs9w1qALgexaru0HYpZuuT2spSdOtoHSBvlQLCrg06qEdHx7MDNHQpZ8PChLLiQvYMOiTPjS93x+/WUXbOAu4luvcMfqmqsiLMvmoC5l4zGXOvmYz5vz4G/SQDQOdcPTFjXWvgffzLSfjpMfun168+bhBevmx8ev1v3x2tyJWO+WLri2G8wCgW1Ui33Pv2Mtz5xlLcM3OZfn3qs+lW3910ynDDfZZ6z4bF5DYfGcPEgbzudvTDYHDaNWCFmULqmvU1WV4q5w3rktWNqW0j9QWb2kJE6KiJ5ygpKrCUi2EY+xjFourGkwpgrxp6sM/gQK2aE0JgX6tIP792qCwrQofiQt2PziKzPkmbOHRSMUysyGvPl5PpY6xwooNkbEAjhaS3WbZ3gYhQW1mSziSsJ3sqY/uh/fSH7KbaZ88Xw3iD0QhrJ151IPPZvGfmMgy8+nnP0xUUOkyDoQcnO2bygbz2fD099yvP63TyBShjthh5mPSOtWMGzbl6kmkbY/p1wfIZ0wBAN/dYKpaCbS+G8QajDxmnXnWtLZeaAmfN1j3oUllqqx6z1rWpJt654mhDL5xUO2pVQ3p1RHFhAT76MjOPIX/oMUkgrz1ffuAkK6+c50te8TodxSn7Za0t1h7z5ahJhmGy0PN8/XPOSv0cghL1yegD2aTORmiNr57VHdCni/PEo6nz7FZVphvk78fk8gwTNGx8eYzR1BJuSem2Gd88yJf6lTbkjC9tt0Aq506cv0ZvOW0YDnKRB4hhvETvWfrp4/Mw2yD/lxUyj+Ylj37kqO4UTrtE9etS/hvplDjrGoZJwcaXx3RwkGxOJtN+SrlNHNzNuj6Hukl2wJK2XBKCY08a0RtPXTQubDEYBoD5rBrZyHi1tMaKUelUTkIziICrph5gsM/DmK/0B53+fs4pyCQBNr48priwALOX2ftClTGWjJSbbsC9z92OWllS3jI7LwyGYYzx+lHS1ufWcDn70EZ3wkhglb5GZtozhok6bHxJ0LVKPjC1TQCn3vmurfrtGi5P/OAwDFO7yfSOdGp8yX69aksVWChKhmHsYcdAkor50pTaZzAUWqYeO9P6uCH1QWekUvg7j0kCeWd8mWVpNuKQRvmZ0Z18WRplpjZiZEMNLploPKGrX7oppXq1sWFkoSgZhrGH154drU5aun4nAG+fV68nME4bXwaajLsdmSSQOONr/lfbTPd/uWmX7TrtTE3hRHHqJS2URVcyn3VTZsxXyvhihcgwXuB1F77eXJHZUw258WlNO6iHi6NzSXvTDUY1spedSQKJy/N139v6U2y4wU7+wLY2ASJ7X5Z79nk7dtpKOV1z/GCM0EzabZeM0Y6caoJhPMWOcSFTVM+Yc2LgBZX7NKVfOOaLSTKJM75SU204oa6q1DSRqAytQqCksMCWHHtanHu+9LBSTRMHdUO9g+7XlFKkDM9Xqk1WiAzjBU49X7e+vBirNu/GSSN64ay7ZwEAHv7uaN36tN6w7Xv2Yf4a8x6DIEnrFB7tyCSYxBlfT7nIWm+UaPDVz9dJ19HSZt/46lXdQbpsNk4C7t1+werFfLE+ZBhvsGd8ZZZ9/INVGbrn0fdX4oDuVaZtfL11j20Z/ST1sWuc5ytIaZLLM/87jufmDZHExXxpue/cQ3DZpIHp9Vd+fAROP6Te9Jg7vj0St5w2LGObVQ6ch84flV7etnufLePmz2eMwI8nD7QuaAMrr7zbB04vwz33BDCMe4QQ+PdHq6XLz1y8MWdbdpiEnqdI6/n6z8fy7QWB1QhqjvnyhiG9OmFwz45hi5G3JNb4qigpxFEHdMXFE9pHBe5XV2k57cWUIT1w0ojetto6uE97/NSitduxbU+L9LHHD+uJ0iLnWfH1zCgr3eR2gt4CnTxfHHDPMO55fdF6fGYxaEjLLS8vytmWHSahF3DfqkazN7e04bbXvrAppd+Ye9NZ1TBJILHG19Sh7SNwSgrbT9MsNml4fbXu9s4VJaZtlWmMp7Xb3M2RJkuvaiVmS2v4ZWNkZFk5vgZ0rdTdnjK+tMdT1j6GYZyzZVez6zqKsp57vWezpVXZttdGvGlQPVTtMV8GAffc78gkgMTFfKW4/qQh6eX3fz5RN7ngxUf3x9mHNkJAYMuufbqxV3d8+2AcPqAWm3Y24/Dfv5az/6HzR6GggDDz8qNwy0uL8a8PV+nK8/RF43D8X2a6OKNM9u9ehdd+ciT66ATOp5TW85ccjsm3vJmz36zb8d0rj0ZVWbH+TlXn6WW4Z33IMOHwp1cWZ6x/urrdc/b03K/wvSP65Rxz4UMfYERDNT76covv8tlFq1NqynM/fHm0I5MEEmt8abvyOpW3GxPaLM1dO5ahTs1e37WqTLeeKUO6AwDKivW7BvvWKt2YvWvKUVlq3H3YudLce+aEVNvZpAyhqjL9n9fM+OrRyTj4P1VvRp4vi+BYhmHk8eIxyh65uH1PC4oKCJdNHojXPl+H95dvBgDbhlfgGe4hcPmUA9DS1oa/vfdlev/W3dbzUDJM1Elct+O3RsrHa5l12VWUFKJ7x3aDLNWFd/iA2vR+AOhSUaopY3w5s7sCZEm1Z4d096CBsnQ6GbZevYc0KtfwiIF1ziplGCaNH11q2/e0oLSoAD84sj9uPnW45/V7TerbsLUN6FBSiOtPHJrWt4DxqHSGiROuPF9EdAqAawEMAjBKCDFHs+9KAOcDaAVwsRDiBXX7FAB/BFAI4G4hxAw3MmTzh5MPwoxvDbUsd97YRgzqYTzSY961x+RsW3zDsSgkQkubQHEhYV+rQEmRXDyZU+PrgfNG2fYqpUobObhk53A0qld7Kgf1rsbiG45FcWHi7HiGCRyZ2S6G9uqET1Zvla5z+5596Y/H0mLnz6mTgTpHDKzDrS8vtvVx1rtG8b6fOap9ZPoF4/vh1peV7tVvjuhlW44UB/XulDPLCREwrr/9j1yGcYPbbsdPAXwTwJ3ajUQ0GMDpAA4E0BPAy0SUyqdwG4BJAFYBeJ+InhJCzHcpR5qCAkKBhHtcazTpoadoUgZGibqvpCizjJmNVGTiFTND9nz05DCysRx7vtpSAfeZFbDhxTDecPohDfjlk58BAD6/bgoO+MV/0/t6dirDO1dOAAA0XvGsYR3ZM2xs39PSbnwZjKz+3vh+uPPNpen15TOm5bThxPga0VCD5TOm2Tqmurwk55hLJw7EpRPdp+R56qJxOduW/daefAzjBa7emkKIBUKIhTq7TgDwqBBirxBiGYAlAEapf0uEEEuFEM0AHlXLBsYxB3ZTBBzm/OvJCLN0C4Wa5DupwP5rjh/suQwAcPmU/QEA1R1Kcrph6zt3QAeD+DUrUvFxPznG27xkDMMoaD8Ks+NMZWelyFZDH6zYjM1qrsJSg4/OUoc6gWEYZ/gVcN8LwHua9VXqNgBYmbV9tF4FRHQhgAsBoKGhwTPB+tVV2v4Sk8Wsc1Db7fj2FUf70n6KU5rqcUqT4rK/6dRhuOnUYRZHyFFWXOjbtWOYMIhi6MQjF4zGXnW+15tOGYahvTvhzUXrccLw9g/GyYO74cX5a23XXWLgpf7u4X3ToyZvPEXRFy/+aHx6tPR/Lz3cdlsMwxhjaXwR0csAuuvs+rkQ4knvRVIQQtwF4C4AaGpqisVQOrPYLKeJTRmG8ZXIhU4ctl97/NG3DlY81wO7ZU4RNHVoD0fGl9E8tVWl7a+Ck3XaPKA7Z0JnGC+xNL6EEBMd1LsagHYen97qNphsjz179mXmEhteX42PVyrDuQuJ8JPJA11ls2cYxluEEAsA3UEo6dAJAMuIKBU6AaihE+pxqdAJz4wvGXY2t8+iMa5/LWYu2WC7jj5dyrFi4y6cNKJX+vy/MaxnTrnsUeEjGqqxZN0OdCwrxpqtu223yzCMf92OTwF4hIhuhvLVOADAbCgJ0QcQUV8oRtfpAM70SYbA2ZaVf+Y/PxybDlotKCBcdPQAvcMYhokekQ2dAIDdzcqoyLNGN+C6E4ag31XP6Zabd+1ktLQK3Zx/r1x2BFqFQLE6GGjR9cfmjMpefMOxOXkB/37BGLS0CZRZDFpiGMYYt6kmTgLwZwB1AJ4loo+FEMcIIT4josegfA22APihEKJVPeYiAC9AiZe4VwjxmasziBDaTscyF0O6GYbxjiSGTqTygXWtKjPsSgSAjkazVQAoKizIeAHojQDXG8lslHCaYRh5XBlfQoh/A/i3wb4bANygs/05APqfaTHn+hOHYP9uVagqK8KR+3cFADx78TjMXrYpZMkYJn9JYujEOYc2YuPOZlw4Xpk66E9njMBf31yKm04dhntnLsOgHh0NRzXfe24TdjfnTrfGMExwkFl6hKjQ1NQk5syZY12QYZjEQEQfCCGaAmrrdQA/SY12JKIDATwCJc6rJ4BXoIRPEIBFACZAMbreB3CmlQefdRjD5BdW+iuxczsyDMNYwaETDMOEARtfDMPkLRw6wTBMGHBUOMMwDMMwTICw8cUwDMMwDBMgbHwxDMMwDMMECBtfDMMwDMMwAcLGF8MwDMMwTICw8cUwDMMwDBMgsUiySkTrAaywcUgtAPszzYYPyx0sLHfw2JG9jxCizk9hgsKmDovr78tyB09cZc8HuU31VyyML7sQ0ZygMmN7CcsdLCx38MRZ9qCI6zViuYMnrrKz3NztyDAMwzAMEyhsfDEMwzAMwwRIUo2vu8IWwCEsd7Cw3METZ9mDIq7XiOUOnrjKnvdyJzLmi2EYhmEYJqok1fPFMAzDMAwTSdj4YhiGYRiGCZBEGV9ENIWIFhLREiK6Imx5tBBRPRG9RkTziegzIrpE3d6ZiF4iosXq/xp1OxHRn9RzmUdEI0OWv5CIPiKiZ9T1vkQ0S5XvH0RUom4vVdeXqPsbQ5a7mogeJ6LPiWgBER0ah2tORD9S75NPiejvRFQWxWtORPcS0Toi+lSzzfb1JaLpavnFRDQ9KPmjBusw32Rn/RWs3Ky/rBBCJOIPQCGALwD0A1ACYC6AwWHLpZGvB4CR6nIVgEUABgP4PYAr1O1XAPidujwVwPMACMAYALNClv8yAI8AeEZdfwzA6eryHQD+R13+AYA71OXTAfwjZLkfAPBddbkEQHXUrzmAXgCWAeigudbnRvGaAxgPYCSATzXbbF1fAJ0BLFX/16jLNWHeNyH97qzD/JOd9VdwMrP+ktBfod1UPlzEQwG8oFm/EsCVYctlIu+TACYBWAigh7qtB4CF6vKdAM7QlE+XC0HW3gBeAXA0gGfUm28DgKLsaw/gBQCHqstFajkKSe5OqhKgrO2Rvuaq8lqpPsxF6jU/JqrXHEBjlvKydX0BnAHgTs32jHL58sc6zDc5WX8FKzfrLwn9laRux9QPnmKVui1yqG7VEQBmAegmhFij7voaQDd1OUrncyuAnwFoU9e7ANgihGhR17WypeVW929Vy4dBXwDrAdyndjncTUQViPg1F0KsBnAjgC8BrIFyDT9APK45YP/6RuK6R4DYXIeY6TDWXwHC+kvuuifJ+IoFRFQJ4F8ALhVCbNPuE4rZHKncH0R0HIB1QogPwpbFAUVQXMq3CyFGANgJxY2cJqLXvAbACVCUb08AFQCmhCqUQ6J4fRl3xEmHsf4KHtZfciTJ+FoNoF6z3lvdFhmIqBiK0npYCPGEunktEfVQ9/cAsE7dHpXzGQvgG0S0HMCjUFz3fwRQTURFOrKl5Vb3dwKwMUiBNawCsEoIMUtdfxyKMov6NZ8IYJkQYr0QYh+AJ6D8DnG45oD96xuV6x42kb8OMdRhrL+Ch/WXxHVPkvH1PoAB6oiKEiiBe0+FLFMaIiIA9wBYIIS4WbPrKQCp0RHTocRRpLafo46wGANgq8YVGhhCiCuFEL2FEI1QrumrQoizALz2/+3dP0pDQRCA8W+riJ0eIY2thYWFhSCksM4F1GNYeQhPYGFhYyvoBYKFqIjo62y8g0UsdoIgihJ03qLfD7bI5hWzk8cw788SYPxJ3LP1jOP4Xq7MptPpM/BUSlmJqS3gjsZzTr1dv15KWYzzZhZ38zn/IJ7v5PcMGJVSluKqeRRz/4017IdZv6xfc8ipX1kvtWUM6m6EB+qOof2+43kX2wb19uU1cBVjm/ps+wJ4BM6B5Ti+AIexlhtgrYE1bPK2W2gITIAOOAEGMb8Qn7v4fthzzKvAZeT9lLobpfmcAwfAPXALHAGDFnMOHFPf63ihXqnvzZNfYDfi74Cdvs/1Hn93a9jvxW/9yovb+vXF8O+FJEmSEv2lx46SJEnNs/mSJElKZPMlSZKUyOZLkiQpkc2XJElSIpsvSZKkRDZfkiRJiV4BnU08GchId2oAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "env.render()"
      ],
      "metadata": {
        "id": "GzLRp9BLNpi_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 286
        },
        "outputId": "d6e557b0-ac45-49e6-ded9-71b5b60bf650"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7fea73624e50>"
            ]
          },
          "metadata": {},
          "execution_count": 15
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQcAAAD8CAYAAAB6iWHJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAMzUlEQVR4nO3db8yddX3H8fdnpZYxRP6Z0JUKKozMPwykqRiShYBkQAxdImbwQMFAuhiZuOzBzJawzCfiHmhiMCwNkIExigFlnSkhJWDUbDBKUzooQzue0K4Zyp9ig+JKvntwrrqb29/d0p7rXOfc3O9XctLrz6/n+zuBfHqd67rO9U1VIUnz/c60JyBpNhkOkpoMB0lNhoOkJsNBUpPhIKlprHBIcmKSzUl+2v15wgLjXk+yrXttHKempGFknPsckvwD8GJV3ZzkC8AJVfXXjXH7qurYMeYpaWDjhsMzwIVVtSfJSuAHVXVWY5zhIC0y44bDy1V1fLcc4KUD6/PG7Qe2AfuBm6vqvgXebz2wHuDoY3Leu97ztiOe26za8+Qx057CxPzB2a9Oewo6TI9vf+3nVfXO1r5DhkOSB4FTGrv+Frhzbhgkeamqfuu8Q5JVVbU7yXuAh4CLq+q/Dlb3rA8eXbduPO2gc1uMvvTes6c9hYl54L+3TXsKOkzLVu58vKrWtPYddai/XFUfXWhfkv9JsnLO14rnF3iP3d2fzyb5AXAucNBwkDRd417K3Ahc0y1fA/zz/AFJTkiyols+GbgA2DFmXUkTNm443AxckuSnwEe7dZKsSXJbN+YPgS1JngAeZnTOwXCQZtwhv1YcTFW9AFzc2L4FuL5b/lfgg+PUkTQ875CU1GQ4SGoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUZDhIauolHJJcmuSZJDu7zlfz969Icne3/9Ekp/dRV9LkjB0OSZYBXwcuA94HXJ3kffOGXceo4c0ZwFeBL49bV9Jk9XHksBbYWVXPVtWvgW8D6+aNWQfc2S3fA1zcdciSNKP6CIdVwHNz1nd125pjqmo/sBc4qYfakiZkpk5IJlmfZEuSLS+/+Pq0pyMtaX2Ew25g9Zz1U7ttzTFJjgLeAbww/42qakNVramqNcefuKyHqUk6Un2Ew2PAmUneneRtwFWM2uTNNbdt3pXAQzVOe29JEzdWxysYnUNIcgPwALAMuKOqnkryRWBLVW0Ebge+kWQn8CKjAJE0w8YOB4Cq2gRsmrftpjnLvwI+0UctScOYqROSkmaH4SCpyXCQ1GQ4SGoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUNFSvzGuT/CzJtu51fR91JU3O2A+YndMr8xJG3a4eS7KxqnbMG3p3Vd0wbj1Jw+jj6dO/6ZUJkORAr8z54XBY9jx5DF9679k9TE9D+ZPfP2faU9Bh27ngnqF6ZQJ8PMn2JPckWd3Y/4Z2eP/Laz1MTdKRGuqE5L8Ap1fV2cBm/r/j9hvMbYe3nBUDTU1SyyC9Mqvqhao6cChwG3BeD3UlTdAgvTKTrJyzegXwdA91JU3QUL0yP5fkCmA/o16Z145bV9JkZVabXR+XE+vDuXja05De0h6sex6vqjWtfd4hKanJcJDUZDhIajIcJDUZDpKaDAdJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNTUVzu8O5I8n+TJBfYnyde6dnnbk3yoj7qSJqevI4d/Ai49yP7LgDO713rg1p7qSpqQXsKhqn7I6KnSC1kH3FUjjwDHz3tcvaQZM9Q5hzfVMs92eNLsmKkTkrbDk2bHUOFwyJZ5kmbLUOGwEfhUd9XifGBvVe0ZqLakIzB2OzyAJN8CLgROTrIL+DtgOUBV/SOwCbgc2Am8Cny6j7qSJqeXcKiqqw+xv4DP9lFL0jBm6oSkpNlhOEhqMhwkNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUZDhIajIcJDUZDpKaDAdJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoaqh3ehUn2JtnWvW7qo66kyenlGZKM2uHdAtx1kDE/qqqP9VRP0oQN1Q5P0iIz5DmHjyR5Isn9Sd7fGmA7PGl29PW14lC2AqdV1b4klwP3Meq4/QZVtQHYAHBcTqyB5iapYZAjh6p6par2dcubgOVJTh6itqQjM0g4JDklSbrltV3dF4aoLenIDNUO70rgM0n2A78Eruq6YEmaUUO1w7uF0aVOSYuEd0hKajIcJDUZDpKaDAdJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNY0dDklWJ3k4yY4kTyW5sTEmSb6WZGeS7Uk+NG5dSZPVxzMk9wN/VVVbk7wdeDzJ5qraMWfMZYz6VJwJfBi4tftT0owa+8ihqvZU1dZu+RfA08CqecPWAXfVyCPA8UlWjltb0uT0es4hyenAucCj83atAp6bs76L3w4Q2+FJM6S3cEhyLHAv8PmqeuVI3qOqNlTVmqpas5wVfU1N0hHoJRySLGcUDN+squ82huwGVs9ZP7XbJmlG9XG1IsDtwNNV9ZUFhm0EPtVdtTgf2FtVe8atLWly+rhacQHwSeA/kmzrtv0N8C74TTu8TcDlwE7gVeDTPdSVNEFjh0NV/RjIIcYU8Nlxa0kajndISmoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUZDhIajIcJDUN1Q7vwiR7k2zrXjeNW1fSZA3VDg/gR1X1sR7qSRrAUO3wJC0yQ7XDA/hIkieS3J/k/Qv8fdvhSTOij68VwCHb4W0FTquqfUkuB+5j1HH7DapqA7AB4LicWH3NTdLhG6QdXlW9UlX7uuVNwPIkJ/dRW9JkDNIOL8kp3TiSrO3qvjBubUmTM1Q7vCuBzyTZD/wSuKrrgiVpRg3VDu8W4JZxa0kajndISmoyHCQ1GQ6SmgwHSU2Gg6Qmw0FSk+EgqclwkNRkOEhqMhwkNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUZDhIajIcJDX18YDZo5P8e9eT4qkkf98YsyLJ3Ul2Jnm0628haYb1ceTwGnBRVf0RcA5waZLz5425Dnipqs4Avgp8uYe6kiaoj3Z4daAnBbC8e81/svQ64M5u+R7g4gOPqpc0m/pqarOseyz988DmqprfDm8V8BxAVe0H9gIn9VFb0mT0Eg5V9XpVnQOcCqxN8oEjeR97ZUqzo9erFVX1MvAwcOm8XbuB1QBJjgLeQaPjVVVtqKo1VbVmOSv6nJqkw9TH1Yp3Jjm+W/5d4BLgP+cN2whc0y1fCTxkxytptvXRDm8lcGeSZYzC5jtV9f0kXwS2VNVGRr00v5FkJ/AicFUPdSVNUB/t8LYD5za23zRn+VfAJ8atJWk43iEpqclwkNRkOEhqMhwkNRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDUZDhIajIcJDUZDpKaDAdJTYaDpCbDQVKT4SCpyXCQ1GQ4SGoaqlfmtUl+lmRb97p+3LqSJquPp08f6JW5L8ly4MdJ7q+qR+aNu7uqbuihnqQB9PH06QIO1StT0iLTx5EDXc+Kx4EzgK83emUCfDzJHwM/Af6yqp5rvM96YH23uu/BuueZPub3Jp0M/HzAekPxcy0+Q3620xbakT4bT3Wdr74H/EVVPTln+0nAvqp6LcmfA39WVRf1VrgHSbZU1Zppz6Nvfq7FZ1Y+2yC9Mqvqhao60Bn3NuC8PutK6t8gvTKTrJyzegXw9Lh1JU3WUL0yP5fkCmA/o16Z1/ZQt28bpj2BCfFzLT4z8dl6Pecg6a3DOyQlNRkOkpqWfDgkuTTJM0l2JvnCtOfTlyR3JHk+yZOHHr14JFmd5OEkO7rb9W+c9pz68GZ+hjD4nJbyOYfuJOpPGF1h2QU8BlxdVTumOrEedDec7QPuqqoPTHs+femufK2sqq1J3s7o5rs/Xez/zZIE+L25P0MAbmz8DGEwS/3IYS2ws6qerapfA98G1k15Tr2oqh8yujL0llJVe6pqa7f8C0aXxVdNd1bjq5GZ+hnCUg+HVcDc27h38Rb4H22pSHI6cC7Qul1/0UmyLMk24Hlg8wI/QxjMUg8HLVJJjgXuBT5fVa9Mez59qKrXq+oc4FRgbZKpfh1c6uGwG1g9Z/3UbptmWPed/F7gm1X13WnPp28L/QxhaEs9HB4Dzkzy7iRvA64CNk55TjqI7sTd7cDTVfWVac+nL2/mZwhDW9LhUFX7gRuABxid2PpOVT013Vn1I8m3gH8DzkqyK8l1055TTy4APglcNOfJYpdPe1I9WAk8nGQ7o3+0NlfV96c5oSV9KVPSwpb0kYOkhRkOkpoMB0lNhoOkJsNBUpPhIKnJcJDU9H9b8//c5ewYxgAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "EtTlr8lIu9VV"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}