第15章-模仿学习.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 17634,
     "status": "ok",
     "timestamp": 1649956504817,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "Q0_qiuMECqjn",
    "outputId": "e923fc32-68ab-48ba-97ca-2e169458759b"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration 0:   0%|          | 0/25 [00:00<?, ?it/s]/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:48: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)\n",
      "Iteration 0: 100%|██████████| 25/25 [00:00<00:00, 29.67it/s, episode=20, return=40.700]\n",
      "Iteration 1: 100%|██████████| 25/25 [00:01<00:00, 15.19it/s, episode=45, return=182.800]\n",
      "Iteration 2: 100%|██████████| 25/25 [00:01<00:00, 14.11it/s, episode=70, return=176.100]\n",
      "Iteration 3: 100%|██████████| 25/25 [00:01<00:00, 14.44it/s, episode=95, return=191.500]\n",
      "Iteration 4: 100%|██████████| 25/25 [00:01<00:00, 14.45it/s, episode=120, return=151.300]\n",
      "Iteration 5: 100%|██████████| 25/25 [00:02<00:00, 12.15it/s, episode=145, return=200.000]\n",
      "Iteration 6: 100%|██████████| 25/25 [00:01<00:00, 13.47it/s, episode=170, return=200.000]\n",
      "Iteration 7: 100%|██████████| 25/25 [00:01<00:00, 13.20it/s, episode=195, return=200.000]\n",
      "Iteration 8: 100%|██████████| 25/25 [00:01<00:00, 14.43it/s, episode=220, return=188.100]\n",
      "Iteration 9: 100%|██████████| 25/25 [00:01<00:00, 13.13it/s, episode=245, return=200.000]\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "import random\n",
    "import rl_utils\n",
    "\n",
    "\n",
    "class PolicyNet(torch.nn.Module):\n",
    "    def __init__(self, state_dim, hidden_dim, action_dim):\n",
    "        super(PolicyNet, self).__init__()\n",
    "        self.fc1 = torch.nn.Linear(state_dim, hidden_dim)\n",
    "        self.fc2 = torch.nn.Linear(hidden_dim, action_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        return F.softmax(self.fc2(x), dim=1)\n",
    "\n",
    "\n",
    "class ValueNet(torch.nn.Module):\n",
    "    def __init__(self, state_dim, hidden_dim):\n",
    "        super(ValueNet, self).__init__()\n",
    "        self.fc1 = torch.nn.Linear(state_dim, hidden_dim)\n",
    "        self.fc2 = torch.nn.Linear(hidden_dim, 1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        return self.fc2(x)\n",
    "\n",
    "\n",
    "class PPO:\n",
    "    ''' PPO算法,采用截断方式 '''\n",
    "    def __init__(self, state_dim, hidden_dim, action_dim, actor_lr, critic_lr,\n",
    "                 lmbda, epochs, eps, gamma, device):\n",
    "        self.actor = PolicyNet(state_dim, hidden_dim, action_dim).to(device)\n",
    "        self.critic = ValueNet(state_dim, hidden_dim).to(device)\n",
    "        self.actor_optimizer = torch.optim.Adam(self.actor.parameters(),\n",
    "                                                lr=actor_lr)\n",
    "        self.critic_optimizer = torch.optim.Adam(self.critic.parameters(),\n",
    "                                                 lr=critic_lr)\n",
    "        self.gamma = gamma\n",
    "        self.lmbda = lmbda\n",
    "        self.epochs = epochs  # 一条序列的数据用于训练轮数\n",
    "        self.eps = eps  # PPO中截断范围的参数\n",
    "        self.device = device\n",
    "\n",
    "    def take_action(self, state):\n",
    "        state = torch.tensor([state], dtype=torch.float).to(self.device)\n",
    "        probs = self.actor(state)\n",
    "        action_dist = torch.distributions.Categorical(probs)\n",
    "        action = action_dist.sample()\n",
    "        return action.item()\n",
    "\n",
    "    def update(self, transition_dict):\n",
    "        states = torch.tensor(transition_dict['states'],\n",
    "                              dtype=torch.float).to(self.device)\n",
    "        actions = torch.tensor(transition_dict['actions']).view(-1, 1).to(\n",
    "            self.device)\n",
    "        rewards = torch.tensor(transition_dict['rewards'],\n",
    "                               dtype=torch.float).view(-1, 1).to(self.device)\n",
    "        next_states = torch.tensor(transition_dict['next_states'],\n",
    "                                   dtype=torch.float).to(self.device)\n",
    "        dones = torch.tensor(transition_dict['dones'],\n",
    "                             dtype=torch.float).view(-1, 1).to(self.device)\n",
    "        td_target = rewards + self.gamma * self.critic(next_states) * (1 -\n",
    "                                                                       dones)\n",
    "        td_delta = td_target - self.critic(states)\n",
    "        advantage = rl_utils.compute_advantage(self.gamma, self.lmbda,\n",
    "                                               td_delta.cpu()).to(self.device)\n",
    "        old_log_probs = torch.log(self.actor(states).gather(1,\n",
    "                                                            actions)).detach()\n",
    "\n",
    "        for _ in range(self.epochs):\n",
    "            log_probs = torch.log(self.actor(states).gather(1, actions))\n",
    "            ratio = torch.exp(log_probs - old_log_probs)\n",
    "            surr1 = ratio * advantage\n",
    "            surr2 = torch.clamp(ratio, 1 - self.eps,\n",
    "                                1 + self.eps) * advantage  # 截断\n",
    "            actor_loss = torch.mean(-torch.min(surr1, surr2))  # PPO损失函数\n",
    "            critic_loss = torch.mean(\n",
    "                F.mse_loss(self.critic(states), td_target.detach()))\n",
    "            self.actor_optimizer.zero_grad()\n",
    "            self.critic_optimizer.zero_grad()\n",
    "            actor_loss.backward()\n",
    "            critic_loss.backward()\n",
    "            self.actor_optimizer.step()\n",
    "            self.critic_optimizer.step()\n",
    "\n",
    "\n",
    "actor_lr = 1e-3\n",
    "critic_lr = 1e-2\n",
    "num_episodes = 250\n",
    "hidden_dim = 128\n",
    "gamma = 0.98\n",
    "lmbda = 0.95\n",
    "epochs = 10\n",
    "eps = 0.2\n",
    "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\n",
    "    \"cpu\")\n",
    "\n",
    "env_name = 'CartPole-v0'\n",
    "env = gym.make(env_name)\n",
    "env.seed(0)\n",
    "torch.manual_seed(0)\n",
    "state_dim = env.observation_space.shape[0]\n",
    "action_dim = env.action_space.n\n",
    "ppo_agent = PPO(state_dim, hidden_dim, action_dim, actor_lr, critic_lr, lmbda,\n",
    "                epochs, eps, gamma, device)\n",
    "\n",
    "return_list = rl_utils.train_on_policy_agent(env, ppo_agent, num_episodes)\n",
    "\n",
    "# Iteration 0: 100%|██████████| 25/25 [00:00<00:00, 32.56it/s, episode=20,\n",
    "# return=40.700]\n",
    "# Iteration 1: 100%|██████████| 25/25 [00:09<00:00,  2.75it/s, episode=45,\n",
    "# return=182.800]\n",
    "# Iteration 2: 100%|██████████| 25/25 [00:11<00:00,  2.27it/s, episode=70,\n",
    "# return=176.100]\n",
    "# Iteration 3: 100%|██████████| 25/25 [00:11<00:00,  2.16it/s, episode=95,\n",
    "# return=194.500]\n",
    "# Iteration 4: 100%|██████████| 25/25 [00:11<00:00,  2.08it/s, episode=120,\n",
    "# return=180.600]\n",
    "# Iteration 5: 100%|██████████| 25/25 [00:12<00:00,  2.03it/s, episode=145,\n",
    "# return=200.000]\n",
    "# Iteration 6: 100%|██████████| 25/25 [00:11<00:00,  2.08it/s, episode=170,\n",
    "# return=185.700]\n",
    "# Iteration 7: 100%|██████████| 25/25 [00:11<00:00,  2.14it/s, episode=195,\n",
    "# return=200.000]\n",
    "# Iteration 8: 100%|██████████| 25/25 [00:12<00:00,  2.05it/s, episode=220,\n",
    "# return=200.000]\n",
    "# Iteration 9: 100%|██████████| 25/25 [00:11<00:00,  2.27it/s, episode=245,\n",
    "# return=196.900]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "executionInfo": {
     "elapsed": 524,
     "status": "ok",
     "timestamp": 1649956505337,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "HsyfI_zoCqjq"
   },
   "outputs": [],
   "source": [
    "def sample_expert_data(n_episode):\n",
    "    states = []\n",
    "    actions = []\n",
    "    for episode in range(n_episode):\n",
    "        state = env.reset()\n",
    "        done = False\n",
    "        while not done:\n",
    "            action = ppo_agent.take_action(state)\n",
    "            states.append(state)\n",
    "            actions.append(action)\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            state = next_state\n",
    "    return np.array(states), np.array(actions)\n",
    "\n",
    "\n",
    "env.seed(0)\n",
    "torch.manual_seed(0)\n",
    "random.seed(0)\n",
    "n_episode = 1\n",
    "expert_s, expert_a = sample_expert_data(n_episode)\n",
    "\n",
    "n_samples = 30  # 采样30个数据\n",
    "random_index = random.sample(range(expert_s.shape[0]), n_samples)\n",
    "expert_s = expert_s[random_index]\n",
    "expert_a = expert_a[random_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 184885,
     "status": "ok",
     "timestamp": 1649956690218,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "QLLD1U-1Cqjr",
    "outputId": "173897ed-3262-4a90-c2c2-ff125678f624"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "进度条: 100%|██████████| 1000/1000 [03:05<00:00,  5.40it/s, return=199.320]\n"
     ]
    }
   ],
   "source": [
    "class BehaviorClone:\n",
    "    def __init__(self, state_dim, hidden_dim, action_dim, lr):\n",
    "        self.policy = PolicyNet(state_dim, hidden_dim, action_dim).to(device)\n",
    "        self.optimizer = torch.optim.Adam(self.policy.parameters(), lr=lr)\n",
    "\n",
    "    def learn(self, states, actions):\n",
    "        states = torch.tensor(states, dtype=torch.float).to(device)\n",
    "        actions = torch.tensor(actions).view(-1, 1).to(device)\n",
    "        log_probs = torch.log(self.policy(states).gather(1, actions))\n",
    "        bc_loss = torch.mean(-log_probs)  # 最大似然估计\n",
    "\n",
    "        self.optimizer.zero_grad()\n",
    "        bc_loss.backward()\n",
    "        self.optimizer.step()\n",
    "\n",
    "    def take_action(self, state):\n",
    "        state = torch.tensor([state], dtype=torch.float).to(device)\n",
    "        probs = self.policy(state)\n",
    "        action_dist = torch.distributions.Categorical(probs)\n",
    "        action = action_dist.sample()\n",
    "        return action.item()\n",
    "\n",
    "\n",
    "def test_agent(agent, env, n_episode):\n",
    "    return_list = []\n",
    "    for episode in range(n_episode):\n",
    "        episode_return = 0\n",
    "        state = env.reset()\n",
    "        done = False\n",
    "        while not done:\n",
    "            action = agent.take_action(state)\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            state = next_state\n",
    "            episode_return += reward\n",
    "        return_list.append(episode_return)\n",
    "    return np.mean(return_list)\n",
    "\n",
    "\n",
    "env.seed(0)\n",
    "torch.manual_seed(0)\n",
    "np.random.seed(0)\n",
    "\n",
    "lr = 1e-3\n",
    "bc_agent = BehaviorClone(state_dim, hidden_dim, action_dim, lr)\n",
    "n_iterations = 1000\n",
    "batch_size = 64\n",
    "test_returns = []\n",
    "\n",
    "with tqdm(total=n_iterations, desc=\"进度条\") as pbar:\n",
    "    for i in range(n_iterations):\n",
    "        sample_indices = np.random.randint(low=0,\n",
    "                                           high=expert_s.shape[0],\n",
    "                                           size=batch_size)\n",
    "        bc_agent.learn(expert_s[sample_indices], expert_a[sample_indices])\n",
    "        current_return = test_agent(bc_agent, env, 5)\n",
    "        test_returns.append(current_return)\n",
    "        if (i + 1) % 10 == 0:\n",
    "            pbar.set_postfix({'return': '%.3f' % np.mean(test_returns[-10:])})\n",
    "        pbar.update(1)\n",
    "\n",
    "# 进度条: 100%|██████████| 1000/1000 [00:50<00:00, 19.82it/s, return=42.000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 295
    },
    "executionInfo": {
     "elapsed": 22,
     "status": "ok",
     "timestamp": 1649956690807,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "RWdMQsYkCqjr",
    "outputId": "d235f1c4-a237-4763-852d-30f1b677a397"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2dd5wb1dWwn6OtLruu627jggumuGBMB9Nr6IFAPiCUAElIwgsJgQCBF94kJLSEECCQECABAoFQQzfNVGODMW5g4957XW/V/f6YGWk0mpFGWmm15Ty/37KjO3fu3Fnhe+aUe44YY1AURVEUgEihJ6AoiqK0HFQoKIqiKDFUKCiKoigxVCgoiqIoMVQoKIqiKDFUKCiKoigxVCgoSitGRCaJyPJCz0NpO6hQUFoEIrJYRHaKyHYR2SQi/xWRgZ4+54jINLvPKhF5RUQOaqb55fTeImJEZFfX50kiErXH3yYiX4nIBbmZfdZzPEdElojIDhF5TkS6F3I+SvOgQkFpSXzLGNMZ6AusAf7knBCRK4E/AL8BegODgHuBk/M9qVzeW0SKU5xeaT9/JfAL4EERGZ35jJuOiOwO/AU4F+uZq7GeWWnrGGP0R38K/gMsBo50fT4e+No+7gJsB76dwXhdgEeBdcAS4HogYp/7HvA+cDuwCVgEHJdinJT3BiYCHwGbgVXAPUCp67wBfgTMt+/1nt22wx77LGASsNwz7jrgDKAMSyittH/+AJTZfRKuA/oBz9jXLgJ+EjDnfsBOoLurbRywHijBEoCPu84NA+qAikL/v6I/+f1RTUFpcYhIR6yF8mO7aX+gHHg2g2H+hLWgDwUOBc4D3OaYfYGvgJ7A74G/iYj4jBPm3o3A/9hj7Q8cAfzQ0+cU+56jjTGH2G1jjDGdjTFPujuKSERETgW6Al8C1wH7AWOBMVhC6HrvJEQkArwIfAH0t+dxhYgc4+1rjFmJJchOdzWfAzxtjKkHdrfHcfp/gyUURqT4OyhtABUKSkviORHZDGwBjgJus9t7AOuNMQ1hBhGRIuA7wLXGmG3GmMXAHVimEIclxpgHjTGNwCNYJqvePsOlvbcxZrox5mNjTIN9r79gCSI3vzXGbDTG7Ewx9X72868HbgTONcZ8BXwXuNkYs9YYsw74X8+zOOwDVBljbjbG1BljFgIP2n8LPx4HzgawBeJ37DaAzljfg5stQEWK+SttgFT2TUVpbk4xxrxpL+onA+/aNvUNQE8RKQ4pGHpimUCWuNqWYL09O6x2Dowx1baS0NlnrLT3FpERwJ3ABKAj1r+r6Z5uy0LMe6UxZoBPez+Sn6WfT79diAsWhyJgij3P7a720Vhmpj+JSF8sDSDq9MUya1V6xq8EtoV4DqUVo5qC0uIwxjQaY/6DZZY5CMvMUYtlggnDeqAea5F0GASsyGI6Ye59HzAPGG6MqQR+CXhNUU1JR7yS5GdZ6dNvGbDIGNPV9VNhjDkewDZVOT9LjTGbgNexTHXnAP8yxjjznI1lqgJARIZi+Ta+bsJzKK0AFQpKi0MsTga6AXONMVuAXwF/FpFTRKSjiJSIyHEi8nvv9bZJ6Cng1yJSISK7AFcC/8x0LiHvXQFsBbaLyCjgByGGXoPl7wjDE8D1IlIlIj3t+fg9y1Rgm4j8QkQ6iEiRiOwhIvukGPtxLH/LGcRNRwCPAd8SkYNFpBNwM/AfY4xqCm2dQnu69Ud/jIlFH+3EMltsA2YB3/X0+S4wDStqZzXwX+CAgPG6YS2c67DeoH+FJ/rI098Au6aYX+C9gUOwNIXtWOaXm93j+40NXIYVqbQZOBOf6CNX33Lgbrv/Kvu43D6XcB2WWekJe46bsJz1R6Z4rg7233u2z7lzgKX2Mz+PK1JJf9ruj9hfvqIoiqKo+UhRFEWJo0JBURRFiaFCQVEURYmhQkFRFEWJ0ao3r/Xs2dMMHjy40NNQFEVpVUyfPn29MabK71yrFgqDBw9m2rRphZ6GoihKq0JElgSdU/ORoiiKEkOFgqIoihJDhYKiKIoSQ4WCoiiKEkOFgqIoihIjb0JBRAaKyNsiMkdEZovIT+327iLyhojMt393s9tFRO4WkQUiMlNExudrboqiKIo/+dQUGoCrjDGjsUoJ/sgumHINMNkYMxyYbH8GOA4Ybv9cgpWjXlEURWlG8rZPwRjjpPnFGLNNROZiVb46GSvdL1hlEN8BfmG3P2qstK0fi0hXEelrj6MoSgtk8tw17N6vC1UVZTwzfTmnje9PcVHqd80PF6xnY3Ud67fVcv4BgxERXpu9mnEDu9KrsjzWb9nGap6evpzquga6dypjZJ/O1NZHOW7PvrE+b8xZw14DutDbdR3A67NX89a8tYzuV8m5++2Cf/ltaIwafvPyXMpLIozoXcHHCzdw8cFD+Wr1NiYO6c5bc9fSubyYCbt04/U5a9hcXceWnfWs3FwDwOl79+ewkb14evpyvjWmH+UlRQC8Nns1b8xZw9CqTkSjhqFVndlncHeemLqU0uIIB+3ak/lrt7GzLsqG7bXUNUbp26UD5+w7iO21Dfxr6lK+XrONPl06gDEsXL8DAwzr2YktO+t59+t1/PyYUZywV1/f52oKzbJ5TUQGA+OAT4DeroV+NfG6uP1JLFm43G5LEAoicgmWJsGgQYPyNmdFUdJz0SPT6NulnB8etis3PDeL7bUNXHjQkJTXnPPXT2LHh43qRf+uHbj0H9MZ2rMTb/1sUuzc716dx0szk98J/3reBI4c3ZvGqOH7j05jcI+OvPPzwxL6XPKPeDXUw0b2YmD3jr5zefLTZfzt/UUJbU9MDVM51eK/X67ir+dN4OdPz2T+2u388vjdALj0H95qrHD6+AE889nylOMdu0cf/vfF2Tw/w6+wXiJPT1+WF6GQd0eziHTGqgV7hTFmq/ucrRVkVNDBGPOAMWaCMWZCVZXvLm1FUZqRVVtq2Li9DoBN1XUZXVtd10jUXgEWrt+RcG7jDv+xLn50GtOXbKSuIQrAko3VKe/xx8nz2VZT73tua0B7JjjPvGF76mdPJxAAdtY3snhD8PN8Z5+BseMg7aep5FVTEJESLIHwmLFq7gKsccxCdsHwtXb7CmCg6/IBZFdTV1GUZsCvQFemy1RtQ5RoQKGvokjwaFt3NlBT3whAxLM4euf19PTlFEeEA3ftSW1DlDP2HhA7V5ziHmFx5p/GahaK2vpG6m1hVyjyGX0kwN+wauze6Tr1AnC+fXw+Vpk/p/08OwppP2CL+hMUpeUSda29JqTC712wnYXdD+9i70bEEihWv8RztT6L6rbaBn78xOf87N9fALBi804gN0KhIeoIBWs5bUo1y9qGKPWNqYXCfd/Nb2BmPs1HBwLnAoeLyAz753jgVuAoEZkPHGl/BngZWAgsAB4EfpjHuSmK0kR83/BdC/klj07jkN+/nXC6vjHxmlSaQqoFOyJCbUOjfUtJMAP5CZoi17yen7GCA299izfnrCGSC6HQmKgpeJ8xE4774xTmr90eeN6YuAaVr1LK+Yw+ep9gbfIIn/4G+FG+5qMoij9vz1vL4J6dGNKzU0bXuRdzv/Xp9Tlrktqchdyhpr4xQeMAmLtqK5uq61Iu2JZQsN6o6xqi7HXT69x2xl58e8JAdvoJBddYny3ZBFi+iZ6dSwPvEZbqOut+juBJ96bfVEb1qQTgpLH98jJ+q06drShK07ng4U8BWHzrCRldl82Lak194oLppykc98cp1u89+gSOE5FkjeDdr9dZQqEuWSgEWaLWp3EOh2FnXQMQNx/V5dknMKhHR+b/+jhKcuHE8EHTXCiKkhV+Zp90xhg/TcEErKFuTWHfId2TbuT1HYgI377/Q464892ksYryFKkDcU3hoQ8WMXvllrxqCiePs7SDfAkEUKGgKEoajDE0+Cx0brPPV6u3hRrLu5CH9Sl44/GNgVqP1iHAp4s3+WowxUXxsXIdyuk2V73z1TpfR3cuGDeoKwcM65mXsd2oUFAUJSWPT13Krte9wvw1iQt/o0sqbNkZLt7fu5Bvr2kIjFtyL93ehbwhapK0Dr+1vmOptcM4VSRTU9la0xA7Nsbk3aeQb1QoKIqSkqenW5uujrrrvYR2d/SL48hNt/Z6F8zVW3YGagpuTcTrc26MRmOhoA5+t+7SoQRIXLiXb0q92S1TXvwivvt444566gKEQv+uHXj0wok5vXc+UKGgKAoAg6/5Ly/NTE6vEPWGBzntrmZHGPzl3YVJ/f7j2snbEE1cMFdsDhYKja528Sz5DY0mKSTTzyz02MX7AokL95tz1yb1yxUPfbCIeauSTWnXHjeKyVcdyiEj0mdhuPrYkfmYWmhUKChKO2D1lhp21Dak7Xf35PlJbQEywXcx9wsHvfKpL9hkp6xwx/Dv1reSbTUNgVFMbmGUrCkYvC/kflrK0KrOMW2hubjzja+T2vp36xBLlpeOM8YPSN8pj6hQUJQ2yM66Ru584+tYeOR+v53MGfd/lPa6r9ds5/FPlia0BZt34u1T5q9POa7Tt8ElFMpLIhiTOM4pf/4gduz4LH4waViST2DKgvVJ8/JqEw5hInWeunT/tH3CstQnF9PQnp1DX5/PyKIwqFBQlDbIn99ewN2T5/OvT+ML/NxVW5P6+e2K/eWzX1JT38isFVuAYE0h1T6F/3qymzqmnffmr4u1lRcXETUmYZwZyzbHjmsboozuW8kvjh2VpAU8/snS2Pzi9/CfS5g1tqqiLH2nLPnhpGHs1rcidP/SYhUKiqLYfLZ0E/e/+03g+Qfe+4bp9o7cVDhmnHQbqYIW9muemcmJf3qfddtqU/gU/NuNMby/IFFziAis2VrDA+/FfQ7FRULUmMBxahsaXQ7s5BX/L+8l+i/e92gr/7jIcuoGaRBu8rmP4epjR2UUBltSFKG8pHBLswoFRWlBnHbvh9z6yrzA8795eR6n3/dh2nGcddYY+MObyTZuh8aABfk5O5//9tqGwGiaIA3CL1zUGFhpJ6FziIgQNcGCqaY+GtvAFiZF0eqtNQmfd+lupe3oXJ4+cUMeZUJoRvWxtImSImHmjcfw9wv2Kcg8VCgoShvEyVr60cIN/OHNZOexQ9Bbuvt8Vee4aaXOlW8oSAvZuKMuaQNX1BjWbatNaIuIZS763xfn+I4zY9lmqm3neNeOmTuLnQ1rYRzN2SbG++LGozk0RERRGB7//n48den+iAilxRE6hnRM5xoVCorSBnHW+rfmJYdffvjNer5Zt52NO+p48YvU2emNMXQqiy9Ozj6D3W98lSPueMf3mm/f/1GSTyFqEvMM/frUPWLO4zfnJifOc3Ayhh42shdXHTUi5Vy9OEKhaxihkKWm0KVDCY/kaO9B906lTHSl8wgbrZRrNCGeorQx6hqiPPzh4sDz5zxolcM8YFgPPvxmQ8qxoiaxNKKzYSxVemi/6BtjTEK46qEjqnh73rqkfkGICGfuM5A7fMI9gyixE9R1LEu/zOVzx3O2OLuxHfKUKTsJ1RQUpQUS5NwNw5PTwtUYXr2lJm2fl2auSohaamiMsixN+Us/oibR6V1SFMn47TzTgjiOplAaIvyokDKhZ2f/yCevMNt/WI/mmI5qCorSEmmIGkrTLIIn3D2FfYf0YHS/yliJycXrd3DDc7NC3SNM6KN3M1tD1HCwp3COwzn7DuLxT5bSo1MpGzz1laPGJAiF4ohk/HZenGH8vhPvX1qc/j7euZw0ph8vuHZBF0UkIddTLnnrZ4f6FgZy+xTe+dkkBnbvmJf7e1GhoCgtkDAL0OyVW5m90nqLH1bViXGDunH5E5+Fvkc28fDbaoJ3RXcsKaK0OOLrvI4aQ11jfOErLooQyfD2GWsKkfCaglcopPrb/O70PfnFM19mNJdUVJaXUFme7Pfo4DIfDc6wAFJTUPORomTBtMUbU9YXbipBoaJBOMVrMrksm52zKWsqR4QikaREdc68Es1HknEKa3f66zA4exzCPKd3n0JJinudtc+gjOaRLWUF2sSWt7uKyEMislZEZrnannTVa14sIjPs9sEistN17v58zUtRmsrSDdWccf9HXB/STJMNjRnW+XVeoosyeJsO8wbtJaVQEKE4Ir7+EK9QKMrCfFSSoWrhCJ0wGpF4umTydwzi0kOHcnITSmY683f2LzQX+TQfPQzcAzzqNBhjznKOReQOwL1P/RtjzNg8zkdRcoJTJH7OyuS0EbnCm000HZEUO3/duNNaZGM+8pbTdFMUseZR77N/wTIfuTSFSIQMX/wz2ktw9OjeseP9hvbg3neCd4lDsvmo2COA/O68S49gG//Tl+3PhMHdA8+H5a2rDqVnHlNw+JE3oWCMeU9EBvudE+v/3DOBw/N1f0XJF876kc8IQT/zkV+eIgdnUUu30LpDSXt0yrxo/ccLg0NYi0Sob4z6Co6tNfU8MXUZFWXFPHrRRCJZaAqZ4P5LhUlX7ZU36eY25+ZjUmoTmTrFgxhaFT6RXq4olE/hYGCNMcYd2jBERD4XkXdF5OCgC0XkEhGZJiLT1q0LH+esKLnCyaWTapHOemx7nXE7mo0xNEZNSn+Bsz6lW8zcGkiH0sw3R93z9oLgOUQkVq/Yy29engvAttoGxg3qBoQri/naFYckfL7hxNGh5pnpd+P9uwWt9yN7W6acjqXFlBUH//1SOcWfvix3GVnzQaGEwtnAE67Pq4BBxphxwJXA4yJS6XehMeYBY8wEY8yEqqrcbC9XlExwLAv52EzkODzdKab/9ekyhv3yZVZtDd5X8Mqs1VTXNaQ1sbg1hYYM/RbpSJVUbuvO5KilMNagbp0So3IuOmhIqLlkGj2aSj65NY3Hvr9vqPFSaRG5MCvlk2YXCiJSDJwGPOm0GWNqjTEb7OPpwDdAZnvaFaWZcDSFdHmDttbUZ1yv13ljdY/97OcrAFi0bkesrcEz7gPvLeSO179m6qKNKcd3X+cXJdQUUgkkvzuFMR9l4wyHpmsKztU/OXxX/nb+hMB+QaSKXmrpFEJTOBKYZ4yJ1egTkSoRKbKPhwLDgeS6forSAoiE9CnsddPr/OCf4fcNQFwL8Vuw3aafG56fnXT+b+8vSju+e9xMndnpSPV27LdIhwkm6liandszU3EXtNh3Li9OCGkN6+sOKzzcAqelkM+Q1CeAj4CRIrJcRC6yT32HRNMRwCHATDtE9WngMmNM6lceRSkQzr/3dJoCpE725odjgnH7FJzl5X+enBFre2HGiozGdXBrLs1pPpq3OrlucRifQrYFZ4KUoH5dyn3bgxZ7by2GsHsrwgqFI3brnb5TM5PP6KOzA9q/59P2DPBMvuaiKLklf+FHER+h4LCpuj52vCPAoZuO52fEUzfkWlPINP10DrYCBBJkPnr0ookceed7Se3uxf6ec8bx+dLNCeed0XI154+uPTxleG8h0R3NipIh6cxHyzdVh0o25zBt8cbYhq+iomChkAtue+2r2HHuNYXM+nvfpscP6spfz5vA1ceObPJcgpS4XpX+moKbE/fqF3h9WA0gXbe+XTowpBlTV2SCCgVFyRDxcQa7Oeh3b7PfbyenHefVWat59KPFnHH/RzFnciz6yG0+ytMbda4dzZnuAvb2/vkxozhydG8uOXgoAL0rs9+0FfTddArpo3DSVu+oS4yaCvuMYUqAtlQ0IZ6iZElTQlK31dRz2T+nxz5vtyuMOdlFG6NR6hujKRPQNYWJg7sXxHyUalF1tKPiogjv/nxSk+YS9N2EXdR7drY29m2wCwM5V4UV0P27dQjXsQWiQkFRMsSxV4dxNAfhXY+7dizhI1fBmzVbaxl+3SsA7D8093n0I5HUhXKyGjPEiunu4b27exf3Lj2aZloxTXT4VFVYZqYdtYlCOYwG0KNTaU5yJxUKNR8pSgq27Kzn2v98yU6XY9dZbpqiKXgFSl1DlPXb4zWMf/hYPJQ104ypYTDGejPfe5dunLvfLgz2yeOTaRqMVNFHDu4u3sdqSmEhL906Bs/9o2vTZ9c5evfefO+AwfziuFEA9LZ9ES2wQFvOUU1BUVLwp8nzeWLqUnbt1Tm2m9ZZzJqS5sJrz9+wo45bX5nn2zcfTmdjrI1sXTuWcsspe/CtP73f5DHDmI/cUT5ewZir5zxkRBW3nrZX4Pm+XfxNO9efsFtMmJQURbjppN1j5566bH+mLtqQVbrx1kbbf0JFaQJRXwFgXP9NpM4nQ6gf3gVw+abgEpe5dgiDtSDXN5rYztts34AnjYyngPCul98ak5w2Oqz5qClcduhQunRMTI9xyyl7cPbE1HUQLj54KKfbFey89O/agVPH+Z9ra6hQUJQUxDKiutaruKaQ3P/Q2/xLVXrxLoD1DcELYmOOHcJgCYXGqMnK9v3nc8bHjt1+hIgIA1wOVj+fi7vOsFfTCqspVHlSSXdyJfYb0buzrw/m3P124ben7Rlq/PaOCgVFSYGz5Lkdl86R36K3KuT+BG8RnfoUC3+u9xOA9Qz10WhWKZ7dFdDcQqWsOMK7Pz+M60/YjTu+PSYpP9PJY/tx73fjAsX75wsrFD6+9gjO33+X2Ocvbjyayw4dBli1lTOt6KYkokJBUVKQUlOwPy/dUM3SDcHmHz+84aCpIoFyZWs/fXzc/BG1Hc0lWWgK7mRv7spipcURiiISM8N4hdkPJg1LyGWUrU+hKCIx2/5PDt+V4qJIzHSVj8y1YRgzoEthbpwHVCgoSgqct85Ej4LtU7BXoENue5tDQpqNHLwL4udLNwX2zZVQKCuJ/3M3xlDfEKUoRVa6oBfuCrvI/PF79uGAYT3j43vqC3h9Id7oJO8Cvkd/32z5vngrzRV6s9i/LtmfXx5vRSq1dkVFo48UJQUx81FIn0JYvAvm8k07Q/fNFnch+JnLrUq4a1LUaAhi3MCu/O70PTl2j74J7d7kdV5h5jXrOKd/f/penDa+f0amrEw3k3l56ccHsbWmnnMe/CS7ATx0KC3i1HED+M3L8wqmreQK1RQUJRWxPEfuSmhOW/Zk4ifIlabgl3H06zWJ2UsnDklfAKYoIpy1zyC6dChJEDRlnvF/9a3RdC4rTrjOTU2DtfejvLQoc99GQKW5sH+pPfp3SdBylDgqFBQlBfHSm/E2R0A0ZUdzJgt9rtJR+JWP9E7j0BD1jN1v/IlCIXH8Eb0r+OfF8UplXvfF5morhUQ3T/hoGJzvxRnzrH0G0qeynNPG989onNu/PYbHLw5XTS0djq+lX9fWm+IC1HykKCm5/91vktpyYT7KJCY/Zz4FH03BGxbaKcO6zW4B4aeJuAWB961+s50KvEuHLISCJN5/YPeOfPzLIzIe54yAfQnZ0LVjKXefPS4vaUmaExUKipIlzaUp5CpHkZ9Q8D6Dd9NXU8dP2McQ8Xc0d+2QWToNSM6w2lI4yWfDXmtDzUdKm+OfHy9h8DX/TYqTbwruN+rYYVMczQXwKfhqCp7PleXZCwU/TcGtHHijj+77f+P52dEjGNg9c3NLPFS4lXt1WyD5LMf5kIisFZFZrrabRGSFiMywf453nbtWRBaIyFcicky+5qW0fX778lwAalKknDjxT1N45MPFocc0Bp78dCnvfr0uJz6Fd75aG7pvrnwKfou29xEqszDlOPibp+LHXp/CLj06cfnhw7PabObn61FyQz41hYeBY33a7zLGjLV/XgYQkdFYtZt3t6+5V0QyM24qSgbMWrGVG1+YHbq/AX7xzJec/9DUwOijnzzxeejx/vLewtB9c5Xlwk8oeAVbUxK+dfQpYFPn0tYyLdcZBpUJuSdvQsEY8x6wMWT3k4F/GWNqjTGLgAXAxHzNTWkf5Mu0EJQ6+4UvVib1zQk5WktLi5Lfs4p9FurrT9gtq/H98ii5EwSGLWUZBr+d5kpuKIRP4XIRmWmbl7rZbf2BZa4+y+02RckYv13ITSVx81pm5qO9b3mDxet3sH57LeNufp2354U3HeUSP/POoxda4Zju5bpnZyvhXNgF9+yJAzlrwkDfc26hEKbeQlj8clIpuaG5o4/uA27B+vd6C3AHcGEmA4jIJcAlAIMGpU6Fq7RvTJZml5r6RmrrowmROH4J8cIuRxt21DHp9ndin3/3qn/dhCDCpuN28/G1RyTViXanuXAY2aciqS1TM89vU9QuqCiPLzGSy1dQUZ9CvmhWTcEYs8YY02iMiQIPEjcRrQDcrxoD7Da/MR4wxkwwxkyoqkq/0UZpfzhLWqaO4Mc/WcrarTWc+Kf3GXPz6wnn/NNcZLci5TuL573fHU+fLuVJ7aU+/gKvyccYk+QQbgrjBnWLHedDU1ByT7MKBRFxJ0s5FXAik14AviMiZSIyBBgOTG3OuSltj0yEwsrNO/nls18y8TeTWbB2e9J54/Mp27fUXC9oVx01IuFz0Lz8Ukn4CYBc2v7zPa4qCrknb+YjEXkCmAT0FJHlwI3AJBEZi/VdLgYuBTDGzBaRp4A5QAPwI2NMo9+4ihKWoPB+v1rAafcC+OxTyHZByvXaWF6S6EAOsrP7OYL9tJZ81ZxPkZA187HU05w38iYUjDFn+zT/LUX/XwO/ztd8lHZEmo1NfhqEnx192cZ4jYTPl22OHacqshNqejledL2+Ave09h/ag48WbgDCm2/yZd7KS/RRzkZUHHRHs9LqWbetlrXbklNARxP8AIY5K7cmtbvPezn13g9jx1Pmr3f1TfxdaMqLvZqCxYfXHM7fL9gn1h629Ga+zEe59Ck4tJTvoC2hQkFp9ezz6zeZ+OvJSe3uN/nHpy7l+LunMGX+Ot83fD/z0frttb73a2mpFZI1BWt+/bp2SDAthRcKuZubm1zKBA1JzR8qFJQ2i3vxn21rCYs3VPsKhUwK2RRiGUpV7tEvJbYfYTcr50tTyKVZSl0K+UOFgtJm8QsjFfzNR5kknWvqQpTN9an2DqTyKbhJVXrTTWsoJ5mPDYqKhQoFpc3hrGmJC711LOIvADLJWtpUk0U2QiGVPd7rU9gvIJ9/WJt+vjQFpXWgQkFpczhvkVGfMFJBfH0CzakpeNfcY3bvnfaasJrC4ltP8N24BlBUlGoMS7BERHyFgt/Gt7AcMCx/RWfUfJR7VCgobZagdd5XU8ggFWmuF6IOJUV0TFPxLNVbfief7KSZji+wMzkAACAASURBVHH3d8Zx+WG7steALr6O5hd/fFCoe/jx0Pf2Yep1mVdFS0UkZj5SqZBrtPKa0mbxK4wj4i8sMnM0x/vurMt8j6Xj9C6OCA1RQ3FRJO0u51SRQx1Li3j84n3p3jl1BbNULoU+Xcr52TEjAX+H8Mg+FfzqxNEM7tkxzUyTKS8pStpg11QkHn6k5BgVCkqbxW+d/+uUhezerzKpPSOfgqvrzS+Fr8ngpcgRChF/kw1Ar4oy1m6r9c1w6tCxtIgDdu2Z9n7FIR3NQfLnwoOGhLq+OVGZkHvUfKS0SmrqG/npvz5n9ZbkTWvO+prgU7CXj2/W7eDiR6YlXZORT8F1vGj9jtDXeXFqGRQXia/P4IYTR8eK3hyxW29OG+efTb5TWdPNR26cubTkBVdd4flDNQWlVfLqrNU8P2NlSvv+Dc/N4pELJ9KprDih35ad9Ul9M/MpxAerz0DD8GIlqGukOBLxfTt3V0orKRLuPGss//ncSh4sEtdYUmkRbtyKwmtXHMKm6jr/fq1gxdUazflDNQWlWaiua+DO17/KqjaAH2Fy30xbsokHpySXvfS7JltNob4x++dxNIWiAPNRqStaqDhF5FDYTWHue4zsUxEYutoaQlKdGs0ZfG1KSFQoKM3Cn95awN1vLeCpacvSd84hjiPYvXb4CabttQ3hB3UN1hQh5ziPI+K/sJcURWJvwt6F2nlBPmJUr9D3C7vYtwqh0PKn2GpRoaA0C87iXJsjTcHBbT4Ycd0rfLV6W4K9uabeFgop3ijfmLOGn/5rRuh7LtkQ9yPkQlOIiPimoHDXPwiKPnrgvAmh7xd2IW0NQsFBrUe5R4WC0izkap35fOkm1myt8U1zUNcY5YmpSxPeusMIoddmr85oDje9OCd2nM6ncMWRwwPPOZvJIhGJmUPcJJiPPELhpDH9rGsz+LuG/Q5akUzQfQp5QIWCkneWbNjBmq1WlFBTHYOn3vshR9zxbnwJ9QznHb+2Ico7X63l82WbAsfMZq+BQzpN4YBh8VDRxbeekHDOEQSC/+JeUhSJPZ737f2OM8fw+Q1HZZRkrm2Zj7RGc75QoaDknUNve4eXv8zsbTwV22sbXI5mv0KZcWobGvne3z9l4brg0NHqugz8CR7S+RTCZIeIiH9IakkK81FJUYRunVJvVvMSdqnPZYW0fHHsHn3oUFLEOfsOKvRU2hyt4OtXlGT8zC1+1NanNx/trM9eU/AzT40f1DV2nOqt2xFoEfHvl0ooZEOmmkJL1hf6d+3A3FuOZUTvikJPpc2RN6EgIg+JyFoRmeVqu01E5onITBF5VkS62u2DRWSniMywf+7P17yUwpJrdd87nvdzOp/C3z9Y1KQ51TYkC5TzDxgcO061EMcEm0jAPgWJzS0XQiG8o7nJt1JaMfnUFB4GjvW0vQHsYYzZC/gauNZ17htjzFj757I8zktpAwQVWYkak/CGm25T2h/enN+kBdfP0ewWBKnGrii39o5uq6kP1BQcbSIXpSzD+h/yVaNZaR3kbUezMeY9ERnsaXvd9fFj4Ix83V9pmeQqWiRsOcZ0OY1KisR34S4pkibtVnZItb52t30Cm3bU+fbLtfkoLG4B9ffv7cOXK7Y0272VwlPINBcXAk+6Pg8Rkc+BrcD1xpgpfheJyCXAJQCDBqmTqb0SpCl4l/H6NFteS4oiSeGeAF07lrJum3+N5nQY4OWfHMycVVtTLuZVFWWAZeLy0xQqyotzaj4Ki/tWh43qxWEZbJBTWj8FcTSLyHVAA/CY3bQKGGSMGQdcCTwuIsmpLAFjzAPGmAnGmAlVVVXNM2ElZ7gX8bqGKHNXbc3gWvcC75+0zSskGtOYj4oDNIVuHUtCz8uLMYbR/So5Y+8BKX0KB+3ak0sPHcqN39rdt1//rh3iIakF0hSU9kezCwUR+R5wIvBdY/8rN8bUGmM22MfTgW+AEc09N6V5ufGF2Rz3xync+Pys9J1JzE80J1CYmARTTDrzUXEkkiAUetlv75Xl2QsFN+kW2GuP240+XcqTFv0pVx+GiMS0lf5dO2Q9h0y1DJUJ7ZtQQkFEfioilWLxNxH5TESOzvRmInIscDVwkjGm2tVeJSJF9vFQYDiQnMlMafW4l+j3F6wD4JGPlrCtJjlzqZdGlxpw9+T51ni+0UfxVS1d8ZwST6I5p7fjBA6Dd9F1z8m7HrsXXPfUvOvwwO5WMZv7/9/enLvfLvSu9C+xGYbXrjiE35+xV+j+qim0b8JqChcaY7YCRwPdgHOBW1NdICJPAB8BI0VkuYhcBNwDVABveEJPDwFmisgM4GngMmPMxswfR2lNuHcS73nT6yl6WjuH/S1BiYv+vz5NTLjXkGbH8dpttbw2e01S/7A1CgA6e/q6nd9egeFecN31HoI2jB27Rx9uOWWP0HPxY9denTlzwsDQ/eOlLpX2SNj/853/k48H/mGMmS1p4taMMWf7NP8toO8zwDMh56K0Ytxv0Ttqw20ae2b6cq769xe8esXBKcdzWL897iBOF0G0uTpRQ3H6B2kKFeXFbKtpSGpz12hwzylZKEBjrJ9LeLSgt/PWsKNZyR9hv/7pIvI6llB4TUQqgNymu1TaBQn1jUPsJH7hi5X89pV5AMxZmexHSPc2m2kWU6d/3y7+NvybT949STPoVOrRFFyTKvUUwHFrCu6ptaS9AWo+at+EFQoXAdcA+9i+gFLggrzNSmmz1NQ1+i7uQfzkic9jb/5+u5PTLfqZpup2fBAHBtQ8jvjsPk7lyC0tChYKbvNRWEfylKsP47kfHRiqb7aoSGjfhBIKxpgosAYYLSKHALsDXVNfpSjJ3P3WAo6/ewpbQziWvdT6aBZT5q9PeY1f6c1UOBFOXTr4Rx+VFkUSIoXcYaMO7s8lHqHwfy7/gNt8dOvpe3LnmWPSzm9g946MHaj/9JT8ETb66HfAB8D1wM/tn5/lcV5KGycou2htQyO3v/aVb+bSXBfoSYXXRORQUhSJve3/YNIwPrjm8KQ+7sXeKxRO33sA5+2/C5AYfVRRXsJp4wc0ddqK0mTCOppPAUYaY7Lb4qkoIXl6+nLueXsBDVHDNceNSjjXrEIhwNFcUhyJmY/OsiN6vDUcdunRKd7fp7ayI1SiWgxAaYGEFQoLgRJAhYKSM0SSo4eclBN+KSZqmpDiOlM6lhT5tpdEJJ5a2pNq48XLD6K8JMJwVzpnPwdyXCjkcMKKkiPCCoVqYIaITMYlGIwxP8nLrJQ2TzRqKBKhwSMVnJ3Ez3y2nB8dNizhXHNqCkFpJSxNwak34MTz23URIiQIhCDiwqRlSoUK+zu48ihNKtAeCSsUXrB/FAWARz5czI0vzGb+r49LspuHoSFq7MU1cWEsL42/oc9YtjnhnF/tglxzzznjeO7zlYHnLZ+CdezVFMIXsbF+t1TzUWlxJKl0qNJ+SCsU7PQT3zPGHNYM81FaMI99soQenco4do8+3PbaV4C11yAbodAYNdYmKc86H01hUwm72a0pnLhXP07cq1/g+eJIculMZ8Zhcww5wiPDLRSK0iyk/ddsjGkEoiLSpRnmo7Qw3Iv0dc/O4rJ/Tgfipo9sY9qjxiTt4n37q7UJO4y9L94rNu/M8m7p6dulnClXp3/vKXWZj5w3/Uz/Fo5QaamagtK+CfuKtx340k6Gd7fzk8+JKYVn/fZaRt3wKre9Ni9t32mLN6YtYu/G0hQSl9EL/v4pV/37i8Brlm2sDjzXVHp2LosloUuF23zkyEtnaQ+7K/mo0b0BOHSEpn5XWh5hhcJ/gBuA94Dprh+ljRKNGn702GfUNUb589vfJC3IzkIYNbBg7TbOuP8jbnlpTuz8N+u2M/ia/7Jg7Tbf8RujJuOUzt48RYWg2BV95Gx0c174w2aHGD+oG4tvPYE9+qvyrbQ8QjmajTGP5HsiSsti4frtfLIonqj26Lveix03Rk1sITTGsMlerN0Fc176YhUAL8zwd9o2GpNxjp2aZnA0p6O0OL6j2Rs9FPQ0j128b+AOaUVpaYQSCiKyCJ/cY8aYoTmfkdIiKPKkynQnr6tvjMbCMKPGXS85TjTN63NjNL1QEM8yG9YEP6RnJxat35G2X5/KclZvrQk837VjSZJ24ms+SjOxoDxKfrx6xcEqQJSCEtZ8NAHYx/45GLgb+Ge+JqVkzscLNzArhwXW/eoWO9Q1RmMLdNQY37j7WBnJgGGiUcgiaCkUT126f9o+N5w4mucvjyeW85vnz48ZGTseae8/KC6SZEdzbIymp5Ib1acyMEOrojQHYc1HGzxNfxCR6cCvcj8lJRu+88DHADmLL09VE/jZz1bEfQrRZEEAcQGxfJN/xFBDNJq3GgJVdklNLwcP78neu3RjcI9OnDKuP1tcWoA3xbWXf168L1Pmr6OyvITR/SqZt3pbLGW2IxxaQ8bppy7dn/ISLZigBBPWfDTe9TGCpTmEL02ltDpSmURufGF27DjqKn/pXPLXKQtZudkyyzw9fbnvGIvW72DllmDTTbb0SVG28ltj+iVUICtzLY7p9lpUVZTFEtb95tQ9OXviIAb1SIxW8pq7WiITh3Qv9BSUFk7Yhf0O13EDsAg4M/fTUVoKYe33UWN4YcYK6xqssNH/++/ctNfdbm9+S8XHC70KapwT9urL4B4d+fPb3yS0P/PDAwB44fIDOemeDxLOeZPTlRWHFwpuykuK2GdwfHHNNPpIUVoyoYvsGGMOs3+OMsZcAtSlu0hEHhKRtSIyy9XWXUTeEJH59u9udrvY+x8WiMhMj3ai5IhF63dw5+tfpXWOht1YdcCtb/HIR0usD8aEFiaNITp66y27+dnRIxnYLXlfQSc7TcZeA7py9sTEusTehd+9r8BPKIR989c9aEpbIqxQeDpkm5eHgWM9bdcAk40xw4HJ9meA44Dh9s8lwH0h59ammbtqKytzuJP3vIc+4e63FrBma+qEt9lk8DSEf1tubGKKUL90E5C4kDd46jOncgSXpfEppMK0Ip+CoqQjpflIREZhVVnrIiKnuU5VAsHGWxtjzHsiMtjTfDIwyT5+BHgH+IXd/qix/oV9LCJdRaSvMWZV+sdouxz3xylA7hzIYXcdZ5OCIZNLmpo2OuLaROZGXGu7V/CkcrD61T3IZC7QsuosK0q2pPMpjAROxCq9+S1X+zbg+1nes7droV8N9LaP+wNue8Fyuy1BKIjIJViaBIMGDcpyCko6sk3rHFaYNFVTKBLxDWl1C4oG1z3uPnsck0b0ChwvXfRRKh6+YB+emracfl3SvicpSosnpVAwxjwPPC8i+xtjPsr1zY0xRkQyWh2MMQ8ADwBMmDBBrbk5ZMP2Wrp0KKG4KJKl+ciEXuwzrZ3sJRLxNwe5LUruuezerzJlmO2VR41Magv74r9rrwp+efxu4TorSgsn7OvRBhGZ7DiMRWQvEbk+y3uuEZG+9jh9gbV2+wrA7RkcYLcpOSToRX5bTT17/9+bscihbMxH0WjTNYAgdu9XmfDZ0hRS+xSC9iu4ufe743ns4n3pk+ItX61CSnsirFB4ELgWqAcwxswEvpPlPV8AzrePzweed7WfZ0ch7Qdsae/+hObESefwxpw1gLXAZ0OYqKJsuOPMMQmfiyLiu/nN3eSu8Ry0rh+/Z9+0aSjOmjAw5XlFaUuEFQodjTFTPW0N6S4SkSeAj4CRIrJcRC4CbgWOEpH5wJH2Z4CXsWpBL8ASQj8MOTclA2IpKez9x1t21rO9toE6u+KLY1vPytEM3PPWgpzM04tTptMhEhB95DYplZcUcekhVnqubh1Ls763hpwq7Ymwm9fWi8gw7EwGInIGHgewH8aYswNOHeHT1wA/CjkfJQvqG6OxUFTHynPcH95j5ZYaXvnpwQCU2t7bbBZCYwwvzWyacvf8jw7k5D9/kNReXlKU8Lk4hKYAcPWxo/j+IUPp1il7oaAo7YmwQuFHWM7dUSKyAmtH83fzNiulSXyzbjsDunWgrDhxIb3zja9jx07OIifVxPZaS/FriqaQC4LMT95w0ohIQnSRu91NUUTo2Tm9byEV6lNQ2hOhzEfGmIXGmCOBKmAUcChwUD4npmTHph11HHHHu1z37KyE9n9NXcp978RTQngX/VwIhVzIEe+GM4dST/xpUURYty05d1KGdXsURfGQUiiISKWIXCsi94jIUUA1lnN4AZr7qEXiLO4ffZOYN+ia/3yZ8Nn7kr3Dvq5IhHvfWcDOuswL2pjkkhsZ0xDg4S72CgWRpDbQDWSK0lTSmY/+AWzCchZ/H7gOK5DjVGPMjDzPrd0TzVN4J/hoCjWWUJi6eCNTF2/koAwKwzh8vWZ7k+fVo1M4U08kInx77wFc6xF2+UAdzUp7Ip1QGGqM2RNARP6K5VweZIzJfc5jJQk/m3mu8O5YdqKPHKrr0gaX5ZxDR1Qxsk9F6P7FRRGOHt2b1+0w2lyjOofSHknnU4htOzXGNALLVSA0H9lsBAsqauMlauDhDxbFPn++dHPC+eYyw3TtGA813TOgkP1tZ+wVeL3zF9q1V+dcTktR2i3pNIUxIuJUYxegg/1ZsKJIK4MvVZpKkH09Ffe+E26fQGPUcNOLc2Kfn/08cfN4czls3bcJumeYMpc/O3okx+7RJzeTUpR2TLrcR0Wpziv5o6a+kYN//3bG14V9w08XXdRcmoKIMG5QV0tTCbhnJIU+2xwFbnLhQFeU1oIWa22hLNlQHUs9kQnetfHMv3zE4be/k9QvnfO0uTQFYwwHD68C4nP/v1P2SOjj5DOae7O3NEd+0UAmpT2iQqGFUt+YXfIh70I2ddFGFq7fkdQvrabQTG5WAzEJ5ZiJzth7QEKfajs8tkOpn+JqF7jJw9wGde8EwOi+aiVV2g8qFArE0g3VfJKiBnG22UbD11ZOfT7IZPP704OdvtlgTHwujkDzCramptnOlv2H9eDlnxzM+QcMLsj9FaUQqFAoEIfc9jZnPfBx4PmmhqOmK5KTbnNakHO3pDj8O/nj3983bZ+oMTGbvTNysUcipRIKA7tbdZrzldtodL9K3RCntCtUKLQQ6hqiRKOGKfPX8cWyzXmrS+Bw++tfpTzvNi+5tYMOJWHTZSVmJt2lR0cOGNYj4F7WbyfraVFEOHK33rHzzm5rgKNG90649prjRvHgeRPYZ3D30PNSFCWY8P/Clbwy4vpXOGVsP+at3sYuPTpy/v6D017z+dJNfLJoI5cdOizWZjy/g5i+ZFPK8/UN8RHKXMnoOvra9f0pc5W4PGfiIC45ZCi/eXkuD06J74/AxAWQ+4Xc0XR6di7jp0cOj7U/eN4Ezz2KkgSFoijZo0KhBfHcjJUMrepEbUM0aYdxQr/PV3DFk/EsIwlCIUc5Gdz3d2dbzUQolLhyE0WNFX563QmjE4SCif3Hv2raYxfv2+Qsp4qihEfNRy2MxqihvjFKXUOwUHjy02Vpx2mqbHBHP7k1BW9dg1R0Kou/c7jNUd9zOW6Nie8CcIfB3nDiaB65cGJGaS8URWk6KhRaAO63+4ZGQ32DSakpFBclOj5zpR24cQsFd9rqEp/MpF6cN/uOpUX8/JiRAGx1OYtvOmn32HHUxBP/uc1HncqKOXREVXaTVxQla1Qo5JDN1XXU1GeecnrItS/HjhuiUWobo9TWJwqFWSu2xI69Bev9nNKrtzYtRZVbU3G/8bsF0rxb/DeTXX3MSObdcizlJUWx6KClG6t9+47uVxnTFJprb4SiKME0u1AQkZEiMsP1s1VErhCRm0Rkhav9+OaeW1MZe/MbnHTP+00aozFqqG+IstkThnnin+LjFnuEgjt81a00bK6uy3oeizdYi/ibVx6aUPWsd2V57DiVKck5t98QKyrouD37+vZ76Px92KO/tTlshJqKFKXgNLuj2RjzFTAWQESKgBXAs8AFwF3GmNube065JNOaAl7TT4PtU1i5OTjbqXcPQZCpqTaFXyKIPfpXMmvF1thnkbjJqG+XcjqXpf9fxl1Ss1dlOYtvPSGpz8Qh3Zm6aCNdOpZwytj+7DWgK8OqNNOpohSaQpuPjgC+McYsKfA88sJFD3/K458sTdnHa/lpbLT8Cau2BAuFtdtqEz7XByz+2bgaXvrxwQmfIyIxn4LjLD55bD/GDuwaOEaYUp5PfH8/5v/6OMCKSlKBoCgtg0KHpH4HeML1+XIROQ+YBlxljEkKpheRS4BLAAYNGtQsk8yWyfPWMnneWs7ZN3ietQ2JPoiGqGHJhmqWbPC3wT8/YwUzliXWPqh31TV2Z/TMJvW2l4jE/QiOAPvjd8bFzh+7ex9enb064Zow++6KIkKR+hAUpcVRME1BREqBk4B/2033AcOwTEurgDv8rjPGPGCMmWCMmVBV1fqjU0b/6rWEz+kW8udnrExqm7c6bu6ZszJ+3NCYXVTSkbv1ih1HRGKObT8F4P5z905qG9itQ1b3VRSl8BTSfHQc8JkxZg2AMWaNMabRGBMFHgQmFnBuBSNdzqO35q1Navve3z+NHW9ypdvOVlP4waT4ZjiRuA8jTOjriN6dmTSyV9p+iqK0TAopFM7GZToSEXd4yqnArGafURPI1V6BVMM88uHijMaqa8huTu6EdBERKsotK6NbWASRytegKErLpyA+BRHpBBwFXOpq/r2IjMVKerDYc67FU5/CVFNT38gNz83i58eOpFdFObNXbgnsm4obX5idUf+bXsysv4N7H0RlhxLKiot8I4j80L0GitK6KYhQMMbsAHp42s4txFxyRSpTzauzVvPv6cuprm/kvzNXNducpi7amNV17l3LYUJQ3WiWaUVp3RQ6JLXNUB/CVLO1QMViMsWbRiMTVCgoSutGhUKOqPdoCve/+01Sn6DCNc3JXWeN4Xen75myj3fHdCZoQRpFad2oUMgR3vDPW1+ZFzt28v6E2dSVT9688hBOHTcg7aa24hBJ74I4ZHjPrK9VFKXwqFDIkmjU8KfJ82P5hepTZDW9842vgez3DeSKXXtZuYXSzaIkS03hlZ8ezLF7+Oc4UhSldaBCIUs++GY9d7zxNdc9a0XOhqmp/NHCDfmeVii8Gsvxe/ZJ+OzNwpoOx2LkhK4qitJ6UaGQJY4Q2G7XD25IoSm0NLzyq09l4g7kTM1HJfa+hpbgM1EUpWnoq12WOMuf89adqihOS8O70c5guOusMbF8S5k6mouLhLpGjTxSlLaAagpZ4o2yaaq/IMxu4XSErZ8c9agKDY2GU8cN4IojRwCZh6Q6QkQ3rilK60eFQpY4y5/z0t3UjKQjejctdfTNJ+8e2nzjyISyYuvr99aDdsxBvSrKQo3nbHZrQiSroigtBBUKWRJLEmfH8qRKcxGGLh1KfNt37ZVaWAzt2QmAHp3KYv6NdDgmL6c6mjdyKhIR7jprDM/84IBQ4zmO6TApsxVFadmoUMgS56XcURDc5qOF6zKrvgZQWe4vFFLZ9x++YB9G9XXCTMOvyHGhYH39tT7+kFPHDYjVV06Hoynkon6DoiiFRYVCljhCwWDYsL2WFZvjRXEOv+PdjMcL0hRWb60JvCbbFNXH79mXirJiztt/MJBsPsoUR1Mo9D4MRVGajgqFLHGcqlEDe//fm/zimS+bNF6QUOjf1QoXferS/bn+hN1i7b86cXTCPDLZLD2gW0e+/N9jGN2vEmi6ULjqaMtB3asynA9CUZSWi4akZolj1cnV/oQOAZFDD18wkRWbdzJ2YFcmDunOvNXbeHr6ci48aEjK8bxlMn9mL9xuymyzj7ckaKacPLY/J4/t36QxFEVpGaimkCVOSOpnSzen6ZmeK48aEZiiuqqiLKFwze3fHhOqtsHZnrrQB+yanJOoNCD6SFGU9osKhSzJ5UatnxwxPOvsoucfMBiAiUO6c+93x8fay4sjnDYu/vZe5DN+x9LihN+Koii6GmRJgROexpg4pHtMczh+z77sO6Q7nyzaSNQk5jjy28OwW98Krjt+N04e16/Z5qsoSsumYJqCiCwWkS9FZIaITLPbuovIGyIy3/7drVDzS0eh02AHEds/YQyXHhrfJe2niIgI3z9kKL0qyptreoqitHAKbT46zBgz1hgzwf58DTDZGDMcmGx/bpG0VKHgpMsY3a+S3fpWxhzMvSt14VcUJT0tzXx0MjDJPn4EeAf4RaEmk4rmkAmOIzgTDhlRleCI/uGkXTnvgMGBm+MURVHcFFJTMMDrIjJdRC6x23obY5zK9quB3t6LROQSEZkmItPWrVuX90ku2bCDV75cldTeHJrCi5cf1OQxIhFRgaAoSmgKqSkcZIxZISK9gDdEZJ77pDHGiEjSymuMeQB4AGDChAl5X5mPuvM96hqjvP+Lw6iqKKOs2NpP0Bx5fkb2qcj/TRRFUVwUTFMwxqywf68FngUmAmtEpC+A/Xttoebn4NRJOOh3b3PlU18AMH3JJjbuqC3ktBRFUfJCQTQFEekERIwx2+zjo4GbgReA84Fb7d/PF2J+Qbw115JRp9/3Yd7vNcTOfqooitKcFMp81Bt41t6wVQw8box5VUQ+BZ4SkYuAJcCZBZofAK/O8vEl5NFu9OPDd2Xh+h1ce9woKgNyISmKouSTgggFY8xCYIxP+wbgiOafUSJrt9UQjcJl//ws6VyqsptVFWWs25a9Wemqo0dmfa2iKEouKPQ+hRbH9CWbmPjryTz56bKkczvrG9lZF5w8bkC3DvmcmqIoSt5pafsUCs681VsB+HTxRt/zc1ZtDby2T8AGsQN37cFjF+8HwB/e/Jo/vDkfgBu/NZoT9urblOkqiqLkFBUKAQRVMkuVUbRzWTF3njmGiAhLNlRz15tfA/HqbAA/PWI4b8xZw+yVW+lVUa4pJhRFaVGo+SgAp3iNF2/tgd+fsRffP9iqbdCprJjTxg/glHH96VAa/9Ou3RavniYilNk7lbPZsawoipJP2v2qtG5bLS/77FgO0hRqPZrC+EFd6dqxFICOrkI5XSmnOAAACuJJREFUFa5dxM6GN4f12+uA4GpriqIohaLdm4++9/epzF65lS9vOpqK8pKYhhBUb9grFESENXYdZXcY6ZkTBlJT30insmIOGNYj4ZorjhzOr/87l3GDuqIoitKSaPdCYdnGaiDR7g/QGLAf4eqnZyZ8jojQz66jfNjIXrH2oohwwYH+JTNPGz+A08YPyHbKiqIoeaPdCwVn6Y8aw9aaen757JcANIZMeBcRuPigIZywZ18Gdu+Yp1kqiqI0D+1eKDg0GsOSddWxz2F3LkdEKC6KqEBQFKVN0O4dzY6q0NBomLMyvgehIaRQyGWtZkVRlEKjQsFme23cdAQwe2XwJjU3frWPFUVRWisqFGyqU6SvSIUKBUVR2hIqFGxq6oN3KqciojJBUZQ2RLsXCo7noLquIbsBVCgoitKGaPdCwSFV9tNUqPlIUZS2hAoFm531KhQURVFUKNhk72jO8UQURVEKiAoFm0zMR3edFS8aJ6opKIrShmh2oSAiA0XkbRGZIyKzReSndvtNIrJCRGbYP8fncx419Y2cdu8HbK+1HMyZaAoVZfHEd6opKIrSlihEmosG4CpjzGciUgFMF5E37HN3GWNub45JfPv+j/hyxZbY5+r68NFHFeXxP5vWRFAUpS3R7ELBGLMKWGUfbxORuUD/5p6HWyAA1GSgKfToXBY79tZKUBRFac0U9DVXRAYD44BP7KbLRWSmiDwkIt0CrrlERKaJyLR169blbC5B5qMxA62aB2fsHU91XVFezLG79+Hbe2v6a0VR2hYFEwoi0hl4BrjCGLMVuA8YBozF0iTu8LvOGPOAMWaCMWZCVVVVzuZTHRCSOqxnJwA6uaqqlRcXcf+5e3Pbt8f4XqMoitJaKYhQEJESLIHwmDHmPwDGmDXGmEZjTBR4EJjYnHPaUl0fO3aXyRzQzSqgU1YSFwplJepHUBSlbVKI6CMB/gbMNcbc6Wrv6+p2KjCrOefl9jGM6N0ZgPv/33hOGmu5O04Z259fnTiaqooyytS5rChKG6UQ0UcHAucCX4rIDLvtl8DZIjIWKx3RYuDS5pzUlp1xTaGqwnIk1zUadu3VmcW3ngDA6H6VXHiQf4lNRVGUtkAhoo/exz+N3MvNNYfN1XWB56ZffyS3vDQHgIbG7DKnKoqitFbapR1k6cbqwHM9OpdRUmT9WeoaVCgoitK+aJdCYa8BXTlyt96B50tsn0G9agqKorQz2qVQAMuJfM8545hy9WFJ5y46aAj9upRzzO59CjAzRVGUwlEIR3OLoLgowol79Utom3nT0QAMq+rMh9ceUYhpKYqiFJR2qyn4UVHWbmWkoigK0I41BTcvXn4Q05ds1DTYiqK0e1QoAHsO6MKeA7oUehqKoigFR81HiqIoSgwVCoqiKEoMFQqKoihKDBUKiqIoSgwVCoqiKEoMFQqKoihKDBUKiqIoSgwVCoqiKEoMMcYUeg5ZIyLrgCVNGKInsD5H02kNtLfnBX3m9oI+c2bsYozxLXLfqoVCUxGRacaYCYWeR3PR3p4X9JnbC/rMuUPNR4qiKEoMFQqKoihKjPYuFB4o9ASamfb2vKDP3F7QZ84R7dqnoCiKoiTS3jUFRVEUxYUKBUVRFCVGuxQKInKsiHwlIgtE5JpCzydXiMhAEXlbROaIyGwR+and3l1E3hCR+fbvbna7iMjd9t9hpoiML+wTZIeIFInI5yLykv15iIh8Yj/XkyJSareX2Z8X2OcHF3LeTUFEuorI0yIyT0Tmisj+7eB7/h/7/+tZIvKEiJS3te9aRB4SkbUiMsvVlvH3KiLn2/3ni8j5mcyh3QkFESkC/gwcB4wGzhaR0YWdVc5oAK4yxowG9gN+ZD/bNcBkY8xwYLL9Gay/wXD75xLgvuafck74KTDX9fl3wF3GmF2BTcBFdvtFwCa7/S67X2vlj8CrxphRwBis52+z37OI9Ad+AkwwxuwBFAHfoe191w8Dx3raMvpeRaQ7cCOwLzARuNERJKEwxrSrH2B/4DXX52uBaws9rzw96/PAUcBXQF+7rS/wlX38F+BsV/9Yv9byAwyw/6EcDrwECNYuz2Lv9w28BuxvHxfb/aTQz5DFM3cBFnnn3sa/5/7AMqC7/d29BBzTFr9rYDAwK9vvFTgb+IurPaFfup92pykQ/5/LYbnd1qaw1eVxwCdAb2PMKvvUaqC3fdwW/hZ/AK4GovbnHsBmY0yD/dn9TLHntc9vsfu3NoYA64C/22azv4pIJ9rw92yMWQHcDiwFVmF9d9Np+981ZP69Nun7bo9Coc0jIp2BZ4ArjDFb3eeM9erQJuKQReREYK0xZnqh59LMFAPjgfuMMeOAHcRNCkDb+p4BbPPHyVgCsR/QiWQzS5unOb7X9igUVgADXZ8H2G1tAhEpwRIIjxlj/mM3rxGRvvb5vsBau721/y0OBE4SkcXAv7BMSH8EuopIsd3H/Uyx57XPdwE2NOeEc8RyYLkx5hP789NYQqKtfs8ARwKLjDHrjDH1wH+wvv+2/l1D5t9rk77v9igUPgWG21ELpVjOqhcKPKecICIC/A2Ya4y503XqBcCJQDgfy9fgtJ9nRzHsB2xxqaktHmPMtcaYAcaYwVjf41vGmO8CbwNn2N28z+v8Hc6w+7e6t2ljzGpgmYiMtJuOAObQRr9nm6XAfiLS0f7/3HnmNv1d22T6vb4GHC0i3WwN62i7LRyFdqoUyJFzPPA18A1wXaHnk8PnOghLtZwJzLB/jseypU4G5gNvAt3t/oIVifUN8CVWZEfBnyPLZ58EvGQfDwWmAguAfwNldnu5/XmBfX5ooefdhOcdC0yzv+vngG5t/XsG/heYB8wC/gGUtbXvGngCy2dSj6URXpTN9wpcaD/7AuCCTOagaS4URVGUGO3RfKQoiqIEoEJBURRFiaFCQVEURYmhQkFRFEWJoUJBURRFiaFCQWnXiMh2+/dgETknx2P/0vP5w1yOryj5QIWColgMBjISCq6dtEEkCAVjzAEZzklRmh0VCopicStwsIjMsPP2F4nIbSLyqZ2r/lIAEZkkIlNE5AWsHbWIyHMiMt3O9X+J3XYr0MEe7zG7zdFKxB57loh8KSJnucZ+R+J1Eh6zd+8iIreKVSdjpojc3ux/HaXdkO5NR1HaC9cAPzPGnAhgL+5bjDH7iEgZ8IGIvG73HQ/sYYxZZH++0BizUUQ6AJ+KyDPGmGtE5HJjzFife52GtSN5DNDTvuY9+9w4YHdgJfABcKCIzAVOBUYZY4yIdM350yuKjWoKiuLP0Vh5ZWZgpR/vgVXMBGCqSyAA/EREvgA+xkpENpzUHAQ8YYxpNMasAd4F9nGNvdwYE8VKUzIYK+1zDfA3ETkNqG7y0ylKACoUFMUfAX5sjBlr/wwxxjiawo5YJ5FJWBk89zfGjAE+x8q7ky21ruNGrAIyDVgVtJ4GTgRebcL4ipISFQqKYrENqHB9fg34gZ2KHBEZYRey8dIFq+xjtYiMwiqD6lDvXO9hCnCW7beoAg7BStrmi10fo4sx5mXgf7DMToqSF9SnoCgWM4FG2wz0MFZdhsHAZ7azdx1wis91rwKX2Xb/r7BMSA4PADNF5DNjpfR2eBardOQXWFltrzbGrLaFih8VwPMiUo6lwVyZ3SMqSno0S6qiKIoSQ81HiqIoSgwVCoqiKEoMFQqKoihKDBUKiqIoSgwVCoqiKEoMFQqKoihKDBUKiqIoSoz/D6dCttC39PTjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "iteration_list = list(range(len(test_returns)))\n",
    "plt.plot(iteration_list, test_returns)\n",
    "plt.xlabel('Iterations')\n",
    "plt.ylabel('Returns')\n",
    "plt.title('BC on {}'.format(env_name))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "executionInfo": {
     "elapsed": 19,
     "status": "ok",
     "timestamp": 1649956690808,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "QsIOztkpCqjs"
   },
   "outputs": [],
   "source": [
    "class Discriminator(nn.Module):\n",
    "    def __init__(self, state_dim, hidden_dim, action_dim):\n",
    "        super(Discriminator, self).__init__()\n",
    "        self.fc1 = torch.nn.Linear(state_dim + action_dim, hidden_dim)\n",
    "        self.fc2 = torch.nn.Linear(hidden_dim, 1)\n",
    "\n",
    "    def forward(self, x, a):\n",
    "        cat = torch.cat([x, a], dim=1)\n",
    "        x = F.relu(self.fc1(cat))\n",
    "        return torch.sigmoid(self.fc2(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 35526,
     "status": "ok",
     "timestamp": 1649956901165,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "GTBwvNo1Cqjs",
    "outputId": "d079ec71-95ec-43aa-b520-26bc9eb44024"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "进度条: 100%|██████████| 500/500 [00:35<00:00, 14.20it/s, return=200.000]\n"
     ]
    }
   ],
   "source": [
    "class GAIL:\n",
    "    def __init__(self, agent, state_dim, action_dim, hidden_dim, lr_d):\n",
    "        self.discriminator = Discriminator(state_dim, hidden_dim,\n",
    "                                           action_dim).to(device)\n",
    "        self.discriminator_optimizer = torch.optim.Adam(\n",
    "            self.discriminator.parameters(), lr=lr_d)\n",
    "        self.agent = agent\n",
    "\n",
    "    def learn(self, expert_s, expert_a, agent_s, agent_a, next_s, dones):\n",
    "        expert_states = torch.tensor(expert_s, dtype=torch.float).to(device)\n",
    "        expert_actions = torch.tensor(expert_a).to(device)\n",
    "        agent_states = torch.tensor(agent_s, dtype=torch.float).to(device)\n",
    "        agent_actions = torch.tensor(agent_a).to(device)\n",
    "        expert_actions = F.one_hot(expert_actions, num_classes=2).float()\n",
    "        agent_actions = F.one_hot(agent_actions, num_classes=2).float()\n",
    "\n",
    "        expert_prob = self.discriminator(expert_states, expert_actions)\n",
    "        agent_prob = self.discriminator(agent_states, agent_actions)\n",
    "        discriminator_loss = nn.BCELoss()(\n",
    "            agent_prob, torch.ones_like(agent_prob)) + nn.BCELoss()(\n",
    "                expert_prob, torch.zeros_like(expert_prob))\n",
    "        self.discriminator_optimizer.zero_grad()\n",
    "        discriminator_loss.backward()\n",
    "        self.discriminator_optimizer.step()\n",
    "\n",
    "        rewards = -torch.log(agent_prob).detach().cpu().numpy()\n",
    "        transition_dict = {\n",
    "            'states': agent_s,\n",
    "            'actions': agent_a,\n",
    "            'rewards': rewards,\n",
    "            'next_states': next_s,\n",
    "            'dones': dones\n",
    "        }\n",
    "        self.agent.update(transition_dict)\n",
    "\n",
    "\n",
    "env.seed(0)\n",
    "torch.manual_seed(0)\n",
    "lr_d = 1e-3\n",
    "agent = PPO(state_dim, hidden_dim, action_dim, actor_lr, critic_lr, lmbda,\n",
    "            epochs, eps, gamma, device)\n",
    "gail = GAIL(agent, state_dim, action_dim, hidden_dim, lr_d)\n",
    "n_episode = 500\n",
    "return_list = []\n",
    "\n",
    "with tqdm(total=n_episode, desc=\"进度条\") as pbar:\n",
    "    for i in range(n_episode):\n",
    "        episode_return = 0\n",
    "        state = env.reset()\n",
    "        done = False\n",
    "        state_list = []\n",
    "        action_list = []\n",
    "        next_state_list = []\n",
    "        done_list = []\n",
    "        while not done:\n",
    "            action = agent.take_action(state)\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            state_list.append(state)\n",
    "            action_list.append(action)\n",
    "            next_state_list.append(next_state)\n",
    "            done_list.append(done)\n",
    "            state = next_state\n",
    "            episode_return += reward\n",
    "        return_list.append(episode_return)\n",
    "        gail.learn(expert_s, expert_a, state_list, action_list,\n",
    "                   next_state_list, done_list)\n",
    "        if (i + 1) % 10 == 0:\n",
    "            pbar.set_postfix({'return': '%.3f' % np.mean(return_list[-10:])})\n",
    "        pbar.update(1)\n",
    "\n",
    "# 进度条: 100%|██████████| 500/500 [04:08<00:00,  2.01it/s, return=200.000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 295
    },
    "executionInfo": {
     "elapsed": 7,
     "status": "ok",
     "timestamp": 1649956902771,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "hDZQKhf-Cqjt",
    "outputId": "7a42b3ea-89cc-4872-df47-98910bd8263f"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2debgkVXm436+7772z78MwDAzDsCPCgCOCyC4KKC4JUYmJqEQk0WjURCWaqElMyOISk/xUXDER3DDBGDQs7hGFQUdWkW2QQZgZBobZ79L9/f6oOl2nqqu6q/fb937v89ynq0/Xcqpu9/nOtx5RVQzDMAwDoNDvDhiGYRiTBxMKhmEYRhUTCoZhGEYVEwqGYRhGFRMKhmEYRhUTCoZhGEYVEwqGMQURERWRQ/rdD2PwMKFg9BwReZWI/FREdonI5nD7j0REEvu9PxzcnpNof62I/Mh7v0FEnt+r/nvXPUFErhORbSLypIjcIiKva+N83xORP0i0aficdorIoyLyYREptt/7lvu4RkRuE5Hd4euafvXF6A4mFIyeIiLvAP4Z+EdgX2AZcClwMjDs7SfAa4Anw9dJhYicBHwH+D5wCLAY+EPg3BbOJSJS77d4rKrOAc4Cfhd4Q/M9bh8RGQauBf4DWAhcCVwbthtTBBMKRs8QkfnAXwF/pKpfU9UdGvBzVX21qo56u58CLAfeAryqEwOPiBRE5L0i8nCooXwh7BMisiqclV8kIr8WkSdE5D11TvePwJWq+veq+kR4H7ep6ivC8y0UkW+KyBYReSrc3t/ry/dE5IMi8n/AbuDfw3v+11Ar+NfkBVX1l8APgaPDc7xBRO4PtZRviMh+Gfc9IiL/FN7XJhH5hIjMzNj3HhF5sfe+FN7D8cDpQAn4qKqOqurHAAHOrPOcjAHDhILRS04CRghmm424CPhv4Cvh+/M7cP3Xhn9nAKuBOUBy8H0ecDjBrPwvReTI5ElEZBbBvXytzrUKwOeAA4GVwJ6Ua/0+cAkwN+zXD4E3q+ocVX1zynWPIhAcPxeRM4G/A15BIDwfBr6U0ZfLgcOANQRazQrgLzP2vRq40Hv/QuAJVf0Z8Azgdo3Xxrk9bDemCCYUjF6yhGCAmXANIvLj0Ca/R0RODdtmAb8DXKWq4wSDbydMSK8GPqyqD6rqTuAyAi2k5O3zAVXdo6q/AH4BHJtynoUEv53Hsi6kqltV9RpV3a2qO4APAqcldvu8qt6lqhPhfWbxMxF5ikBIfppA2Lwa+Kyq/izUsC4DThKRVf6BoRnuEuBtqvpk2Je/BV6Vca2rgJeE/wMIzFVXh9tzgKcT+z9NINSMKUKp8S6G0TG2AktEpOQEg6o+F0BENhJNUl4OTADXhe+/CNwoIktVdUsb19+PYEbteJjgN7DMa3vc295NMBAmeQqoEMzQf5l2oXBQ/QhwDoEQAZgrIkVVLYfvH8nZ7+NV9f7E+fcDfubeq+pOEdlKoAVs8HZdCswCbvP8+AIUw/N8i0D7AHijqn5RRO4BzheR/wZeAhwXfr4TmJfo2zxgR877MAYA0xSMXnIzMAq8tMF+FxEMxr8WkceBrwJDBLPWdvgNgTnHsZJA+Gxq5iSqupvgXn67zm7vIDBDPUdV5wGnhu1+hFWyRHEzJYtj9yIiswmc3Y8m9nuCwHT1DFVdEP7NDx3XqOq5oblqjqp+MTzGmZBeCtztCaS7gGMSUWLHhO3GFMGEgtEzVHUb8AHg/4nIBSIyN3T+rgFmA4jICgJ7/osJbOBrCEw4f099E9KQiMzw/tK04KuBt4nIQSIyh8CM8mXfnNUE7wReKyJ/JiKLw74fKyLOrj+XYDDeJiKLgPflOOcmAl9HHq4GXheGiI4Q3MtPVXWDv5OqVoBPAR8RkX3Cfq4QkRfWOfeXgBcQRFNd5bV/DygDbwmd187v8Z2cfTYGABMKRk9R1X8A3k4wqG4K/z4JvAv4MYHzdb2qXq+qj7s/4GMEs9SjM059HcEg7P7en7LPZwmifH4APATsBf64xfv4MUHUzZnAgyLyJHAFkcnro8BMgpn6T4Bv5zjtPwMXhNFKH2tw/RuBvwCuIfBtHEy2n+BdwP3AT0RkO3AjgRaTde7HCDSh5wJf9trHgJcRCOdtwOuBl4XtxhRBbJEdwzAMw2GagmEYhlHFhIJhGIZRxYSCYRiGUcWEgmEYhlFloJPXlixZoqtWrep3NwzDMAaK22677QlVXZr22UALhVWrVrFu3bp+d8MwDGOgEJGHsz4z85FhGIZRxYSCYRiGUcWEgmEYhlHFhIJhGIZRxYSCYRiGUaVrQkFEDhCR74rI3SJyl4i8NWxfJCI3iMh94evCsF1E5GPh8oK3h8v/GYZhGD2km5rCBPAOVT0KOBF4U7ic4LuBm1T1UOCm8D0EC54fGv5dAny8i30zDMMwUuhankJYfvexcHtHuJrTCoKFO04Pd7uSoEb7u8L2L4Trv/5ERBaIyPLwPJOSR57czddu28ickRKvO3kVZVU+/38b2DUalOc/+6h9eeb+8/nFI9soiPDM/eez/pFt/PBXW1i+YCa/3rqreq6Zw8E5ZgwVY9fY+NRuvrpuI66a7QGLZnHcyoU8sGUnYxMV9o6XmTujxDH7LwDg3sd3sGj2MD/41RZ+9zkreXz7Xm68ezMHLJrJhid2USgIFzxrf/ZfOIu942W+eftjPO+QJdzx6NOcfdQytu4c5ZaHnuSwfeeyafteRkpFHn96L/du2gGJiroHLZ3Nr7fuoVypUCwUqtdwrF46h4ee2EWyEm+xUOCQfeZw6LI5HLYsWsnxu/du5rBlcykI3P2b7Zx15DK+eftveO7BS7jx7k3MnVFi7apFLJ07wjd+8Rvu37SDkaEiv3figcyfOcRvtu3hK+seYeZQkZcft4JbNzzFi45Znvq/2/hU8L+rVJRVS2bzyJN7OHif2SyaPcxIqcCMoSIPbtnFfZt6s6hY1rPKy/nH7sf8WUNc/dNHKFcqHe6dMRk5bN+5vPiY/Tp+3p4kr4Xrxh4H/BRY5g30jxMthbiC+PKEG8O2mFAQkUsINAlWrlzZtT7n4SvrHuFfvhMsSnXSwYvZNTrB330rWp3xnsd38KnXrOVvr7uHYkG46g0n8sH/uZtbNzxV3UckGmuP2m8epx22NHGNjXzspvti+6WxYNYQRRG27hrjhFWLuGXDkyyaM8y373ycH973RGxfVXjb2Ydx+bd+yed/vKHa/uDfnsfFV65j/SPbUq/hr7dVry/JvtY77muXnsTaVYsAeN3nbmXujBKzh0s8vn0vP3rXGbz5qp/H9j9y+Ty+9dZT+NOv/oKxiWDwO2DRLF5y7H58/Wcb+eiN9wHwT9ffy3hZWbvqLJbNm1HTx6+u28g/33Rf9k0AI6UCoxOVWP+7Qdazaub4TdtHOXzfuXzkxl+1fB5jsHjxMfsNplAIV7i6BvgTVd3ur+SnqioiTU2NVPUKgsVMWLt2bV8Xg5ioRJcfK1coh++/dMmJ/N119zBerlQ/c2t7jZejY1538ired/4zuPPRp3nxv/yI0fEySSbKFUoF4f6/PY9P//BB/uZ/7knty7bd0brve8LzjE9UeHjr7mr7S47dj+vueIyJcCa5afve2DnGyhU2PrUn9fyLZg/zs784u/r+jf++jv+9K1jF8ld/cy6HvfdbALzomOX82+8ezys/eTM/fehJ5s4occf7o0W+9o6XOeIvovVmLvjEzWy4/EXV9zv2TrBjb/CwRidqZ7wbn9xdfS7nH7sf//2L31Rnxv6zdduj4+mz5nJFKRaEs49cxrfvejx1n/FyhTefcQh/+sLM9Wg6wms/dwvfu3cLc0ZK3PmBeguipXPy5d9hoqLV/+udH3ghc0YGuliB0Ue6Gn0kIkMEAuGLqvr1sHmTiCwPP18ObA7bHwUO8A7fn9r1ZicVFW+KV6lobIHdYkGqQqJS0eqsdrgYPfLZw8EPt1QMBOWusQlufmBr7BoKFEJBWiw0N/0bK1f4zbZokB8qFiiIUM6wLoxlfUB8YeFkX4aK0XYpbHf3VEr0eaiY/ytXqDPdDZ5LsO2sJZUU9aWcodIoilB/Rl3R6BrdxD0T/zk2Q6CZaVXjMCXBaIduRh8J8BngHlX9sPfRNwgWZid8vdZrf00YhXQi8PRk9icAsWXWy5X4j7JUKDARzlbLqoxOBLN3f+CaORz4D9yg8K5r7uDCT/2Ehzy7fEW1+ivPKxScMNqwdXdMmxkuCYUCmXbr8ZSZuSM5ePqCSkSqfXOvWYIs2D/XbXDrQ0+mdCR4UY3O7Z5pmlBIa3PHizQ2sxR6IBWGS4XYa7MURFCir2M9YWoYjeimjnkywXq7d4jI+rDtz4HLga+IyMXAw8Arws+uA84jWEt2N/C6LvatI/jDTbmiOF3BDZJucJ4oa9UU4g/Ss51QKASDgdMm9vpmJG+2mlcouIHQd/qCrykEnyfHy7Fytv1cpHZwByh6g3+5opGmUEgXChDcbz2txPHOa25PbXdCzV3b3UclZfyvpDUS/O8EqbmvJMUeDLAjxXaFQvA/d/93kwlGO3Qz+uhHZGuyZ6Xsr8CbutWfbuDPuMu++i6B+cTXDsaqQiEaDGeNxM1HDj8CqaKKEB+AG+EG/c07RmPtQ8UCRZGqSSVu8IqEUho15qOwL6E8o1QQxggiiyASBqVC7UBXKgpjte6T3LjnnE9TSD+H08AaPdGeagpNmNZ8CiJUtL7z3zDyYhnNbeAPOGXPpyDEfQrliqcpeM7QWQnzkcMf/CstaApu0N8cOpJHwkGnVBQKBckcPMYmKpmDZI35KEVTgFoNIUUm1PgZmkGIBv9IKASfpWkFE1nhmaFVrqGm0FPzUbHBnhlIXCCa+choBxMKbeAPrhXVquYgEgx8zlRUUapaQzlmPgo0haSD0f+BqzbvaHaCZ/veCYaKwqLZw0AwEy0ImeajeuGXyYEm0hQkvIdI8ECkIaRpCs04m9Nw3a7VFGr39YVw8hySQ1PohflouG3zkaCqVaFoMsFoBxMKbeCbX7buHPMGc0nVFFQ1JhSyNAU/YqYVR/O4Z69fNHu4etxQsRD0K0NVaCb6qJDQCJKagvs8rcvtzr6T5iOtYz7K0hQ0NMs16kovzUcjLZuPgmfia6qG0SoWzNwG/hj0Z1+7naNXzAOcplCoagouMimIJfeFQrpPIRkd1Kym4AuFxbNH2L43yGFwjubWoo+SjubwVZJmo0LsfZqmkDaTzZvJKyKR+Uji5qO0U4xnaQrV6KNGjuZc3WoLJxSGSq1drBA+E/dczHxktINpCm2QHMjufHQ7kO5TgMA8M1H2Hc3x6COHbwapqFYH0bymDN9hvHjOcHV7qCix6KOa48qVqlM7SfLSSfNRjabQpCBrxUma1BTKFa3pZ13zETnMRz3UFLKefR58R7PJBKMdTCi0QdY4JiKhTyEYnJ25ZnS8nNAUAqFQKEhs8GnXp+CbgRbOGq4OElXzUSW9/2N1fAqNHM1JB3O9kNQ0mpEJmY5m1ZoInvFM81GoJUwG81HY52Q0WF6c9lc1H5lUMNrAhEIbZM1unabgZqkVT1Mop5iPIB6R449jQUhqQH7zUXSNGUOFaj+HioW6yWv1Q1Lj104O+m4gck7zpAbRiPzmo3ohqVGklSNbUwieayNTSy9MMa06mB2FgtMUajUlw2gWEwptkJUt6/IUqj6FcL+xiUqqpgBxZ3OsfIZGA26zGc3BMdF5q+ajVhzNGZqCO301oSzhU8jTZ3+Wm4dk9JG7HVWtCeucyLgnDe1Hkyn6qFWqmoKak9loH3M0t0Gm+SgZfVSOawq/ddwKLnjW/jFB4IelxsdsbTpPwcfXQIZLYfJavZDUjPNkhaQmB82kMMgnFJrzKdQ6miOfwlBRYtrEeFb2Gi5Pof61ehl91CriktdQczIbbWOaQhtkmo9c9FE54VOYKDNerrB07gjPPWRJ7JhSlqZQiQauVoRCvHBdoXHyWsagkmyNktOSUUnpvoV6lFVz29OF6LkXanwKgfDyHffZmoIiIg2du21O4nPRtlDAlbkwJ7PRPiYU2iJ7ICulRB+NhZpC2uA+nCEU/NlfK6aMZAVTP3kt2f/xevWIkuajLE0hUR01jyCreCVCchHuW0r4FJxN3Q/xbZS8lpZx7d9ST3wKbZuPIm2rnQgmwwATCm1RqaSXOxaBYtHPaA5e944HPoVSyiDgD2Q1PoWUffIS8ymUCtWYdkgpiFfH0VxjPqo6mAlfnTAIrteMozlpPmo0Diejj/zktWJBYtpJ3eij4Go1n/n3Ogghqe5/qpij2WgfEwptoKTP+gVJ1RTc4jdpJhV/Ruubwauhk7Q2a435FIpxoZCkqYJ4iXtwA3NNldQcffYrzDZCwjLRED0P97zKznxU9M1HdaKPJL2Mt9/UC0dzWoJfMwSO5ighzzDawYRCG6hmZ+wWw4xmDW29ALvHghXF0gSJq40E8cJufphhK4NH0qfgO8CT1C+dHX+fJaBq1lVowXzU6IjaPIVIU0iaj7JMYkH+R/q1YuajHmgK7Q7kIlHtLXM0G+1iQqENlHSTjiuIB/Ewz91j2ZqCv2xkMqPZ/dBbmVCWYkIhqJJaLQuR2LeZPIUsM7h7Hm6GnWeIqlSaS16rOprDk0dlLoJn5QvPiQwBqGHv0gZR39neC02hGlLbYvKai7byTY2G0SomFNqgopquKRBlKPsD7a7ReppCtJ+fyKVEg18rmkIpFvZaqC7Ikka9KqlNawqhcMgzzPkVZvPgL2YUOFldkmAwiA/FHM31MprTZ+kx81EPNIV2qfoUlMzoMcPIiwmFdtD0WX9MU4gJhUBTSCsd7WsUcU3BT15rvoulpPmoQe2jLLJWXkt+7gRXMzPsciJ5rd7A5oekihDzkZSr5qPoQWUVxMOt0dygb4Mwxpqj2egkXUteE5HPAi8GNqvq0WHbl4HDw10WANtUdY2IrALuAe4NP/uJql7arb51CiV9Jileu68B1PMp+MRrH3kF8dr2KTjzURTC6VPOHECbdzQ3M8OuqKKePGp0ZGQ+kuqqY64Pgfko2je7dHZ2lVT/+fdCU3jO6kW8Yu3+/PGZh7Z0vEjwXbSMZqMTdDOj+fPAvwJfcA2q+kq3LSIfAp729n9AVdd0sT8dRzUj+ihDU6jnU/BJFsRze7di3y7F8hRC81GGQuAv/ZkkKY8yzUfFFoRCBbSY33xUXYuYyMkatAf9LNI4+sjda9pt+IpUL3wKQ8UC/3DBsS0fHy3HqWY+Mtqmm2s0/yDUAGqQ4Jv7CuDMbl2/F/hLZcYRiqEJwzfJ7GpFU/CT11rKU4iOCaKiJDMip15GbFZBvCTJkNRcjuZk9FGdg9ys2G27cEx3nqJILGKo4XoKKT30zWu9iD5qF5GoftQAdNeY5PTLp3AKsElV7/PaDhKRn4vI90XklKwDReQSEVknIuu2bNnS/Z7WIciKbUJTCH0KjZLQYlVS/TIXrWQ0J6b4seS15HXrOHuzCuIliUJSXTnoxlSSPoUGoiRa9lRCzSfKB5FkmYss8xH5ah8NkqO5YpqC0QH6JRQuBK723j8GrFTV44C3A1eJyLy0A1X1ClVdq6prly5d2oOuZqNeWWufuE8hyj+INIX6jz2e0expCi3WPvrrlx7NgYtnsc+8kWAAySiIV18oJBzNGYNP1dHcxDerUslfOhs8RzMkfArBTNlfway+ptA4h3gQ4v6dSVAzNVfDyE/Pq6SKSAn4LeBZrk1VR4HRcPs2EXkAOAxY1+v+NYOrn5NEvLBIP//A+RSGGpqPktdoLBSyktJKReGMI/bhjCP2ifbLGIDrxblnOZqTp6rmKTThFE9qCvVHaok5muM+BVfmIkdBvPCKjcxDg6ApuCzv4N8/+ftrTG76oSk8H/ilqm50DSKyVESK4fZq4FDgwT70rSmyHHuBphA82tFyvjyFS087OHbe2DXC7XoD1IyMSpvJYwJNITx3Yt965oe85qMo+iizqzWUW8xoDgraSazdF8iQnbxG1adQn144mtslytWwkFSjfbomFETkauBm4HAR2SgiF4cfvYq46QjgVOB2EVkPfA24VFWf7FbfOkUyBNCNT75P4fv3Rn6PavRRik/h3ecewfVvOxWorX3kJr51hcJQMbU96RCul7xWyRpASdEUGpa5cEXeGqNNJ6+FfZK4j6RSCe7P1xQyy1yExzdejjN3t/qGECWvDYBiY0xyuhl9dGFG+2tT2q4BrulWX7pFsgCZhPUG/Izmz/94Q/Xz3Q18ClHZhrhPwVm+64WyZguF+LXq1T7yl/6s7VvCp5BV5iKx8lralf7wtIN5/3/fXX1fTpS5aBh9FD6fgnM0e+ajkhTylc52IakN11OY/KOsW46zXkixYeRlAOZBk5fkSldpmoKPy2jOGtyd6SYekhqdt57Tc8ZQhvkooZXES2fHB8w6ikLTZS7q9fW1Jx/Exc87yLtu0nyU3+fiVh1z5ykW4lVSs/wnTlNoNOYPgqNZxFuOc/J315jkmFBog+Qg6g9maTPMeqWzwS8F7WsKVH/prWkKCaHgFcRLonVSYmsL4tUXCo3wd2umdHZApCkI/noKwQDp33OWVuRutdEgOghCoVo6m8HorzG5MaHQBskCZG5TpNZs45OVp1Bdc9gzgwelG4LtepEyWUIhOUgX6/gUsgZQyHY0tzoG+YOXWwsg61qxfhAJ46pPIXxelfBZ+bWP6lVJzbcc5+QfZJ0JrV5IsWHkxYRCW2jC0ewGSqkZTPyBLsun4PZJlrnIM/vLMh8lhVOhTkG8uiGpiQ/yRuVk+igSUUPx5LX6RHkKtT6FQiL6qJxZ+0jzJa8NwMy7KijNfGR0ABMKbeBHBoHnU6BWG5jlzeQzzUcpsf/1nL8+M0r5NIVCQTKjjOqGpOY0H+UlZj5qsXR2QRI+hQq16ynUWaMZaVxqehCij/yMZjMfGe0yAF/5yUsy2sMPlUwOmjOHo0E764frDtm0fW91kMyrKQxn5Ckk15AuJjKAfeqNy8nBMXl/px8eJMfNnznUsK+QNB9p3Wv7iETmtSBPIb5Gc0FIaAp18hTIkacwAOYj8XwKJhOMdjGh0AbJH2FUvVNqtAFfKGT5FNxA+aEbfsWXbn0kOmeOH3qWUKjVFKKInKRzNzmAHrV8Hs9etRDI1hTcYP7eFx3Jj999JotmD8f2yxrr/Rl6uZL0KdS/4fgiO1JjPirlSF5zkWNTwXzkktfyTiAMox4mFNogOV77DtCRhDln9nCUEpI1+fR/zzc/sDW4Rp39fdIW7oF0n0Jd85H3/qClszlkn7k1fXPniV2nWGC/BTMbdzTEH2zdAjF5yap95Epn+/dcN/pIGoe/DkKVVPcM8poaDaMeJhTawK9LBFFGsFDr+M1nPqqNZFLNl5CUJRRqoo+8RXaS7TXJeGSbI/KaVTIdzb6GVclf5kK82keSUvso6WjO1BQ0clTXYxA0BfcMrPSR0QlMKLSBvyoaeFFDKZqCP2hnDajxRLgwPFXzOTuHM0xStWUuougjfyAupRTKcxnDyb5B+4NlPPooYWZqcGrf0eyvp+DMJ35Iamb0UVgnqGGewgBoClVnu5mPjA5gQqENMs1HCCMJG/+wN1BlZgNnaAp5fuiZ5qPUjOaU/VI0CN+8Upun0LBLdfHP12z0UTxPIRLG5UqYp+AN5Gmawjkf/QH/e9em8BwNzEcDMMYGfVQzHxkdwYRCGwSzzdqfoUhtMplv0siafYr333CDcb3SE7HzZziaa2sfpSevlYqFME8hrq1UE/IS+zcyHy2YFUQhrViY7meo9SlE1DUfSXKRnVpHc6zMRcoD/OXjO6rHN2IQoo+i5Tgt+shon56vpzCVyKpKKdRGA/nvs8aZtDpKeUsX5PUp1DMfJc1hzpGb7Fva+yQnrl7Mx199fHUdhyQxs1slrik0GqwjjSyepxDkjUiugnju+Eb3MQjmGKctJWtxGUYrmKbQBllVKdMymv0fa5Y93j8k5mj22r/yxpN48xmH1Byb26eQ4WguFVMynVP60wznPnN5ZvmNeIJe/RyJlKPDPom3loCXp5Aj+ig4PkdG8wBoChJGlOXVKg2jHqYptIEqqbaOtGEkz2Lwseij8CzJ2PMTDlrEY0/vqTm2lKUp1PgU0k1SpUKhpj0ex9+9wbHGfNTI0Rzu7BzNrt+BTyFfnoK7TqO7GpToI1c/yjQFo11MU2iDrByCtN+lPzZl/XD9ZjfZTXMepplXZo+ky/ekplD0zUfeUFwqSo2zN24+Sj19y9Saj+LXzTyOuEM/rimEVVJzRB+546dC9FFBguU4k1qlYbSCCYV2SDhmHWlt/tiSbT7y2yNNISkE0sapUw5ZwmXnHsGchHCo8QVUM5HjA3Fa/oIrTQ21gq7dgpw15qOmktfSax+pKsVCfA3sRprCVJhZRz4FczQb7dPN5Tg/KyKbReROr+39IvKoiKwP/87zPrtMRO4XkXtF5IXd6lcnqYQzs6ve8Jz4B4kf5mtOOpC1YbkIiEcZ+aQlrzk7efz0tb/8YkF442kHs3TuCAB//bKjed/5R9XY9N01PvmD+BLYQ4UC5Uo8msq3uXdzRa+aNZpzOpqRKHEraHfmo5w+BW9733kzOHDxrGa7PilwEVh5w5cNox7d1BQ+D5yT0v4RVV0T/l0HICJHEazd/IzwmP8nIukeykmEm5k99+AlrPBKPCR/l+84+/BYuew8juZq9FFKmGE9k5XLhzh4yWxed/JBNfs5x+nl3/plbMAMzEfxgVIkGqCz+pDly2iGpNZSPyRVotpHSCx5rVmfAiLV+xsuFXj72Ye1egv9RdxynJbQbLRPN9do/oGIrMq5+0uBL6nqKPCQiNwPnADc3KXutcVvtu3hwzf8irGJCrPC8hXJUE6foZLg+3uzIlpis3RnPkoJM0yTKW6fkbC8Rh5n9vpHtlW305PXIpt7sg+H7DOHN562mlefcGDqdRoRS16rNLlATMzRHGkKToDmjj6CmHlsUGfZgWB0ZS4G8x6MyUM/fApvFpHbQ/OSs6msAB7x9tkYttUgIpeIyDoRWbdly5Zu9zWV9/7XnXztto3c8ejTXr/wtuM/zKFiISYI8vxu3e6VFE0hzQsUjAwAACAASURBVLziBrThBjN3X1b4s+hiyjKdwaAp0ZtEHy4790hWtmhySfoU4ueuf2yU0SyhTyEyHxUTmkK5kp0t7TKiXX8GdTwtVKOPak2NhtEsvRYKHwcOBtYAjwEfavYEqnqFqq5V1bVLly7tdP/y9qG6XTWveKNm8ndZKkhsIM8T5njlzQ/zi0e2hRElCU0hZX83GLgkufFyVs2fdEpF51Pwz5md0dxJktFHjYjXPoqERCVMXkuuIZGlLQjp2tmgEfkUzHxktE9P8xRUdZPbFpFPAd8M3z4KHODtun/YNulxP8K0xLPofTyZLa+Z4qX/9n+sXjK75oeedrxvGwcYm0gXCrtHJ1Lbh4opGc3eTLqbppXa0tn5HM3O5OPCTsth/5OlPSYqStrCdL55DOL/wwuetT8zMxLvJhsuAssymo1O0FNNQUSWe29fDrjIpG8ArxKRERE5CDgUuKWXfWsGf/iqLsHZYMbpawfNxL6nlblIK0bnhM5IA6GwI0MoFAu1tY9881Gnx5pYnkKTGc2RpuZqH0XtvvnIaQx1NQWvP36fznvmvvz1y47O36k+4rqd1PQMoxW6pimIyNXA6cASEdkIvA84XUTWEIx1G4A3AqjqXSLyFeBuYAJ4k6qWu9W3dkkLn0xG7SRpNQkqb0iq28eV7B7NEAo796YLhbTS2dJF85F/qXKzGc3ha0GikNR1G55kvBzMlJ1fZbhYYLxczoxA8qOrkvkgeYrlTRb8MuuDagIzJg/djD66MKX5M3X2/yDwwW71p5OkGjpSfovvOe9IbrgnsJi16gCspPkUUs7l9rnk1NV8/1dbOPmQJann21FHKNRkNHuDZjfNEpoond3oSrVVUuGCTwSBagWJwmSHSgUYK2drCgnzkX/dQTLDVIMSKjoQtZqMyY3VPmqBuKM5eE1LPHvDqat5w6mrgdYLq6VFxaRHHwWvR6+Yzy/e94LM8+0MzUfL5o2waftotX3Ilc5OOpqrF22l9/koN1HMTSS5HGet49+V9nAaw0RGqYtYdBXpVWoHAaeFBr6TAeq4MSmxMhdtkmo+SjXvtCEUEudLG7Dynv/1z1sFwKmHxiO30spcBNE5ubvaMoFAqBW09fd3K8PF+10sROspuNdsTcHL2JZagTgouK5mVe01jGYwodAmvqOSlG1H65pCrU8hbcDKO4idecQyNlz+IpYviC9+UyoEhfKSvpFeDDK1BfHqXzMyH4WRN54iEJiP4k73rDUV/DWakxVGB0gmVJ9XuaJtr4hnGPYVaoG4ozl8rZOnAO34FFLMRyn7NTuIJcMtXZkLYiaUxgvbd4Jk6ex6iEQ6hQuZ9TUF1Sij2WkKWdnSNUJvQDUF9z8KhPrg9NuYnJhQaIFKzCmaL2SzZfNRapmL9s1TM4fi//pgOc7alOZejI21BfGy9xUih3iy9hHA3okyM4eL7DN3hIOWzAay6x+JH5Ma3xyooTWKPrKQVKN9TCi0iVPXG4Uztmo+cmsE+KSGvDarKQwnNIVUn4Kk3kuncSUaous23h9CTSGx5vSesQrDpQK3vOf5nH/sfkC9PIV4afCYo3mAPM2u2xOV9DXDDaMZTCi0QJr9u9Ess9VBJm3hlHZ8Co5kSe1SoUClkgzL7JGjudLMagp+nkK89hHAnvEovcUJ4kyfgicIklFeAyQTIk2hUrsgk2E0iwmFFogNYS4k1XuS6TP51qOPapLXUk7Vrk9hqJhWJbU3juaJRJXUerNdl6wW9S+eN7JnLMrDcKGpjdZpdhQaaHuTFdfVshXEMzqACYUWSKv9H3M0p5mPWhxk0sIM09dTaNKnkDAfuZBU34wTX6O5e4yXK9mV+lJIrtHs/z9imkIYhZSZpzBlktecpjBYwsyYnJhQaIF47aOcjuYWn3RQ+yje1okffk30UcEVVfOuk3LtbjA2UcktE3aNltkz5gZ+qYk+2jMeCYBGmkLgZ47+f/5zHaQZdzz6yDDaw4RCK6SGpMbfJ2lZU0hxHnbih1+zTGchimCJLiQ9MR+NlSu5o4+e2DnKO6+5vbpfMnnNNx9VfQo5ktdqfQqDM7y670c5pSSKYTSLCYU2iYRBrcPZp2VHM7WDZCcGrBrzkdQ6ZbvpaF4VhouC0xSacTUHVB3NlUgreMZ+86ufuxLaeddTGNTkNddvq5JqdAITCi2g1DpFo3IJ6b/K9hzNjUNSmyVpPiqkmFq6qSW88Bn7cu2bTmbFgpmBUGheJsRqH61YOJND95nDn593ZPVzl9mcVUZcJH6Hg6spBK+B/8kw2sOEQgtU6pmPMo5pPU+h9ofeEU0hIRT8CJboOt0dHI89YAEjpQKj5UpTpbMdUe2jQJg9c8X86iJDAAtmDgGwbc9Y6vGCL8wH2dEcvJYrtsiO0T4mFNrEzTUbOZxb9SmodieRKmk+8mPdHb7NvVsMlwqMT1RSs8QdV77+hNRj/eS1SkVrntPi2SMAbN2ZIRQ8oVe7nkLz99Ivqj4FMx8ZHcCEQguklc6ONIb0X2WrP9ZuaQojpfi/vupTqMRNY90eY4ZLBcYahKQ+e9XCzM/cUpQVrRW882aWKBWErbsiobDq3f/jH13HfJTzBiYBvk/BNAWjXUwotEDaIjvV4SVLU2jL0ZzIUwj/a+2sIZxVOqOc0BS6Xe5huFiocTQnx7Us0VQoSNWnUNbaCqEiwuI5wzxZR1OYCslrvvnInApGu3RNKIjIZ0Vks4jc6bX9o4j8UkRuF5H/FJEFYfsqEdkjIuvDv090q1+dwHeKFiQuDDrtU0grc9Gp+fvnXvfs6rY/2/Sv0xNNIeFoTl4za3wOHM0SmY9Sdlw0e4Stu0ZrDyYefTTIPgVzNBudpJuawueBcxJtNwBHq+oxwK+Ay7zPHlDVNeHfpV3sV9ukOUXdmJ81lrQ64U4rc+G/v+kdp/Hp16xt6dxnHL5PzTl9oVDwPbFdwpmP6kUfZT9Tz9Gs6UtRLpkzHDMfJY93RySjvAbRfFRJiVQzjGbpmlBQ1R8ATybarldVl130E2D/bl2/q6T5FKqF8doLSf3iHzwn9r6iaaWzo+2Dl87h+Ucty3XueqQlegWO2LZPXZfIfBRRm6yX7acRCZ5Rlj190ezhqqO5kshXSGYxD25I6mA6yI3JST99Cq8HvuW9P0hEfi4i3xeRU7IOEpFLRGSdiKzbsmVL93vZkHjUUWb0Uc7R9eRDlsTep5mJu2Hvdues1JiPujvKDFXNR9mlsxuZj1SzF62fM1Jid5jlnMxsTpqM/OsM0uDq3/YAdduYpPRFKIjIe4AJ4Ith02PASlU9Dng7cJWIzEs7VlWvUNW1qrp26dKlabt0nXjto+A1ma+QpJ2ZZ42juQsjVtWnkNCCuj04jhQD81GdQqaZ9ysS1T6qaLrgHSkVGQ3rISUzmwXx1sOIaySDpCnEzV6D029jctJzoSAirwVeDLxaw+mhqo6q6tZw+zbgAeCwXvctL5UU81GUp5AdKdMqtY7mzuO615eQ1IkKvqhNyoesPrg8g3JFw7o/tfuMDBUYDTOaa6qleoIgyAeJPhqkwVUy3xhG8/RUKIjIOcA7gZeo6m6vfamIFMPt1cChwIO97FurJIfNzOijNgaZ5ADVFU0hpcxFtzOaId3RrFpr5kmjIMJIeHy5oqnP2H1eqWiKpkDsHxbXFJq9k/6RVb/JMFqh1K0Ti8jVwOnAEhHZCLyPINpoBLgh/CL/JIw0OhX4KxEZByrApar6ZOqJJwFpFT2rP8ysAawN8ZvXxt4O6SGpKRfvMGmO5hpNIct8BMydMYTWiT4aKQW5HGPlSopPITt5bZBm3OZTMDpJLqEgIm8FPgfsAD4NHAe8W1WvzzpGVS9Maf5Mxr7XANfk6ctkIC1PoVBfJrSnKSQGu+4Ihdq2XpqP4ppCvmNFYO6M6CucNkt2mduj45Wa6CNNRHb5z6Cd/1evGdTqrsbkJO/89fWquh14AbAQ+H3g8q71apKTNma532Knq6QG50y+7/wvP22WXZDuXMtnqFhgIvQJOOotn+kjIsydMVR9n6opDIVCYaKcuq5C/PaiN8OlwUn2H1RfiDE5yWs+ct+084B/V9W7ZJDqAHSY9NpH8dDUJG05mnMsx9kuqf/OHvyLh6sz+WgZzeRa0VkkNYV65qPRlPLZSrYfYZCEQnwp2D52xJgS5P3m3yYi1xMIhf8VkbkEtv9pT5S0Ruw1SatlLiAto7kbIan52jpN1bzjDdp5NYWCSEwopEYflfJrCr5gHGrHCdRjsrQdw2iFvJrCxcAa4EFV3S0ii4HXda9bg0OhRlNI/1G2Y6PuTUhq7Vl7sRSnm5H7C+HklAlVR7MjK/oIYO94BW/X6BzeIb4Q7HYhwE7iTzgGqNvGJCWXUFDViohsAo4Ska5FLA0K6dFH4WvGMe1M7mvLXHRDU0gRCj3UFPZ45qNkSGoWIjCvkfloKIo+ciuxRRdKmF4GdJY9VIy0GjMfGe2SN/ro74FXAncD7terwA+61K9JTTx5LWE+yvhRtmM+StKr6KNezDpnDgdfwT1jkVAo5xQKhYSjuVH00XCx1iQ0qKUtfGJCYUAFmzF5yDvrfxlwuKqm1yCexiQzmrN0hUmfvNYn85FbE2LX2ES1LRk6Wo8ZQ9GAmCZ4hz2fwmyNrz+hxEtND6pQ8IWdmY+MdsnrTXsQSLHITk/ii+zEo466EX2UPLQrPoWUb0IvBslZ4bKgu0f96KN8xxZE4tm8qdFHkSM7zdHsHzOoAXVDpcG/B2PykFdT2A2sF5GbgKq2oKpv6UqvJjn1l+PsPL0oiDeUalrpgaYwnKIpNOFT8El3NEchqcmoJk1UoB3UWbb5FIxOklcofCP8M6i/HGcv7P3SwWjJZfNGeGr3OPNn1iqCQt2lkztCVVMYayFPIXwdKgrjZU0d1Ee8PIiJcoOQ1AG1xw+bT8HoIA2FQlio7rWqekYP+jMYpJS5iDSFrmSWxd928NT/964zUeDxp/fWfFYoQLnL2SizhoKv4K5R36cAr1i7P19Zt7Huse7ZFwuhUKib0ZyuKfhPc1A1BT/RzjQFo10azjlVtQxURGR+D/ozEKQtx9koo7kdupm8VioWGCoWWDArTVPo/ggzYzj4CiY1haQ5a/XS2TXHusfgEs0amY9qSmeTeLYDOqAOmaPZ6CB5DRE7gTtE5DMi8jH3182ODQrJmKOu+BSS5bm7cJE5IyVKPSi8l2RWGJLq+xTS1ka47i2ncNLqxbE2J4hd/kF6mYso+igtU3oqlJ0e8vIvzNFstEten8LXwz+DRHJV0nzU4Ef5jP1SF5SrSy9mfyLCgllDPLFzLNbWba+CC0n18xRUawXhjKEis0fSv67FUFOoG300nh59NAUUhUSegmG0R96M5iu73ZFBohLzKbjXxj/Hmy87k3lptRYSHLHvXH75+A7vGr35qS+YNRwXCj24ZrEQLJTj+xQgXUvJynR2M+U085GIMFwKVl+rKZ1NXCMZXE3B9ykM5j0Yk4e8Gc0PkTJlVNXVHe/RgFFTEK/Ob3L5/Jm5zvnVS0/iO7/czFu/tN5dJIYzkzz7oEXNdLUhCxN+hV4NkjOHi+zYmxAKKftlRSU581GWRjVSKmQWxJsKaxEUC0KxECxLOqj3YEwe8pqP1nrbM4DfATo7Ig0QSm2eghvFOvGjnDtjiFWLI8dqcgY8UiryrbeewoGLZ7V/MY/5M4dj73s1wMwaKrJt93ji2rUXz0pqG6pjPoLgeWVHH9W/5qAwVAyEgjmajXbJ5WhW1a3e36Oq+lHgRY2OE5HPishmEbnTa1skIjeIyH3h68KwXUIH9v0icruIHN/yXXWZmEshfK2GpnbI6OLPYNMcqEcun1d10nYKl0jWa9Kum2o+yjjePZ+sUiIjpUK2T0HStwcNZ0KyPAWjXXIJBRE53vtbKyKXkk/L+DxwTqLt3cBNqnoocFP4HuBc4NDw7xLg43n61g9iy3EW8puPmsEvO9HJYnr1SBaMG+92kkJImnBLG9yyfAqlsN9Zz2lkqBBGH8XvJ7nIziAPp+5/N8iCzZgc5J1qfsjbngAeAl7R6CBV/YGIrEo0vxQ4Pdy+Evge8K6w/Qsa/PJ/IiILRGS5qj6Ws499pdNlLvwBrmdCIbHaWFoGcDcYSpa0Jn1wy/QphM+nkfkovfaRtz3AI2pVUxjgezAmB7kX2VHVB/0GETmoxWsu8wb6x4Fl4fYK4BFvv41hW0woiMglBJoEK1eubLEL7ZFa+6ha5qLz5qNeDVYjSaFQqVTDPbtJKeUaqY7mDMWlVCf6CJyjOav20eA7miF6BgN8C8YkIe8v/ms525oi1Aqamo6q6hWqulZV1y5durTdLrSNG1TcuNapH2Ujn0I3SAqF8R5pCjWL39CcplB1NNeJPnps2x7+8tq76l5namgKfe6IMfDU1RRE5AjgGcB8Efkt76N5BFFIrbDJmYVEZDmwOWx/FDjA22//sG3S4Q9N0UDUWfuRP8AlM427Ra35qAJD3Xc+pwm9NI0rq06eEypZ0UnDpQL3bd6Z8omm/rt6JYQ7SdWEZlLBaJNGmsLhwIuBBcD53t/xwBtavOY3gIvC7YuAa73214RRSCcCT09Wf0Iry3E2S2zd3T45mtNs8N0gTeil3bFmKJXO0ZxW2wii+kdppAmfRbOHU/ac3BQLZj4yOkNdTUFVrwWuFZGTVPXmZk8uIlcTOJWXiMhG4H3A5cBXRORi4GEih/V1wHnA/QTrN7yu2ev1iliegjMf5SxzkZeY+ahHs78aTaFXQiFlLYe00S2rO06oZDnGR4bS5z6qcXPLjKEiH3jJMzj98P6bJZvFmY96NYEwpi55Hc1bwwV2lqnq0SJyDPASVf2begep6oUZH52Vsq8Cb8rZn76SqinQ2ZlaYVJEH/UmJDVdU0hLXqsffZQlxJK+kvh14lz03FWZ+05mBtHkZUxO8jqaPwVcBowDqOrtwKu61anJTrx0tos6IvbaLsU+OJqTQqF3jua0Vd9q98vUFIpOKHTGfDSIuLDeKXI7Rh/JKxRmqeotibaJ1D2nAWkZzdFrp8xH0XaadaUbJNcwyBpkO01un0KmphD6FLLMRxmagmZcZxApVSOwpsodGf0ir/noCRE5mHCSLCIXkMgfmF6krdHc2Zmabz7q1Q/dzTbPOmIf5swocckpB3PDPZu6fl1fKJxw0CJueejJuiGpf/bCw2MC4L0vOpJiQTjn6H1Tz5/lU4CpM7O2PAWjU+QVCm8CrgCOEJFHCTKaX921Xk1yUjWFDv8affNRWnJXN3CawshQgX9+1XE9uSbE8xSqZpDUMhfB62mHLeXoFdFCgPvMm8FHXrkm8/xZ5qMszWMQOXL5PH543xM8lrKsqmE0Q971FB4Eni8iswlMTrsJfAoPd7FvA0FU+6iLGc09Nh+NTfR2sPSFXr0kLOdTaPYR13M0TxXectahPLhlJ+cfu7zfXTEGnLq/FhGZJyKXici/isjZBMLgIoKw0Ya1j6YqMUeze+1wnkI/C+L1qhCew7+/qNpnLW5m36zfxhcKs/tUCbbbzBkp8emLns2zDpy2Fe2NDtFIU/h34CngZoJktfcQ/F5frqrru9y3SUvacpxRnkJnrtGPPIWhPgmFUqHWfJT2IJ1PoVnNacTLyrbQTcOoTyOhsFpVnwkgIp8mcC6vVNVpbbhM1xQ662juR5VUNyD3qjqqww9Jdaak1IJ4YbeaFZLDxfRnOXU8CobRORrNuarLYalqGdg43QUCJNZTqC6uQ/jahYzmXgkFt8h9HzUFv4bPy49bwfErF1Q/e+tZhwKwYmG+ZU2jc0Zf815UfTWMQaaRpnCsiGwPtwWYGb4XgiTkeV3t3SQltXR2p0NS+1C9s+pTmOixUEiZyYtQE1F0/rH7cf6x+7V1fn/thikUfGQYHaNR7aOp6ZVrk0qdkNRuFMRLKy3dDdJ8CmsPXAjAy9as6Np1Syn32sk77ocpzjAGlc4u8jtNqKQuspNoaBPpi6O5tobQqiWz2XB5w+W42yLVp9DBWy5lCAVTFAyjFjOwNomqxhyxnTYbpdGrypdL544AcMGz9u/J9Rxpg3YnaxL1wz9jGIOKaQpNMjpRYcwzr7ghJulw7iS90hTmzhji/g+e2/OBs9vmnZhPwXM0T6WMZsPoFCYUmuTpPeOx95KMPurCeFrskU8BMtY26OE1fUdzpygWas9vGEY6Zj5qkqRQKCWy1gZZU+gXvvnIaVydnMSnObId82cOAXD2Ucs6d0HDGGB6rimIyOHAl72m1cBfEiz5+QZgS9j+56p6XY+715CkUChU4+q7d82pPruN+xSC10oHV32rZ55aMGuYW9/z/IFcgtMwukHPhYKq3gusARCRIvAo8J8Ey29+RFX/qdd9aobtGZpCp5LW0pjyQiGWp+DWW+6cUIiV0Yj5FIJX52A3DKP/5qOzgAdUdWCqrdZoCj2IPpr65iPP5h/ea9bSm63gC1VLaDaM+vT7J/Iq4Grv/ZtF5HYR+ayILOxXp+qR5VPo5rA91RdjTzMflTuqKdTmQQCoZSoYRg19EwoiMgy8BPhq2PRx4GAC09JjwIcyjrtERNaJyLotW7ak7dJVkkLBzULdwG3DTPPEbf7BV7LcJU1hqpviDKNd+qkpnAv8TFU3AajqJlUtq2oF+BRwQtpBqnqFqq5V1bVLly7tYXcDtu+ZiNXkTw4yFvrePEOxkNTgtdzBSq2+z+LgpXOq2/a/Moxa+ikULsQzHYmIv2TUy4E7e96jHOydKDMzRSi4EhGdtIVPF4opIand0hTeec7hvP7kgzp2bsOYavRFKITLep4NfN1r/gcRuUNEbgfOAN7Wj741YmyiUq0mCtGA42zV4z1ei2Aq4M/knX+hkyGpvs9ipFTgrCP36di5DWOq0ZeMZlXdBSxOtP1+P/rSLGMTFYZLvuMy1BTCtnKlt2WnpwKllIzjbmkKnaypZBhTkX5HHw0cSaHgHMzDfVq1bCrgawpu0O7kOj8li0M1jNzYr6UBv9m2h+f87Y1seGIXEKw1kKYplLqQdDVdSKuS2q2MZh9z/xhGLSYUGvBf6x9l0/ZRrr711wCMleM+BecYdeajiR4vZTkVcIP2/JlD1e1uZTRDd3NKDGPQMaHQABca6QaW0aRPITR9DHVhMJsuHLZsLn94+sHc8LZTu5PRnFFl1pLXDKMWK53dADfIu6SqsYkKc2dEj80NYi7W3oRC8wwVC7zrnCMAz9HcRU3BVAXDyMY0hQa4wckNLGMTFUZKKSGp4Wx03MxHbdENoWA+BcPIjwmFBkSaQigUEo7mYjX6yIWk2kjTDt3RFOJf825WtDWMQceEQgNc3kGpIFy7/lHu37wzNXktcjSbUGiHbmQ0Z5U7sv+UYdRiQqEBTlMoFQu89UvrAVI1hVLV0Wzmo3boRkZzMmHN8tcMIxsTCg1wZgw/gCVNKDhHs1mP2qMbIamZ2P/KMGowodAANzj5VqHholcQLxF9ZLSHyxDvZmFBUxQMIxsbyRrg8hT8pLShkl/ALXiEQxmx8EZzlLrgaM7C8hQMoxbLU2iA0xR8c8aIn9EcbnZDU3jJsfuxZ7zc8fNOZo7ebz4Al552cNeuYUXxDCMbEwoNcNFHfv5BqejXPnKaQueFwscuPK7j55zszJ81xIbLX9TVaxy2LFho5w9OWd3V6xjGIGJCoQFjoTDwQ019U1KkKdjsc1BYMGu464LHMAYV8yk0YO94KBQ889GorzWEUqFkjmbDMKYANpI1YG9o0/e1g/GJSEC4xKhhEwqGYUwB+mY+EpENwA6gDEyo6loRWQR8GVgFbABeoapP9auP4AkFT1MYK0fOX+e0LJn5yDCMKUC/p7dnqOoaVV0bvn83cJOqHgrcFL7vK8585DuaR8drs5ZrKnEahmEMIP0WCkleClwZbl8JvKyPfeGRJ3eze2wCiDuaT1y9uGZfC3M0DGMq0E+hoMD1InKbiFwSti1T1cfC7ceBZcmDROQSEVknIuu2bNnS0oV3jk5w64Yn2bZ7LHOfp3ePc8o/fJcNW3cDMB6Gpv7eiSv57Wft39J1DcMwJjv9FArPU9XjgXOBN4nIqf6HqqqkVKdR1StUda2qrl26dGlLF35g805+5xM38/Nfb8vcZ8foeOy9MxktmzujpWsahmEMAn1zNKvqo+HrZhH5T+AEYJOILFfVx0RkObC5G9d2RdfqLYiTNAc5h3PW0o7G5OZnf3F2tU6VYRjZ9EVTEJHZIjLXbQMvAO4EvgFcFO52EXBtN67fytKZeycCoTBUmGxuGCMPi2YPM3/WUL+7YRiTnn5pCsuA/wxn4yXgKlX9tojcCnxFRC4GHgZe0Y2L51k6s5xYLMdFIWUt7eho9LlhGMZkpi9CQVUfBI5Nad8KnNXt6+epxJlcLMeZj+rlI9z49tOYN9MqhxiGMbhMyxHMlaSot3RmUmC4aqXJ9X59DtlnTgd6ZxiG0T+mpYF8yDma6yydOZ4QGC76yJLUDMOYykxLoVDMYT5KflaNPjKhYBjGFGZaCgVnPkpqAz5JLSKPT8EwDGPQmZZCwa19MFEv+iipKUw489G0fGSGYUwTpuUI50xA9fIUkuGqTkiY+cgwjKnMtBQKLgGtmeij6rFmPjIMYwozLYVCoSAUpDYXwSdLYJimYBjGVGZa5ilA4Gyu52j2TUsjpQKjCZ/CJ37vWVQ0f5kMwzCMQWBaagoQ5Bt84vsPsP6R9EqpZU+LWDhruLrtNIVzjt6X8565vLudNAzD6DHTVii4ipkv+7f/S/3c1yIWeIXUzKdgGMZUZtoKhXID04/vaPaFgvkUDMOYykxbodCobLYfkrpgZmQ+sjwFwzCmMtN2hHOaQNbEP0tTsIxmwzCmMtNeKGSZg8ZjQsHXFEwoGIYxdZm2QsGRXHbTUS770UfmUzAMY3ow7YVC1rq9ExnmI7eUp2EYxlSk5yOciBwgIt8VkbtFezoRNAAACcdJREFU5C4ReWvY/n4ReVRE1od/5/WiP1kz/4kM85FpCoZhTGX6kdE8AbxDVX8mInOB20TkhvCzj6jqP/WyMxmKQszR7CevzZtpi78bhjF16blQUNXHgMfC7R0icg+wotf9cGQ6mj2fwvyZQ8wdKfHy41cwZ2TaVgYxDGMa0NcRTkRWAccBPwVOBt4sIq8B1hFoE0+lHHMJcAnAypUr2+5DIcvR7GkKw6UC69/3AjMdGYYx5emb11RE5gDXAH+iqtuBjwMHA2sINIkPpR2nqleo6lpVXbt06dK2+5EmFH69dTdP7Bytvi8VxASCYRjTgr5oCiIyRCAQvqiqXwdQ1U3e558CvtmLvqQFE536j9+NvbeENcMwpgv9iD4S4DPAPar6Ya/dLzn6cuDOXvQnqSlUUspfZIWtGoZhTDX6oSmcDPw+cIeIrA/b/hy4UETWAApsAN7Yi84khcL2veO9uKxhGMakpB/RRz8C0qbe1/W6LxCEpI6XK+wZLzNvxhBbd43V7DNjuNiHnhmGYfSeaZ+eW6kof3zVzznm/dcDsHVnJBSWzBnm+392OvNmWG6CYRjTg2kvFMbKyrfvejzYnqjw5K4o6mjmcJEDF8/uV9cMwzB6zrQVCvPDzGQ/SW3b7jGe8DQFWzvBMIzpxrQd9W58+2mccuiSmFB4avc492/eWX1fabA6m2EYxlRj2gqFpXNHeOaK+TGh8NATu7jqp7+uvl88ezjtUMMwjCnLtBUKEJTBHi9H2sCl/3EbY+UKLzomSJlYZf4EwzCmGdNaKAyX0m//yH3nAnD0ivm97I5hGEbfmdYlP7OK4b3h1NXMnzXMhc8+oMc9MgzD6C/TWig8vac2e/nfLz6BkVKR3z/xwD70yDAMo79Ma6Gwecfe6vbs4SLX/NFzOWLfeX3skWEYRn+Z1kJhbCKIPDp+5QI+c9GzWWjRRoZhTHOmtVD4ixcfxYGLZ/H2sw+39RIMwzCY5kJh2bwZ/NkLj+h3NwzDMCYN0zok1TAMw4hjQsEwDMOoYkLBMAzDqGJCwTAMw6gy6YSCiJwjIveKyP0i8u5+98cwDGM6MamEgogUgX8DzgWOIli3+aj+9sowDGP6MKmEAnACcL+qPqiqY8CXgJf2uU+GYRjThskmFFYAj3jvN4ZtVUTkEhFZJyLrtmzZ0tPOGYZhTHUGLnlNVa8ArgAQkS0i8nAbp1sCPNGRjg0Ods/TA7vn6UGr95xZ8XOyCYVHAb9e9f5hWyqqurSdi4nIOlVd2845Bg275+mB3fP0oBv3PNnMR7cCh4rIQSIyDLwK+Eaf+2QYhjFtmFSagqpOiMibgf8FisBnVfWuPnfLMAxj2jCphAKAql4HXNejy13Ro+tMJuyepwd2z9ODjt+zqGrjvQzDMIxpwWTzKRiGYRh9xISCYRiGUWVaCoWpWl9JRD4rIptF5E6vbZGI3CAi94WvC8N2EZGPhc/gdhE5vn89bx0ROUBEvisid4vIXSLy1rB9yt63iMwQkVtE5BfhPX8gbD9IRH4a3tuXwwg+RGQkfH9/+Pmqfva/HUSkKCI/F5Fvhu+n9D2LyAYRuUNE1ovIurCtq9/taScUpnh9pc8D5yTa3g3cpKqHAjeF7yG4/0PDv0uAj/eoj51mAniHqh4FnAi8Kfx/TuX7HgXOVNVjgTXAOSJyIvD3wEdU9RDgKeDicP+LgafC9o+E+w0qbwXu8d5Ph3s+Q1XXePkI3f1uq+q0+gNOAv7Xe38ZcFm/+9XB+1sF3Om9vxdYHm4vB+4Ntz8JXJi23yD/AdcCZ0+X+wZmAT8DnkOQ2VoK26vfc4IQ75PC7VK4n/S77y3c6/7hIHgm8E1ApsE9bwCWJNq6+t2edpoCOeorTTGWqepj4fbjwLJwe8o9h9BEcBzwU6b4fYdmlPXAZuAG4AFgm6pOhLv491W95/Dzp4HFve1xR/go8E6gEr5fzNS/ZwWuF5HbROSSsK2r3+1Jl6dgdA9VVRGZkjHIIjIHuAb4E1XdLiLVz6bifatqGVgjIguA/wSO6HOXuoqIvBjYrKq3icjp/e5PD3meqj4qIvsAN4jIL/0Pu/Hdno6aQlP1laYAm0RkOUD4ujlsnzLPQUSGCATCF1X162HzlL9vAFXdBnyXwHSyQETcRM+/r+o9h5/PB7b2uKvtcjLwEhHZQFBS/0zgn5na94yqPhq+biYQ/ifQ5e/2dBQK062+0jeAi8Ltiwhs7q79NWHEwonA055KOjBIoBJ8BrhHVT/sfTRl71tEloYaAiIyk8CHcg+BcLgg3C15z+5ZXAB8R0Oj86Cgqpep6v6quorgN/sdVX01U/ieRWS2iMx128ALgDvp9ne7346UPjlvzgN+RWCHfU+/+9PB+7oaeAwYJ7AnXkxgR70JuA+4EVgU7isEUVgPAHcAa/vd/xbv+XkEdtfbgfXh33lT+b6BY4Cfh/d8J/CXYftq4BbgfuCrwEjYPiN8f3/4+ep+30Ob93868M2pfs/hvf0i/LvLjVXd/m5bmQvDMAyjynQ0HxmGYRgZmFAwDMMwqphQMAzDMKqYUDAMwzCqmFAwDMMwqphQMAxARMphJUr3V7d6rohcKiKv6cB1N4jIknbPYxidwkJSDQMQkZ2qOqcP191AEE/+RK+vbRhpmKZgGHUIZ/L/ENa0v0VEDgnb3y8ifxpuv0WC9RxuF5EvhW2LROS/wrafiMgxYftiEblegnUQPk2QcOSu9XvhNdaLyCfDondFEfm8iNwZ9uFtfXgMxjTChIJhBMxMmI9e6X32tKo+E/hXgkqdSd4NHKeqxwCXhm0fAH4etv058IWw/X3Aj1T1GQS1bFYCiMiRwCuBk1V1DVAGXk2wXsIKVT067MPnOnjPhlGDVUk1jIA94WCcxtXe60dSPr8d+KKI/BfwX2Hb84DfBlDV74QawjzgVOC3wvb/EZGnwv3PAp4F3BpWeJ1JUOjsv4HVIvIvwP8A17d+i4bRGNMUDKMxmrHteBFBzZnjCQb1ViZbAlypwQpba1T1cFV9v6o+BRwLfI9AC/l0C+c2jNyYUDCMxrzSe73Z/0BECsABqvpd4F0EJZrnAD8kMP8Q1v9/QlW3Az8AfjdsPxdYGJ7qJuCCsG6+80kcGEYmFVT1GuC9BILHMLqGmY8MI2BmuJKZ49uq6sJSF4rI7QRrI1+YOK4I/IeIzCeY7X9MVbeJyPuBz4bH7SYqdfwB4GoRuQv4MfBrAFW9W0TeS7DKVoGg0u2bgD3A58I2CJaPNYyuYSGphlEHCxk1phtmPjIMwzCqmKZgGIZhVDFNwTAMw6hiQsEwDMOoYkLBMAzDqGJCwTAMw6hiQsEwDMOo8v8BK/drVvRTnLIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "iteration_list = list(range(len(return_list)))\n",
    "plt.plot(iteration_list, return_list)\n",
    "plt.xlabel('Episodes')\n",
    "plt.ylabel('Returns')\n",
    "plt.title('GAIL on {}'.format(env_name))\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第15章-模仿学习.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}