第19章-目标导向的强化学习.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 6862,
     "status": "ok",
     "timestamp": 1650010812842,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "98nP9Uh9GUTL"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import random\n",
    "from tqdm import tqdm\n",
    "import collections\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "class WorldEnv:\n",
    "    def __init__(self):\n",
    "        self.distance_threshold = 0.15\n",
    "        self.action_bound = 1\n",
    "\n",
    "    def reset(self):  # 重置环境\n",
    "        # 生成一个目标状态, 坐标范围是[3.5～4.5, 3.5～4.5]\n",
    "        self.goal = np.array(\n",
    "            [4 + random.uniform(-0.5, 0.5), 4 + random.uniform(-0.5, 0.5)])\n",
    "        self.state = np.array([0, 0])  # 初始状态\n",
    "        self.count = 0\n",
    "        return np.hstack((self.state, self.goal))\n",
    "\n",
    "    def step(self, action):\n",
    "        action = np.clip(action, -self.action_bound, self.action_bound)\n",
    "        x = max(0, min(5, self.state[0] + action[0]))\n",
    "        y = max(0, min(5, self.state[1] + action[1]))\n",
    "        self.state = np.array([x, y])\n",
    "        self.count += 1\n",
    "\n",
    "        dis = np.sqrt(np.sum(np.square(self.state - self.goal)))\n",
    "        reward = -1.0 if dis > self.distance_threshold else 0\n",
    "        if dis <= self.distance_threshold or self.count == 50:\n",
    "            done = True\n",
    "        else:\n",
    "            done = False\n",
    "\n",
    "        return np.hstack((self.state, self.goal)), reward, done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1650010812843,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "hhrV6UDwGUTP"
   },
   "outputs": [],
   "source": [
    "class PolicyNet(torch.nn.Module):\n",
    "    def __init__(self, state_dim, hidden_dim, action_dim, action_bound):\n",
    "        super(PolicyNet, self).__init__()\n",
    "        self.fc1 = torch.nn.Linear(state_dim, hidden_dim)\n",
    "        self.fc2 = torch.nn.Linear(hidden_dim, hidden_dim)\n",
    "        self.fc3 = torch.nn.Linear(hidden_dim, action_dim)\n",
    "        self.action_bound = action_bound  # action_bound是环境可以接受的动作最大值\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc2(F.relu(self.fc1(x))))\n",
    "        return torch.tanh(self.fc3(x)) * self.action_bound\n",
    "\n",
    "\n",
    "class QValueNet(torch.nn.Module):\n",
    "    def __init__(self, state_dim, hidden_dim, action_dim):\n",
    "        super(QValueNet, self).__init__()\n",
    "        self.fc1 = torch.nn.Linear(state_dim + action_dim, hidden_dim)\n",
    "        self.fc2 = torch.nn.Linear(hidden_dim, hidden_dim)\n",
    "        self.fc3 = torch.nn.Linear(hidden_dim, 1)\n",
    "\n",
    "    def forward(self, x, a):\n",
    "        cat = torch.cat([x, a], dim=1)  # 拼接状态和动作\n",
    "        x = F.relu(self.fc2(F.relu(self.fc1(cat))))\n",
    "        return self.fc3(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1650010819329,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "bxiqOl_vGUTR"
   },
   "outputs": [],
   "source": [
    "class DDPG:\n",
    "    ''' DDPG算法 '''\n",
    "    def __init__(self, state_dim, hidden_dim, action_dim, action_bound,\n",
    "                 actor_lr, critic_lr, sigma, tau, gamma, device):\n",
    "        self.action_dim = action_dim\n",
    "        self.actor = PolicyNet(state_dim, hidden_dim, action_dim,\n",
    "                               action_bound).to(device)\n",
    "        self.critic = QValueNet(state_dim, hidden_dim, action_dim).to(device)\n",
    "        self.target_actor = PolicyNet(state_dim, hidden_dim, action_dim,\n",
    "                                      action_bound).to(device)\n",
    "        self.target_critic = QValueNet(state_dim, hidden_dim,\n",
    "                                       action_dim).to(device)\n",
    "        # 初始化目标价值网络并使其参数和价值网络一样\n",
    "        self.target_critic.load_state_dict(self.critic.state_dict())\n",
    "        # 初始化目标策略网络并使其参数和策略网络一样\n",
    "        self.target_actor.load_state_dict(self.actor.state_dict())\n",
    "        self.actor_optimizer = torch.optim.Adam(self.actor.parameters(),\n",
    "                                                lr=actor_lr)\n",
    "        self.critic_optimizer = torch.optim.Adam(self.critic.parameters(),\n",
    "                                                 lr=critic_lr)\n",
    "        self.gamma = gamma\n",
    "        self.sigma = sigma  # 高斯噪声的标准差,均值直接设为0\n",
    "        self.tau = tau  # 目标网络软更新参数\n",
    "        self.action_bound = action_bound\n",
    "        self.device = device\n",
    "\n",
    "    def take_action(self, state):\n",
    "        state = torch.tensor([state], dtype=torch.float).to(self.device)\n",
    "        action = self.actor(state).detach().cpu().numpy()[0]\n",
    "        # 给动作添加噪声，增加探索\n",
    "        action = action + self.sigma * np.random.randn(self.action_dim)\n",
    "        return action\n",
    "\n",
    "    def soft_update(self, net, target_net):\n",
    "        for param_target, param in zip(target_net.parameters(),\n",
    "                                       net.parameters()):\n",
    "            param_target.data.copy_(param_target.data * (1.0 - self.tau) +\n",
    "                                    param.data * self.tau)\n",
    "\n",
    "    def update(self, transition_dict):\n",
    "        states = torch.tensor(transition_dict['states'],\n",
    "                              dtype=torch.float).to(self.device)\n",
    "        actions = torch.tensor(transition_dict['actions'],\n",
    "                               dtype=torch.float).to(self.device)\n",
    "        rewards = torch.tensor(transition_dict['rewards'],\n",
    "                               dtype=torch.float).view(-1, 1).to(self.device)\n",
    "        next_states = torch.tensor(transition_dict['next_states'],\n",
    "                                   dtype=torch.float).to(self.device)\n",
    "        dones = torch.tensor(transition_dict['dones'],\n",
    "                             dtype=torch.float).view(-1, 1).to(self.device)\n",
    "\n",
    "        next_q_values = self.target_critic(next_states,\n",
    "                                           self.target_actor(next_states))\n",
    "        q_targets = rewards + self.gamma * next_q_values * (1 - dones)\n",
    "        # MSE损失函数\n",
    "        critic_loss = torch.mean(\n",
    "            F.mse_loss(self.critic(states, actions), q_targets))\n",
    "        self.critic_optimizer.zero_grad()\n",
    "        critic_loss.backward()\n",
    "        self.critic_optimizer.step()\n",
    "\n",
    "        # 策略网络就是为了使Q值最大化\n",
    "        actor_loss = -torch.mean(self.critic(states, self.actor(states)))\n",
    "        self.actor_optimizer.zero_grad()\n",
    "        actor_loss.backward()\n",
    "        self.actor_optimizer.step()\n",
    "\n",
    "        self.soft_update(self.actor, self.target_actor)  # 软更新策略网络\n",
    "        self.soft_update(self.critic, self.target_critic)  # 软更新价值网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "executionInfo": {
     "elapsed": 303,
     "status": "ok",
     "timestamp": 1650010821234,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "aw60NZwLGUTS"
   },
   "outputs": [],
   "source": [
    "class Trajectory:\n",
    "    ''' 用来记录一条完整轨迹 '''\n",
    "    def __init__(self, init_state):\n",
    "        self.states = [init_state]\n",
    "        self.actions = []\n",
    "        self.rewards = []\n",
    "        self.dones = []\n",
    "        self.length = 0\n",
    "\n",
    "    def store_step(self, action, state, reward, done):\n",
    "        self.actions.append(action)\n",
    "        self.states.append(state)\n",
    "        self.rewards.append(reward)\n",
    "        self.dones.append(done)\n",
    "        self.length += 1\n",
    "\n",
    "\n",
    "class ReplayBuffer_Trajectory:\n",
    "    ''' 存储轨迹的经验回放池 '''\n",
    "    def __init__(self, capacity):\n",
    "        self.buffer = collections.deque(maxlen=capacity)\n",
    "\n",
    "    def add_trajectory(self, trajectory):\n",
    "        self.buffer.append(trajectory)\n",
    "\n",
    "    def size(self):\n",
    "        return len(self.buffer)\n",
    "\n",
    "    def sample(self, batch_size, use_her, dis_threshold=0.15, her_ratio=0.8):\n",
    "        batch = dict(states=[],\n",
    "                     actions=[],\n",
    "                     next_states=[],\n",
    "                     rewards=[],\n",
    "                     dones=[])\n",
    "        for _ in range(batch_size):\n",
    "            traj = random.sample(self.buffer, 1)[0]\n",
    "            step_state = np.random.randint(traj.length)\n",
    "            state = traj.states[step_state]\n",
    "            next_state = traj.states[step_state + 1]\n",
    "            action = traj.actions[step_state]\n",
    "            reward = traj.rewards[step_state]\n",
    "            done = traj.dones[step_state]\n",
    "\n",
    "            if use_her and np.random.uniform() <= her_ratio:\n",
    "                step_goal = np.random.randint(step_state + 1, traj.length + 1)\n",
    "                goal = traj.states[step_goal][:2]  # 使用HER算法的future方案设置目标\n",
    "                dis = np.sqrt(np.sum(np.square(next_state[:2] - goal)))\n",
    "                reward = -1.0 if dis > dis_threshold else 0\n",
    "                done = False if dis > dis_threshold else True\n",
    "                state = np.hstack((state[:2], goal))\n",
    "                next_state = np.hstack((next_state[:2], goal))\n",
    "\n",
    "            batch['states'].append(state)\n",
    "            batch['next_states'].append(next_state)\n",
    "            batch['actions'].append(action)\n",
    "            batch['rewards'].append(reward)\n",
    "            batch['dones'].append(done)\n",
    "\n",
    "        batch['states'] = np.array(batch['states'])\n",
    "        batch['next_states'] = np.array(batch['next_states'])\n",
    "        batch['actions'] = np.array(batch['actions'])\n",
    "        return batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 506
    },
    "executionInfo": {
     "elapsed": 890109,
     "status": "ok",
     "timestamp": 1650011748151,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "0wLycs-3GUTT",
    "outputId": "a73c3a0d-d7ac-486b-87aa-b141aa9d4d0c"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration 0:   0%|          | 0/200 [00:00<?, ?it/s]/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:23: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)\n",
      "Iteration 0: 100%|██████████| 200/200 [00:08<00:00, 24.96it/s, episode=200, return=-50.000]\n",
      "Iteration 1: 100%|██████████| 200/200 [01:41<00:00,  1.96it/s, episode=400, return=-4.400]\n",
      "Iteration 2: 100%|██████████| 200/200 [01:37<00:00,  2.06it/s, episode=600, return=-4.000]\n",
      "Iteration 3: 100%|██████████| 200/200 [01:36<00:00,  2.07it/s, episode=800, return=-4.100]\n",
      "Iteration 4: 100%|██████████| 200/200 [01:35<00:00,  2.09it/s, episode=1000, return=-4.500]\n",
      "Iteration 5: 100%|██████████| 200/200 [01:34<00:00,  2.11it/s, episode=1200, return=-4.500]\n",
      "Iteration 6: 100%|██████████| 200/200 [01:36<00:00,  2.08it/s, episode=1400, return=-4.600]\n",
      "Iteration 7: 100%|██████████| 200/200 [01:35<00:00,  2.09it/s, episode=1600, return=-4.100]\n",
      "Iteration 8: 100%|██████████| 200/200 [01:35<00:00,  2.09it/s, episode=1800, return=-4.300]\n",
      "Iteration 9: 100%|██████████| 200/200 [01:35<00:00,  2.09it/s, episode=2000, return=-3.600]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEWCAYAAACNJFuYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZgcZbn38e+dfZkkk8m+T3YSQhKSIQlLEmLCTgwCAqLsgnjgAB4URTyIC6+Ioh6PHgQXFvWAHBXZEYIKyCIECIGELUAgOwnZ9+1+/6jqmequ7pnumeklM7/Pdc013U9VV91V1V131fNUPWXujoiISFSLYgcgIiKlR8lBRERilBxERCRGyUFERGKUHEREJEbJQUREYpQcpEkws6lm9lYtwyvNzM2sVSHj2t+Y2RYzG5Jh2Llm9s8CxXG7mX23luFuZsMKEUtzpeQgAJjZEjPbbmabzWyDmT1rZhebWYvIOLeb2a5wnM1m9rqZfc/MukTGOdfM9oY7mU1mNt/MTowM72RmPwrnt9XMPjSzP5rZ5IbE7+5Pu/vIlOWZVd/ppds5pSaYyPrYEvl7NWXcRPkSM/tafeOpR/z1Ws/uXubu72Ux/VvM7ObI+9bhfNKVTWnY0kgxKDlI1Gx37wQMAm4Avgr8OmWcG8NxegDnAVOAZ8ysY2Sc59y9DCgPP3+PmXU1s7bA34CDgBOBzsAo4G7guPwtVl7dGO5QE3/jUoaXh+viVOA/zeyofAdUn/VcjzOqp4BpkfdVwIfA1JQygJdymbCZtcwxFskDJQeJcfeN7n4/cDpwjpmNSTPODnd/Efgk0I0gUaSOsw/4DdAeGAqcBfQHTnL31919r7tvdfc/uvt16WIxszvM7Mrwdb/waPyS8P1QM1tnZi3M7EgzWxaW/xYYCDwQHrVfFZnkZ8Oj6LVmdk09V1HO3H0esBAYn2kcMzvMzF40s43h/8Miw/5hZt8xs2fCs7bHzKx7hklltZ4T69LM3gHeiZQNC193M7P7wzPAFwi2YcJTwKhIDFMJkk/HlLLn3H23mY0Kl2GDmS00s09G4rjdzG42s4fNbCswI826+YqZrTSzFWZ2fqZ1KI1HyUEycvcXgGUkHw2mjrMZeDzdOOHR6OeBLQQ7n1nAX919aw5hPAkcGb6eDrxHzRHrdODpMAlFYzqL4Ch2dng0f2Nk8BHASGAmcK2ZjcohlnoLq1bGAIszDK8AHgJ+SpBsfwQ8ZGbdIqOdSZCEewJtgC9nmF0u6/kkYDIwOs2wnwM7gD7A+eEfAO6+FPiAmu0+DXgaeDal7Ckzaw08ADwWxv7vwO/NrLoaMFy264FOQFK7hpkdGy7rUcDwcPkkz5QcpC4rgIocx5liZhuAVcBngE+5+0age1gGgJmND48kN1nmxuQngSPCto9pwI3A4eGw6eHwXHzL3be7+6vAq0BqNVDUl8P4NoTLs6CucczsjpTha81sO/Ac8D/AXzLM6wTgHXf/rbvvcfe7gDeB2ZFxbnP3t919O3APmc9CclnP33P3deE0iXymJXAKcG141vE6kLpsTwLTwm0zCXieIEEkyg4Px5kClAE3uPsud/8b8CDBdyPhPnd/xt33ufuOlPmcFi7762HCuy7DcksjUnKQuvQD1uU4zvPuXu7u3d19irvPDcs/JjgKBcDd57t7OXAy0DbdhN39XWArwY5wKsFOZUV41Fmf5LAq8nobwU4rkx+Gy1Eexjm2rnHc/ZyU4d3DeVxJcAbUOsO8+hIciUd9QLBuc409l/W8NMM0egCtUoanxpdodzgIeM/dtxEc9SfK2gP/Ili2pSlneKnLlikOEp+vJQ7JAyUHycjMDiH4AWe8fNHMyghO85/OYpJPAEenNF5n40mCBt027r48fH8O0BWYn+EzJdPdcFjn/yOCKpp/yzDaCoILAaIGAsvrMctc1nOm9bQG2AMMSIkn6imCM68TqNn+C8PPnAC8GJ4FrAAGWOTKN+LLVtv2WllHHJIHSg4SY2adLbj89G7gd+7+Wppx2prZRIJqkvXAbVlM+k6CH/q9ZjbGzFqaWTtqrmrJ5EngUoKdEcA/wvf/dPe9GT6zGkh7vX4R3QBcFS5zqoeBEWZ2ppm1MrPTCdoBHqzHfOq7nquF6/XPwHVm1sHMRhMk5Og4iwnW8+WEycGDZwD8KyxLbK9/EZzpXBVe3nokQXXZ3VmGcw9wrpmNNrMOwDezXQ6pPyUHiXrAzDYTnMJfQ9AomnoV0lXhOB8T7IReAg7LpvEzPIqcASwiaHzdBLwFHEJQr5zJkwQNlYmdzT+BDpH36XwP+EZY156p4bYxXGXJ9zmsrWXchwgS6YWpA9z9Y4LLTq8kWLdXASe6e23TS6sB6znVpQRVV6uA20l/APAUQRXUM5Gypwkanp8K49lFkAyOA9YStL2c7e5vZrk8jwA/Ibg8d3H4X/LM9LAfERFJpTMHERGJUXIQEZEYJQcREYlRchARkZgm0X1x9+7dvbKysthhiIjsV1566aW17t4j3bAmkRwqKyuZN29escMQEdmvmFnGu81VrSQiIjFKDiIiEqPkICIiMUoOIiISo+QgIiIxSg4iIhKj5CAiIjFN4j6H/dmStVu595XlnDC2D0N7lPGnl5ZxysT+PLhgBUN7lLFh2242bt/NoG4dWLNlJzNG9gTgmcVr6dK+Nbc9s4TLZg6jQ5tWvPzhej78eBvXP/wGp1cNYMvOPfTu0o5hPctYuWE7DyxYycXTh7B3H+zcs5ePNu/kgVdXcEhlBZ3atWLuotV0aNuK8QPKad2yBT06taVzu1Z8dvIgdu7Zy/cffZPJg7tx0sH9Ysvx+TtepEObVhw5sgfHHNib9dt2cc+8ZfTv2p7hPcv425sfsXPPPiZVVrBnn/PO6s3MHteXy/8wn3+fMYwBFR1Yu2Unyzdsp2uHNry2bANfPHIYF9zxIsN6lvHtOWMA2LF7Lw+8uoIhPcpo17oF67fu5v89/AYXThvMlp17+cebHzFhUFfatW7JBUcMZsnarSxdvw13qOzWkYHdOnDvK8u4+4WlXDZzOL948l127N7LpMEVnDWlkkdfX4mZsfijLVw2czg3PfYWG7btZsKgcrbt2svIXp3468JV7Nnn3HDKWH773AdMGVLB4o+28PryjWzdtZczJw/koQUrmTmqJ4tWbGLTjj1s2r6bLx01gr+/+REzR/Xk8UWrOaSygr8uXMXQHmX06NSW3z3/AfM+WM9xY3ozrGcZ767ZSttWLVizeScDKzrQp0s7Jg2uoFtZW5au28bMm55k19593HhqzQPqWrc0PjmuH7c89S7zlqynV+e2PPnWGo4+sDe9u7TjmcVrOWxodzbv2E1lt450aNuSe+YtY9rw7swc1YtXl27go807qKqs4KNNOwHYunMPJ0/ox4JlG3nizY/o3K4Vn64aQJf2rdm9dx9f//NrtDCjV5d2TBvenW279vL3tz7itmeW8OgVU/nXe+vYsXsvZ0wayJ69+7jp8bfZuH03kwdXcPahlfz874up7NaRyu4d+OvC1WzesZsendoyvGcnBlS0Z8uOPVRVVvDyh+vZsXsvi1Zs4tl3P2bGAT2p6NCGe19ZRp8u7Rneq4zHF61m3dZd/OT08by9eguLP9rCI6+vpHP71pw6oT+d27emc/tWPPHGRyxbv40vTB/KA6+u4LzDBvOTJ95mypBuTBncjdk/+yfDepYxrEcZQ3p05K3Vm2nTsgXPvvsxEwaWs3ufU96+NcN7lrFh+24mDurKP99ZywF9OrF1517+9f46du/Zx6WfGMZtzyzhuk+OplO71uzcs5f75q9g3z7nlqfe4/Bh3fj0xOA5RmP7d+HPLy9nZO9O3PbMEiYPruDYg3pT1qYVd734IVt37mH5+u20aGGcd9hg/vO+13lr1WbmXjmdsraNvytvEl12V1VV+f56E1zl1x6qfv3tOQdy7X0LOa2qP/fMW5Z2/CU3nBD7HMABvTvx5qrNeYnxc1MGsnTddp58ew0A7/2/42nRwqqHv7FyE8f9V82D4E6Z0J8n3lzNhm27GzTfkyf0488vBw8Le/Zrn6BveXu+9/Ab3PLUe1l9/q3vHsvIbzyaVLbkhhNi6y5h2ogePBUuYzaOHt2Lxxatznr8ARXtWbpuO/3K27N8w/a6P5DG+AHl/OWSwzMuA8DpVQP4w7zanrqZu3u+cCin3fJc9fuZB/Tk1+cews//vpgf/DXT47+TfeWYkWzYtotfPv1+ddlzV3+CQ79X9+MZattu6bRt1YKde/bVPWKBnDC2Dz8/cwLfe+QNbnky/ff3rgun8JlfPp9UdvxBvTl6dG+u+EOmBx4G39s7z59Ur7jM7CV3T/sQKFUrlZB1W3cB1GvH8cHH2xo7nGorN+xg5caamFJ/dLv3Jr9ftWl7gxMDwNJ1Ncu0d19wELN6U+qz5zPbsTu3ncP7a7fkNP6H63Jb50vXBeuwvokhmEbd81yxsf7Tz2TH7uQH7iWWYcO2XVlPY83mnXy8NXn8PXvzc3BaSokBYNXG4Hu7fH3mbbNpR/w3s3rTTtbXsY7f/Si37222lBykTrv3Oa1a1HxVUncU0WEAhtEYoj/wxJmKWfbT3rk70xNE09uX4/5k6649OY3fqkXD10sOi9+oWrVMP+NWLbPfhezd57Rt1Tx3OYmDm121JK10tTgtre5f0849uX3Ps6U2h/3M+2u3snVnfKe0PccdYS5Sq1q+89AiDh3SjfIObViydmts3v9cnPOTLdNasGxj9es7nl3CgIoO3PvK8lo+kezHc9+Jlf32+YxdyeR8RJ84E8jWnn0NP0peu2UX/1dHldHT7zTO+o/6Xcp6e3PVZn719Hv89fVVWU/jt89/QL/y9kllN2ZZJfWrp7OrSixV85du4FdPv1drNWS6quQXlqxj2frazxa378rPb19tDkUWrUe9YtZwfjL3HQ4f1o1nFn9cxKhEZH+RaIeqj9raHHTmIFJiupe1Ye2W7Ovypfn6+ZkTqKrsmpdpN88KQJESdlC/LsUOodnr2qF1rcMbo/2oMZwwtg+9OrfLy7SVHERKTCM0TUgD1bUNGqP9qNQpOYiUmH1NoB1wf1dXW+yYfp0LFEnxKDlITl74+syk91WDauo7T5nQPzb+4O4dq1/fdeEUzju8stbpP3rFVL567AHV728795DYOD+I3BH8zdmjq1/PGBl/2uEvz67ify+cHCtfcN3RSe8PHlie9P7wYd2S3n/rkwdWv/7fCyfzmUkDq9+P6lOzo/jvzxzMF6YNoW+X9Kf6s0b1Slt+zfGj0pbfcf4kvn58zfo4Ylj3pOHTR/Tg95+PLx/ArWdN5P5Laxoqv3LMyKThc8b35cSxfdJ+NuHq4w7ghpMPSiob2asTQF4vS/3eyQfx9FUzah3nzMkDY2XPXz2TH356HP/9mYMbNP+60vPvLpjMI5dPbdA8cnHh1MFpf1/5pOTQTAzt0bHukerQuV0rekbqNzu2CbqoSDj9kAGxz0R3PsN7lfHN2QfGxoka3L0jXzxyaPX7QwZXxMaZPa5v9evzDq+Z/2lVwfwP7BvsrDu1bcVRo3tx2NDkHWqwLMl1yocNTU4GI3slHxkec2DvyLjduXzmcCCom47uJGaP68vVx4/i+pQdas1849eATB/RgwunDal+Hz1zmD6iBxdNq1kfN5ySPN3LZg7n8GHx5QOYcUBPBlXUbPeLpw9NGn7okG4MqOhQ/T6R6M89rLI6zpMO7scZk5J3wokG0KsiSbyxfWbSQPp3Tb7stVNKFxEDunYgVe8u7Th1Yv+k70h97Kul2qh7WVvKO7RJOijIZOKg9I3FkyqTv9flHVpTXks7x+emDOKm08Yxa1TPOufZWJQcmokWjXD3VOrPpU2rFuyO/Ihapmmki56dt2vdss55tE65oa5lmrjragxMF0dd6qrJadc6Oa7ELFq2yO0ntDfdjU4p8dYWSy43AbZqYVgkvNTVYpaciGpuNKypU2/XKr7NEvHmu0k2dVl3p9ylmM8bAhvrvqFMX8XU2I2aG+XSSWymXLZ/Qyk5NDPZVgWk2wF37dAm6X3PTu1oHRkv3U65fZuWVHQMPtcui3m3SJlGun1vpp1/m3D6ndIcndelrvbFtml2khBcdpqLjmk6SOvSPvmIsbZO1HLJe6k7knQ7lvaRhJ24QqesbSvKw5jahkmxTWTbJQ400nX3kE+p66ljm7oPNnLRq3Pb6teN1ZFduu0NNeswcdDRtWOb2O8rKvGdL+RFUkoOTdzvLpjMdbNHVx+p3HLWRAZ1qzkd/85JQW+nHdu05JazJlaX33H+JK49saY+/ztzDozVbf/63CqOjlS3tDCSpnHlUSO44IjB/OmLh3H9p8bU2tXChVMH8/1IlcldF07htnMPSTpzmDKkgm+cMCq2k3vw34/gv84Yz4yRPfnKMSOT2gcSenaq+eE/esXU6nWT2Om5O/fVciNR+zYtOfvQQdXVST07t+Obs0fzmzRtIrWJtqd8adYILvvEMK4L4/3TFw/jF5+byPdPGcsJY/vw0zT15oZx23nxeT58Wfr679Tqs2gVmGFcPH0oBw8sZ0y/zvz0MwfzH0eN4JIZw7jroil896Qx1Wd7j1w+lcOGduPhy6ZWJ4zVm3byuwsm819njOerxx7A3RdNSarOeexL06pfj+rTmT998bDq92P753a57gkH9eEPFx2aVHbGpIEM61nGvf92GH+55HD+57MTsppW9C7taFXbnedP5j9PHM03Z4/m0SumceVRI6qH/ezMgzk57I043cH7jaeO5c//dlis/AvThsZHJvg+feekMTx2xXS+PedA7jx/UtrtCsG6S1T/RQ+M7r5oSi1L2XC6Ca4JGNu/S1JXE1FHDO/OEcO7c9cLQZcLvbu047zDKrnugUWcc+ig6rrlGQf05NBIvfvhw7pz+LDufPvBRQCcdWhlbNr9yttjZhzYtzMLV2zCsKS6+X8Pd6SDu3dMaphO58qjRyZVOyViiV41cnfKziFhTL8ujAnvDbhkxjDWbN4ZG2faiB788aVl3HjqWA7o3bl63Vx51Ai+98ibODBuQHnsc1GJbsMTou0d2Yq2OVw+a3jSsGj99M/PTL+jM4MZI3ty8MByXvlwQ3X56L7J9d/pGuch2NGcOrE/f3xpGVhQ1Xfvv9UkxcvCbTaoW0cGdavZZkN7lPG/FwY7o1eWrq8uP2J4cnvH6k07eODVFQCM6NUJs6BK5ObPTqAy8h04alSv2Hf2rCmDMnZvcubkgUmfB2jdsgVz/2N69fvxdWy/hCeunM4R3/8ba7fs4oIjBvOLJ98Nl7kDI3vXbNPPTx3CTY+/DcCJY/syrn85f87QfctpVQPYnOZMqm3rFiy54QSuf2hRUm+07s5ZUwYBcHb424r2u5TowRdIatezpIOl5HayxqYzhyYgl/aEWuuzc5xv4otaUx+a4wQiMi1DfepYczn1Tsy3tgbIxtTQOuPGqHJu6JWyNV3BZT+h1KrAXXvjHdDV1lbUmLUpmb5rqVWuqVWaLepoa6ntd5hmcWOiyx/tbi+6Whqj7TBbSg5NQDY7w8R3KnXHEN1Z1XfHlWjUbMj3tj6NyJnk8gNKjLq/3NPUGD3eerhTr++UMn2X0s4rw4FDut5JW2fo+TWYQJbBZSHTdy31+5/6PaorhNq+dqn3rqT7rSUngZrX0XgL2eagaqUmIJud+g8/PY4fPvYWw3qW8cL76Tv1SzeVId078v7HW5PKbjv3EB5csDLN54MpfPXYA6itJ+cvTBtCq5bG/KUbOGtKJQ8sWFHrl/7kg/tx4rjk6/FPq+rPzAz3DKQ2agNcPnM4y9Zv49gxvZPKT53Yn7lvrOai8FLSb84ezeYde6qfR9G9rE31sFxMGdyNI4Z156pjR3LDI2+yd59z/EG131OQjdo29SUzhtK1Qxv+8dYarjlhdMbxvjRrBCs2bI+ti1ylSw6JbZJ6TX7qDvmCIwbz2vKNXHPCKK6851VatTQ+P3UIHdu2ijU8p/Prc9L2FZfkc1MGMrJXJx56bSUnjO3LLU++y7gB5bQw+MXnJnLLU+9R0bENvzy7iifeiPeWGksOlvy/rvGh5jd18fSh3P7sklrjNTOOG9ObuW+s5qbTxnPKzc9Wlyd8+eiR3Dd/Ra3TaSxKDk1ANgcTY/p14fbzkp8W5Smv033p//blI2NlMw7oyYwDaq63TuwkEqfh0fsU0rk65YavunZSPzp9fKzsxlPHZRw/3XIMqOiQts2ivEObpPJEO8K3HwjaWi6ePpTPT809ObRv05LfhQ34ibr6xlDbWdFXjgkau+uKN9O6yFZNpVI8O5S1bVX9tMKo1EuSe3ZuV71eHr2ipuH6ilkjksabNLiCF95fFztjynRgEPXdk4ILHBLtZYk6foCqygqqwnsNjhrdi6NGx6cXu/Q3h9OXcf278OqyjdU79t4ZbopMdfPnggs69kTqoaJxRO9LyTdVKzUBDanOsaTXDaxWyvuV71IKa7g+37d6t7U0QntWfeUaczRx11XjVteUo9NKd69PISg5NAH13SkbyV/iBv5+i/aUsuaklNZxLg3b9W1Tamj7SCGlW8RMcdeZPCz6WslB6indd2dQtw4cneZUOdfpZCNRJ9835SlfxZK4sSv1UtFcJNo4po9If0losSR2FOeH1V9D6rhEOC8xhLu8XNrwEzvOXNtvEtV8w8P+nM4/fHCjXryQi3TVaJ86uB/dy4J7aKI78cTlp9F7inL5PUanNW5A8j0hPTq15aTxDeseJBtqc2gCUnfqBw8sT7p2vdbPJr2u34/utKoB1f0alYLWLVukrffOxYSBXRs8jXxIbOvZ4/o2uP+g+gcR/MvlzCFxkcDXjx/F1zN0MpjO8Qf1SdoO184ezbWzMze2F0L0d/LjSHtYNGfNGd+POeP7JX3u1rOrmLtoNZ+/c15Ov7Q+XZIPul68ZlZO8daXzhyagNSden2vYy+lKgtJrxQ2UX1iKFa9eWOq63dVrOqffFFyaAIa5TvppbHjkdoV8iaoTKpvfsyhYqkU4m4sDVmU/eR2GkDJoUlI1GVOTtO9dTqJroYnDOpaEg1fTcWAiuzbXEZn0d1zOvXZRKnPqmio6hByqlZq1BCKoizs+iTdZa9R2bQt7A8/NbU57OfuvmgKU4Z04/iD+vDhum2c+ovn6vzM5CHdeO7qT9CnS/vqfnCgsHdfNjXzrz0qY8+tqV75z6NoX88eRevTLnTXhVPYtqtxuqCOyuUouClUK3Vu15oXrplJRS29p877xqxYZ4fp7A8P+ytKPjezT5vZQjPbZ2ZVKcOuNrPFZvaWmR1TjPj2J33DxqqendtVN/pl871LbeQCnTk0RHmHNlnv8Lt2bJPVsy3Sqc8mate6ptv0xlDTfUbzq1bq2aldrb0Ldy9rm9S9+f6sWGcOrwMnA7dEC81sNHAGcCDQF5hrZiPcvfEPe5qIhv7mmshvttkohe1VnRxy+Ey6Lk2as1LYjnUpSopz9zfc/a00g+YAd7v7Tnd/H1gMTEoznoSiR6CJ7qCHFuHadymMUrgLPXFdf7rHdKZqjMfTNmfRBxAVWqm1OfQDno+8XxaWxZjZRcBFAAMHxh803lz0iDzEZljPTtx+3iFMHpx7P++5XHkixVMKB+BTh/fgV2dXMT3DMyOi7vnCoby7Zmud4zUXuVTF/emLhxa0L6VUeUsOZjYXSNej2jXufl9Dp+/utwK3AlRVVWnPFjpyZG4PIC+FI1HJXqm0C83K8m7fbmVt6VZWvKPf0lX3dpw4KLurD/Mlb8nB3etzG99yIHqrbf+wTETQvShSOKXWrH4/cIaZtTWzwcBw4IUix9SkJR6wUt+rZ6QwEg+8L5ETB6mn1uGVTvW9lLmQitLmYGafAv4b6AE8ZGbz3f0Yd19oZvcAi4A9wCW6Uim/Zo7qxeUzh1d35Pbdk8ZwSGVxT2cl7r5LD+ef76wtmWql5uK/zhhPv0bsUHL6iB5cMWs45x5W2WjTzJeiJAd3vxe4N8Ow64HrCxtR89WyhfGlo2oesPK5yANRpHQM7VHG0B5lxQ6j2UntPK+hWrSw2AONSlWpVSuJiEgJUHIQEZEYJQcREYlRchARkRglBxERiVFyEBGRGCUHERGJUXIQEZEYJQcREYlRchARkRglBxERiVFyEBGRGCUHERGJUXIQEZEYJQcREYlRchARkRglBxERiVFyEBGRGCUHERGJUXIQEZEYJQcREYlRchARkRglBxERiVFyEBGRGCWHEuKe/F9EpFiUHEREJEbJoYSYJf8XESkWJQcREYlRchARkRglhxKiBmkRKRVKDiIiEqPkUELUIC0ipULJQUREYoqSHMzsB2b2ppktMLN7zaw8MuxqM1tsZm+Z2THFiE9EpLkr1pnD48AYdx8LvA1cDWBmo4EzgAOBY4H/MbOWRYqx4NQgLSKloijJwd0fc/c94dvngf7h6znA3e6+093fBxYDk4oRo4hIc1YKbQ7nA4+Er/sBSyPDloVlMWZ2kZnNM7N5a9asyXOIhaEGaREpFa3yNWEzmwv0TjPoGne/LxznGmAP8Ptcp+/utwK3AlRVVakiRkSkEeUtObj7rNqGm9m5wInATPfqWvblwIDIaP3DMhERKaBiXa10LHAV8El33xYZdD9whpm1NbPBwHDghWLEWAxqkBaRUpG3M4c6/AxoCzxuQQX78+5+sbsvNLN7gEUE1U2XuPveIsUoItJsFSU5uPuwWoZdD1xfwHBKhhqkRaRUlMLVSiIiUmKUHEREJEbJQUREYpQcREQkRslBRERiskoOZna5mXW2wK/N7GUzOzrfwYmISHFke+ZwvrtvAo4GugJnATfkLSoRESmqbJND4sr744HfuvvCSJmIiDQx2SaHl8zsMYLk8Fcz6wTsy19YIiJSTNneIX0BMB54z923mVk34Lz8hSUiIsWUVXJw931mthoYbWbF6o9JREQKJKsdvZl9HzidoEO8REd4DjyVp7hERKSIsj0LOAkY6e478xmMiIiUhmwbpN8DWuczEBERKR3ZnjlsA+ab2RNA9dmDu1+Wl6hERKSosk0O94d/IiLSDNSZHMysJXCuu88oQDwiIlIC6mxzCB/TuVIeQwMAAA7TSURBVM/MuhQgHhERKQHZVittAV4zs8eBrYlCtTmIiDRN2SaHP4d/IiLSDGR7h/Qd+Q5ERERKR7Z3SL9PcEd0Encf0ugRiYhI0WVbrVQVed0O+DRQ0fjhiIhIKcjqDml3/zjyt9zdfwKckOfYRESkSLKtVpoQeduC4ExCvbOKiDRR2e7gb4q83gO8D5zW+OGIiEgpyPphP+7+XrTAzAbnIR4RESkB2fbK+scsy0REpAmo9czBzA4ADgS6mNnJkUGdCa5aEhGRJqiuaqWRwIlAOTA7Ur4ZuDBfQYmISHHVmhzc/T7gPjM71N2fK1BMIiJSZNm2OXxsZk+Y2esAZjbWzL6Rx7hERKSIsk0OvwSuBnYDuPsC4Ix8BSUiIsWVbXLo4O4vpJTtqe9Mzew7ZrbAzOab2WNm1jcsNzP7qZktDodPqGtaIiLS+LJNDmvNbChh53tmdiqwsgHz/YG7j3X38cCDwLVh+XHA8PDvIuDmBsxDRETqKdub4C4BbgUOMLPlBHdIf7a+M3X3TZG3Hanp8XUOcKe7O/C8mZWbWR93b0giEhGRHGX7PIf3gFlm1pHgbGMbQZvDB/WdsZldD5wNbAQSz6fuByyNjLYsLIslBzO7iODsgoEDB9Y3DBERSaPWaiUz62xmV5vZz8zsKIKkcA6wmDr6VjKzuWb2epq/OQDufo27DwB+D1yaa+Dufqu7V7l7VY8ePXL9uIiI1KKuM4ffAuuB5whuersGMOBT7j6/tg+6+6wsY/g98DDwTWA5MCAyrH9YJiIiBVRXchji7gcBmNmvCKp3Brr7jobM1MyGu/s74ds5wJvh6/uBS83sbmAysFHtDSIihVdXctideOHue81sWUMTQ+gGMxsJ7CNot7g4LH8YOJ6g2mobcF4jzEtERHJUV3IYZ2aJK4sMaB++N8DdvXN9Zurup2Qod4Iro0REpIjq6lupZaECERGR0pHtTXAiItKMKDmIiEiMkoOIiMQoOYiISIySg4iIxCg5iIhIjJKDiIjEKDmIiEiMkoOIiMQoOYiISIySQwlxT/4vIlIsSg4iIhKj5FBCzJL/i4gUi5KDiIjEKDmIiEiMkkMJUYO0iJQKJQcREYlRcighapAWkVKh5CAiIjFKDiIiEqPkUELUIC0ipULJQUREYpQcSogapEWkVCg5iIhIjJKDiIjEKDmUEDVIi0ipUHIQEZEYJYcSogZpESkVSg4iIhKj5CAiIjFKDiVEDdIiUiqUHEqQkoOIFFtRk4OZXWlmbmbdw/dmZj81s8VmtsDMJhQzvkJTg7SIlIqiJQczGwAcDXwYKT4OGB7+XQTcXITQRESavWKeOfwYuAqIVqLMAe70wPNAuZn1KUp0IiLNWFGSg5nNAZa7+6spg/oBSyPvl4Vl6aZxkZnNM7N5a9asyVOkhaUGaREpFa3yNWEzmwv0TjPoGuDrBFVK9ebutwK3AlRVVTWp3anTpBZHRPZDeUsO7j4rXbmZHQQMBl61oOW1P/CymU0ClgMDIqP3D8uaheoGadQiLSLFVfBqJXd/zd17unulu1cSVB1NcPdVwP3A2eFVS1OAje6+stAxiog0d3k7c6inh4HjgcXANuC84oYjItI8FT05hGcPidcOXFK8aIqrukFabQ4iUmS6Q7oE6WolESk2JYcSojukRaRUKDmIiEiMkoOIiMQoOZQQ3SEtIqVCyaEEKTeISLEpOZSQmjukRUSKS8lBRERilBxERCRGyaGE1NwhLSJSXEoOpUjZQUSKTMmhhFTfGa0WaREpMiUHERGJUXIQEZEYJYcSUn1ntNocRKTIlBxKkJ7nICLFpuRQQvQMaREpFUoOIiISo+QgIiIxSg4iIhKj5FCC1CAtIsWm5CAiIjFKDiIiEqPkICIiMUoOIiISo+RQglzt0SJSZEoOIiISo+QgIiIxrYodgNTP/118KMN7lhU7DBFponTmsJ/q0r415R3aFDsMEWmilBxKkNqjRaTYlBxERCRGyUFERGKKkhzM7DozW25m88O/4yPDrjazxWb2lpkdU4z4RESau2JerfRjd/9htMDMRgNnAAcCfYG5ZjbC3fcWI0ARkeaq1C5lnQPc7e47gffNbDEwCXguHzN78u01fPfBRfmYdL38ZO47ALz0wfo6x9WDREUkn4qZHC41s7OBecCV7r4e6Ac8HxlnWVgWY2YXARcBDBw4sF4BlLVtxfBexb1XoGUL481VmxnUrQOjenfm0YWrOPbA3jy6cBUDKtqzdN12AA4f1o0la7exatMOThzbh2G6x0FE8ihvycHM5gK90wy6BrgZ+A7BVZvfAW4Czs9l+u5+K3ArQFVVVb2u/pw4qCsTB02sz0dFRJq0vCUHd5+VzXhm9kvgwfDtcmBAZHD/sExERAqoWFcr9Ym8/RTwevj6fuAMM2trZoOB4cALhY5PRKS5K1abw41mNp6gWmkJ8AUAd19oZvcAi4A9wCW6UklEpPCKkhzc/axahl0PXF/AcEREJIXukBYRkRglBxERiVFyEBGRGCUHERGJMW8CT7M3szXAB/X8eHdgbSOG01hKNS4o3dgUV24UV26aYlyD3L1HugFNIjk0hJnNc/eqYseRqlTjgtKNTXHlRnHlprnFpWolERGJUXIQEZEYJYew874SVKpxQenGprhyo7hy06ziavZtDiIiEqczBxERiVFyEBGRmGadHMzsWDN7y8wWm9nXCjzvAWb2dzNbZGYLzezysPw6M1tuZvPDv+Mjn7k6jPUtMzsmj7EtMbPXwvnPC8sqzOxxM3sn/N81LDcz+2kY1wIzm5CnmEZG1sl8M9tkZlcUY32Z2W/M7CMzez1SlvP6MbNzwvHfMbNz8hTXD8zszXDe95pZeVheaWbbI+vtF5HPTAy3/+Iw9gY9lTZDXDlvt8b+vWaI6w+RmJaY2fywvJDrK9O+obDfMXdvln9AS+BdYAjQBngVGF3A+fcBJoSvOwFvA6OB64Avpxl/dBhjW2BwGHvLPMW2BOieUnYj8LXw9deA74evjwceIXis9RTgXwXadquAQcVYX8A0YALwen3XD1ABvBf+7xq+7pqHuI4GWoWvvx+JqzI6Xsp0XghjtTD24/IQV07bLR+/13RxpQy/Cbi2COsr076hoN+x5nzmMAlY7O7vufsu4G5gTqFm7u4r3f3l8PVm4A0yPC87NAe42913uvv7wGKCZSiUOcAd4es7gJMi5Xd64Hmg3JIf5pQPM4F33b22u+Lztr7c/SlgXZr55bJ+jgEed/d1Hjw//XHg2MaOy90fc/c94dvnCZ6umFEYW2d3f96DPcydkWVptLhqkWm7Nfrvtba4wqP/04C7aptGntZXpn1DQb9jzTk59AOWRt4vo/adc96YWSVwMPCvsOjS8PTwN4lTRwobrwOPmdlLZnZRWNbL3VeGr1cBvYoQV8IZJP9oi72+IPf1U4z1dj7BEWbCYDN7xcyeNLOpYVm/MJZCxJXLdiv0+poKrHb3dyJlBV9fKfuGgn7HmnNyKAlmVgb8CbjC3TcBNwNDgfHASoJT20I7wt0nAMcBl5jZtOjA8AipKNdAm1kb4JPA/4VFpbC+khRz/WRiZtcQPF3x92HRSmCgux8M/Afwv2bWuYAhldx2S/EZkg9ACr6+0uwbqhXiO9ack8NyYEDkff+wrGDMrDXBxv+9u/8ZwN1Xu/ted98H/JKaqpCCxevuy8P/HwH3hjGsTlQXhf8/KnRcoeOAl919dRhj0ddXKNf1U7D4zOxc4ETgs+FOhbDa5uPw9UsE9fkjwhiiVU95iase262Q66sVcDLwh0i8BV1f6fYNFPg71pyTw4vAcDMbHB6NngHcX6iZh3WavwbecPcfRcqj9fWfAhJXUtwPnGFmbc1sMDCcoCGssePqaGadEq8JGjRfD+efuNrhHOC+SFxnh1dMTAE2Rk598yHpiK7Y6ysi1/XzV+BoM+saVqkcHZY1KjM7FrgK+KS7b4uU9zCzluHrIQTr570wtk1mNiX8jp4dWZbGjCvX7VbI3+ss4E13r64uKuT6yrRvoNDfsYa0qu/vfwSt/G8THAVcU+B5H0FwWrgAmB/+HQ/8FngtLL8f6BP5zDVhrG/RwCsiaolrCMGVIK8CCxPrBegGPAG8A8wFKsJyA34exvUaUJXHddYR+BjoEikr+PoiSE4rgd0E9bgX1Gf9ELQBLA7/zstTXIsJ6p0T37FfhOOeEm7f+cDLwOzIdKoIdtbvAj8j7EmhkePKebs19u81XVxh+e3AxSnjFnJ9Zdo3FPQ7pu4zREQkpjlXK4mISAZKDiIiEqPkICIiMUoOIiISo+QgIiIxSg4iITPba8k9v9ba86eZXWxmZzfCfJeYWfeGTkekMelSVpGQmW1x97IizHcJwbXpaws9b5FMdOYgUofwyP5GC/rsf8HMhoXl15nZl8PXl1nQ//4CM7s7LKsws7+EZc+b2diwvJuZPWZBX/2/IriJKTGvz4XzmG9mt5hZy/DvdjN7PYzhS0VYDdLMKDmI1GifUq10emTYRnc/iOAO2J+k+ezXgIPdfSxwcVj2LeCVsOzrBN05A3wT+Ke7H0jQd9VAADMbBZwOHO7u44G9wGcJOqfr5+5jwhhua8RlFkmrVbEDECkh28Odcjp3Rf7/OM3wBcDvzewvwF/CsiMIul3A3f8WnjF0JnjIzMlh+UNmtj4cfyYwEXgx6F6H9gSdqz0ADDGz/wYeAh6r/yKKZEdnDiLZ8QyvE04g6N9mAsHOvT4HXgbc4e7jw7+R7n6dBw9qGQf8g+Cs5Ff1mLZITpQcRLJzeuT/c9EBZtYCGODufwe+CnQByoCnCaqFMLMjgbUe9Mv/FHBmWH4cwSMcIehU7VQz6xkOqzCzQeGVTC3c/U/ANwgSkEheqVpJpEZ7Cx8oH3rU3ROXs3Y1swXAToJuw6NaAr8zsy4ER/8/dfcNZnYd8Jvwc9uo6W75W8BdZrYQeBb4EMDdF5nZNwiewteCoLfQS4DtwG1hGcDVjbfIIunpUlaROuhSU2mOVK0kIiIxOnMQEZEYnTmIiEiMkoOIiMQoOYiISIySg4iIxCg5iIhIzP8H5SuDHq9kvMcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "actor_lr = 1e-3\n",
    "critic_lr = 1e-3\n",
    "hidden_dim = 128\n",
    "state_dim = 4\n",
    "action_dim = 2\n",
    "action_bound = 1\n",
    "sigma = 0.1\n",
    "tau = 0.005\n",
    "gamma = 0.98\n",
    "num_episodes = 2000\n",
    "n_train = 20\n",
    "batch_size = 256\n",
    "minimal_episodes = 200\n",
    "buffer_size = 10000\n",
    "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\n",
    "    \"cpu\")\n",
    "\n",
    "random.seed(0)\n",
    "np.random.seed(0)\n",
    "torch.manual_seed(0)\n",
    "env = WorldEnv()\n",
    "replay_buffer = ReplayBuffer_Trajectory(buffer_size)\n",
    "agent = DDPG(state_dim, hidden_dim, action_dim, action_bound, actor_lr,\n",
    "             critic_lr, sigma, tau, gamma, device)\n",
    "\n",
    "return_list = []\n",
    "for i in range(10):\n",
    "    with tqdm(total=int(num_episodes / 10), desc='Iteration %d' % i) as pbar:\n",
    "        for i_episode in range(int(num_episodes / 10)):\n",
    "            episode_return = 0\n",
    "            state = env.reset()\n",
    "            traj = Trajectory(state)\n",
    "            done = False\n",
    "            while not done:\n",
    "                action = agent.take_action(state)\n",
    "                state, reward, done = env.step(action)\n",
    "                episode_return += reward\n",
    "                traj.store_step(action, state, reward, done)\n",
    "            replay_buffer.add_trajectory(traj)\n",
    "            return_list.append(episode_return)\n",
    "            if replay_buffer.size() >= minimal_episodes:\n",
    "                for _ in range(n_train):\n",
    "                    transition_dict = replay_buffer.sample(batch_size, True)\n",
    "                    agent.update(transition_dict)\n",
    "            if (i_episode + 1) % 10 == 0:\n",
    "                pbar.set_postfix({\n",
    "                    'episode':\n",
    "                    '%d' % (num_episodes / 10 * i + i_episode + 1),\n",
    "                    'return':\n",
    "                    '%.3f' % np.mean(return_list[-10:])\n",
    "                })\n",
    "            pbar.update(1)\n",
    "\n",
    "episodes_list = list(range(len(return_list)))\n",
    "plt.plot(episodes_list, return_list)\n",
    "plt.xlabel('Episodes')\n",
    "plt.ylabel('Returns')\n",
    "plt.title('DDPG with HER on {}'.format('GridWorld'))\n",
    "plt.show()\n",
    "\n",
    "# Iteration 0: 100%|██████████| 200/200 [00:03<00:00, 58.91it/s, episode=200,\n",
    "# return=-50.000]\n",
    "# Iteration 1: 100%|██████████| 200/200 [01:17<00:00,  2.56it/s, episode=400,\n",
    "# return=-4.200]\n",
    "# Iteration 2: 100%|██████████| 200/200 [01:18<00:00,  2.56it/s, episode=600,\n",
    "# return=-4.700]\n",
    "# Iteration 3: 100%|██████████| 200/200 [01:18<00:00,  2.56it/s, episode=800,\n",
    "# return=-4.300]\n",
    "# Iteration 4: 100%|██████████| 200/200 [01:17<00:00,  2.57it/s, episode=1000,\n",
    "# return=-3.800]\n",
    "# Iteration 5: 100%|██████████| 200/200 [01:17<00:00,  2.57it/s, episode=1200,\n",
    "# return=-4.800]\n",
    "# Iteration 6: 100%|██████████| 200/200 [01:18<00:00,  2.54it/s, episode=1400,\n",
    "# return=-4.500]\n",
    "# Iteration 7: 100%|██████████| 200/200 [01:19<00:00,  2.52it/s, episode=1600,\n",
    "# return=-4.400]\n",
    "# Iteration 8: 100%|██████████| 200/200 [01:18<00:00,  2.55it/s, episode=1800,\n",
    "# return=-4.200]\n",
    "# Iteration 9: 100%|██████████| 200/200 [01:18<00:00,  2.55it/s, episode=2000,\n",
    "# return=-4.300]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Cc0b1OlFGUTV"
   },
   "outputs": [],
   "source": [
    "random.seed(0)\n",
    "np.random.seed(0)\n",
    "torch.manual_seed(0)\n",
    "env = WorldEnv()\n",
    "replay_buffer = ReplayBuffer_Trajectory(buffer_size)\n",
    "agent = DDPG(state_dim, hidden_dim, action_dim, action_bound, actor_lr,\n",
    "             critic_lr, sigma, tau, gamma, device)\n",
    "\n",
    "return_list = []\n",
    "for i in range(10):\n",
    "    with tqdm(total=int(num_episodes / 10), desc='Iteration %d' % i) as pbar:\n",
    "        for i_episode in range(int(num_episodes / 10)):\n",
    "            episode_return = 0\n",
    "            state = env.reset()\n",
    "            traj = Trajectory(state)\n",
    "            done = False\n",
    "            while not done:\n",
    "                action = agent.take_action(state)\n",
    "                state, reward, done = env.step(action)\n",
    "                episode_return += reward\n",
    "                traj.store_step(action, state, reward, done)\n",
    "            replay_buffer.add_trajectory(traj)\n",
    "            return_list.append(episode_return)\n",
    "            if replay_buffer.size() >= minimal_episodes:\n",
    "                for _ in range(n_train):\n",
    "                    # 和使用HER训练的唯一区别\n",
    "                    transition_dict = replay_buffer.sample(batch_size, False)\n",
    "                    agent.update(transition_dict)\n",
    "            if (i_episode + 1) % 10 == 0:\n",
    "                pbar.set_postfix({\n",
    "                    'episode':\n",
    "                    '%d' % (num_episodes / 10 * i + i_episode + 1),\n",
    "                    'return':\n",
    "                    '%.3f' % np.mean(return_list[-10:])\n",
    "                })\n",
    "            pbar.update(1)\n",
    "\n",
    "episodes_list = list(range(len(return_list)))\n",
    "plt.plot(episodes_list, return_list)\n",
    "plt.xlabel('Episodes')\n",
    "plt.ylabel('Returns')\n",
    "plt.title('DDPG without HER on {}'.format('GridWorld'))\n",
    "plt.show()\n",
    "\n",
    "# Iteration 0: 100%|██████████| 200/200 [00:03<00:00, 62.82it/s, episode=200,\n",
    "# return=-50.000]\n",
    "# Iteration 1: 100%|██████████| 200/200 [00:39<00:00,  5.01it/s, episode=400,\n",
    "# return=-50.000]\n",
    "# Iteration 2: 100%|██████████| 200/200 [00:41<00:00,  4.83it/s, episode=600,\n",
    "# return=-50.000]\n",
    "# Iteration 3: 100%|██████████| 200/200 [00:41<00:00,  4.82it/s, episode=800,\n",
    "# return=-50.000]\n",
    "# Iteration 4: 100%|██████████| 200/200 [00:41<00:00,  4.81it/s, episode=1000,\n",
    "# return=-50.000]\n",
    "# Iteration 5: 100%|██████████| 200/200 [00:41<00:00,  4.79it/s, episode=1200,\n",
    "# return=-50.000]\n",
    "# Iteration 6: 100%|██████████| 200/200 [00:42<00:00,  4.76it/s, episode=1400,\n",
    "# return=-45.500]\n",
    "# Iteration 7: 100%|██████████| 200/200 [00:41<00:00,  4.80it/s, episode=1600,\n",
    "# return=-42.600]\n",
    "# Iteration 8: 100%|██████████| 200/200 [00:40<00:00,  4.92it/s, episode=1800,\n",
    "# return=-4.800]\n",
    "# Iteration 9: 100%|██████████| 200/200 [00:40<00:00,  4.99it/s, episode=2000,\n",
    "# return=-4.800]"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "第19章-目标导向的强化学习.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}