3T-9L_Culture_Tuning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Culture Optimization - 3T-9L\n",
    "\n",
    "### **3T-9L: 3 Teams composed of 9 agents (3, 3, 3) **\n",
    "\n",
    "This is to make the 3 teams more evenly matched - each team is composed of 3 agents.\n",
    "\n",
    "We perform cultural parameter optimization search on the 5 cultures and examin how they affect the behaviors of these teams."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python version:  3.6.4\n",
      "Pytorch version: 0.4.1.post2\n",
      "OpenAI Gym version: 0.9.2\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import random\n",
    "import time\n",
    "import pickle\n",
    "import platform\n",
    "import torch\n",
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "# This is the Gathering Game Environment based on Tribal Organization of agents\n",
    "from tribes_env import GatheringEnv\n",
    "from tribes_model import *\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap'] = 'gray'\n",
    "\n",
    "# for auto-reloading external modules\n",
    "# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "print(\"Python version: \", platform.python_version())\n",
    "print(\"Pytorch version: {}\".format(torch.__version__))\n",
    "print(\"OpenAI Gym version: {}\".format(gym.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Culture = Individualist (Baseline)\n",
    "\n",
    "Run finish_episode() specific to this culture."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def finish_episode(learners, optimizers, gamma, cuda):\n",
    "    \"\"\" \n",
    "    Note that in RL, policy gradient is calculated at the end of an episode and only then used to \n",
    "    update the weights of an agent's policy. This is very different compared to image recog.\n",
    "    \n",
    "    The code will perform policy update on each learning agent independently. Reward for each time \n",
    "    step is stored in the list policy.rewards[] --> r(t)\n",
    "    \"\"\"  \n",
    "    \n",
    "    num_learners = len(learners)\n",
    "    total_norms = [0 for i in range(num_learners)]\n",
    "    policy_losses = [[] for i in range(num_learners)]\n",
    "    losses = [[] for i in range(num_learners)]\n",
    "    T_reward = []\n",
    "\n",
    "   \n",
    "    for i in range(num_learners):\n",
    "\n",
    "        R = 0\n",
    "        saved_actions = learners[i].saved_actions\n",
    "        \n",
    "        for t in tribes:\n",
    "            if t.name is learners[i].tribe:\n",
    "                T_reward = t.tribal_awards()\n",
    "                \n",
    "                # For debug only\n",
    "                # print('Agent{} receives tribal award from Tribe{}'.format(i,t.name))\n",
    "                # print (T_reward)\n",
    "                # print (learners[i].rewards)\n",
    "        \n",
    "        # Do not implement actor-critic for now\n",
    "        # value_losses = []\n",
    "        \n",
    "        rewards = deque()\n",
    "\n",
    "        for r,T in zip(learners[i].rewards[::-1],T_reward[::-1]):\n",
    "            # The agent is incentivized to cooperate by an award of 30% of what the tribe takes\n",
    "            # in by all its members\n",
    "            R = r + T + gamma * R\n",
    "            rewards.appendleft(R)\n",
    "            \n",
    "        rewards = list(rewards)\n",
    "        rewards = torch.Tensor(rewards)\n",
    "        if cuda:\n",
    "            rewards = rewards.cuda()\n",
    "\n",
    "        # z-score rewards\n",
    "        rewards = (rewards - rewards.mean()) / (1.1e-7+rewards.std())\n",
    "        \n",
    "        #Debug     \n",
    "        #print (rewards)       \n",
    "        \n",
    "        \"\"\"\n",
    "        Do not implement actor-critic for now!!!\n",
    "        for (log_prob, state_value), r in zip(saved_actions, rewards):\n",
    "            reward = r - state_value.data[0]\n",
    "            policy_losses.append(-log_prob * Variable(reward))\n",
    "            r = torch.Tensor([r])\n",
    "            if cuda:\n",
    "                r = r.cuda()\n",
    "            value_losses.append(torch.nn.functional.smooth_l1_loss(state_value,\n",
    "                                                               Variable(r)))\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss = torch.stack(policy_losses).sum() + torch.stack(value_losses).sum()\n",
    "        loss.backward()        \n",
    "        \n",
    "        \n",
    "        \"\"\"\n",
    "        for log_prob, r in zip(saved_actions, rewards):\n",
    "            r = torch.Tensor([r])\n",
    "            if cuda:\n",
    "                r = r.cuda()\n",
    "            policy_losses[i].append(-log_prob * Variable(r))\n",
    "\n",
    "        optimizers[i].zero_grad()\n",
    "        losses[i] = torch.stack(policy_losses[i]).sum()\n",
    "        losses[i].backward()\n",
    "        \n",
    "        # Gradient Clipping Update: prevent exploding gradient\n",
    "        total_norms[i] = torch.nn.utils.clip_grad_norm_(learners[i].parameters(), 8000)\n",
    "        \n",
    "        optimizers[i].step()\n",
    "        \n",
    "    # clear all agent's history only at the end of episode; they are needed for tribal reward calc\n",
    "    for i in range(num_learners):\n",
    "        learners[i].clear_history()\n",
    "\n",
    "\n",
    "    return total_norms\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Learner agent 8\n",
      "........................................................................................................................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:0\tRunning mean: 0.02686\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.004948\n",
      "Learner:4\tReward total:0\tRunning mean: 0.01949\n",
      "Learner:5\tReward total:0\tRunning mean: 0.3813\n",
      "Learner:6\tReward total:1\tRunning mean: 0.4549\n",
      "Learner:7\tReward total:13\tRunning mean: 1.166\n",
      "Learner:8\tReward total:9\tRunning mean: 5.836\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '133.66', '108.37', '96.08']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0899\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:12\tRunning mean: 0.1584\n",
      "Learner:4\tReward total:0\tRunning mean: 0.002612\n",
      "Learner:5\tReward total:8\tRunning mean: 12.37\n",
      "Learner:6\tReward total:12\tRunning mean: 8.678\n",
      "Learner:7\tReward total:3\tRunning mean: 10.05\n",
      "Learner:8\tReward total:2\tRunning mean: 23.11\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '114.88', '0.00', '111.92', '103.07', '88.06', '102.08']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.03797\n",
      "Learner:1\tReward total:11\tRunning mean: 2.74\n",
      "Learner:2\tReward total:0\tRunning mean: 0.002575\n",
      "Learner:3\tReward total:0\tRunning mean: 1.796\n",
      "Learner:4\tReward total:13\tRunning mean: 1.476\n",
      "Learner:5\tReward total:46\tRunning mean: 21.75\n",
      "Learner:6\tReward total:11\tRunning mean: 14.77\n",
      "Learner:7\tReward total:2\tRunning mean: 18.58\n",
      "Learner:8\tReward total:36\tRunning mean: 30.17\n",
      "Max Norms =  ['0.00', '55.50', '0.00', '0.00', '105.82', '132.80', '86.48', '100.94', '118.32']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.005087\n",
      "Learner:1\tReward total:5\tRunning mean: 3.812\n",
      "Learner:2\tReward total:0\tRunning mean: 0.000345\n",
      "Learner:3\tReward total:3\tRunning mean: 4.654\n",
      "Learner:4\tReward total:9\tRunning mean: 10.54\n",
      "Learner:5\tReward total:9\tRunning mean: 24.79\n",
      "Learner:6\tReward total:20\tRunning mean: 14.89\n",
      "Learner:7\tReward total:15\tRunning mean: 19.95\n",
      "Learner:8\tReward total:31\tRunning mean: 27.51\n",
      "Max Norms =  ['0.00', '55.66', '0.00', '77.36', '68.08', '44.02', '86.84', '80.49', '111.10']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.009883\n",
      "Learner:1\tReward total:7\tRunning mean: 3.814\n",
      "Learner:2\tReward total:0\tRunning mean: 4.622e-05\n",
      "Learner:3\tReward total:1\tRunning mean: 5.508\n",
      "Learner:4\tReward total:4\tRunning mean: 10.94\n",
      "Learner:5\tReward total:6\tRunning mean: 25.39\n",
      "Learner:6\tReward total:25\tRunning mean: 15.19\n",
      "Learner:7\tReward total:57\tRunning mean: 27.76\n",
      "Learner:8\tReward total:24\tRunning mean: 20.19\n",
      "Max Norms =  ['0.00', '33.42', '0.00', '52.13', '53.37', '76.64', '127.42', '105.50', '113.27']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.2678\n",
      "Learner:1\tReward total:0\tRunning mean: 3.06\n",
      "Learner:2\tReward total:0\tRunning mean: 0.01386\n",
      "Learner:3\tReward total:8\tRunning mean: 6.726\n",
      "Learner:4\tReward total:2\tRunning mean: 10.27\n",
      "Learner:5\tReward total:24\tRunning mean: 23.73\n",
      "Learner:6\tReward total:6\tRunning mean: 13.06\n",
      "Learner:7\tReward total:52\tRunning mean: 30.27\n",
      "Learner:8\tReward total:17\tRunning mean: 17.82\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '39.73', '41.80', '75.96', '154.37', '94.03', '104.42']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.4493\n",
      "Learner:1\tReward total:2\tRunning mean: 1.953\n",
      "Learner:2\tReward total:0\tRunning mean: 0.001856\n",
      "Learner:3\tReward total:12\tRunning mean: 6.361\n",
      "Learner:4\tReward total:7\tRunning mean: 8.374\n",
      "Learner:5\tReward total:10\tRunning mean: 23.16\n",
      "Learner:6\tReward total:38\tRunning mean: 11.17\n",
      "Learner:7\tReward total:18\tRunning mean: 29.65\n",
      "Learner:8\tReward total:16\tRunning mean: 19.63\n",
      "Max Norms =  ['0.00', '71.64', '0.00', '58.41', '51.13', '130.54', '75.19', '87.73', '59.44']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:3\tRunning mean: 0.7954\n",
      "Learner:1\tReward total:12\tRunning mean: 2.823\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0002487\n",
      "Learner:3\tReward total:20\tRunning mean: 9.967\n",
      "Learner:4\tReward total:4\tRunning mean: 8.205\n",
      "Learner:5\tReward total:11\tRunning mean: 22.29\n",
      "Learner:6\tReward total:2\tRunning mean: 12.01\n",
      "Learner:7\tReward total:26\tRunning mean: 28.45\n",
      "Learner:8\tReward total:39\tRunning mean: 21.26\n",
      "Max Norms =  ['25.98', '46.19', '0.00', '91.07', '52.10', '91.07', '63.32', '102.99', '161.24']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:1\tRunning mean: 0.9981\n",
      "Learner:1\tReward total:8\tRunning mean: 3.15\n",
      "Learner:2\tReward total:0\tRunning mean: 3.332e-05\n",
      "Learner:3\tReward total:7\tRunning mean: 9.67\n",
      "Learner:4\tReward total:4\tRunning mean: 7.391\n",
      "Learner:5\tReward total:28\tRunning mean: 20.12\n",
      "Learner:6\tReward total:10\tRunning mean: 13.24\n",
      "Learner:7\tReward total:33\tRunning mean: 33.56\n",
      "Learner:8\tReward total:23\tRunning mean: 24.3\n",
      "Max Norms =  ['17.15', '66.57', '0.00', '66.02', '50.60', '71.84', '104.83', '82.85', '54.03']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:2\tRunning mean: 1.098\n",
      "Learner:1\tReward total:1\tRunning mean: 2.875\n",
      "Learner:2\tReward total:0\tRunning mean: 4.465e-06\n",
      "Learner:3\tReward total:11\tRunning mean: 11.84\n",
      "Learner:4\tReward total:7\tRunning mean: 7.784\n",
      "Learner:5\tReward total:20\tRunning mean: 18.98\n",
      "Learner:6\tReward total:19\tRunning mean: 13.54\n",
      "Learner:7\tReward total:42\tRunning mean: 29.69\n",
      "Learner:8\tReward total:14\tRunning mean: 20.94\n",
      "Max Norms =  ['29.78', '51.89', '0.00', '66.91', '62.13', '91.03', '76.29', '82.20', '116.70']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.007\n",
      "Learner:1\tReward total:8\tRunning mean: 2.779\n",
      "Learner:2\tReward total:0\tRunning mean: 0.01267\n",
      "Learner:3\tReward total:9\tRunning mean: 10.75\n",
      "Learner:4\tReward total:17\tRunning mean: 8.1\n",
      "Learner:5\tReward total:4\tRunning mean: 22.16\n",
      "Learner:6\tReward total:21\tRunning mean: 11.81\n",
      "Learner:7\tReward total:32\tRunning mean: 31.79\n",
      "Learner:8\tReward total:15\tRunning mean: 19.56\n",
      "Max Norms =  ['0.00', '22.69', '0.00', '28.50', '66.13', '92.12', '158.39', '75.40', '71.94']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2400 complete\n",
      "Learner:0\tReward total:1\tRunning mean: 1.16\n",
      "Learner:1\tReward total:0\tRunning mean: 2.559\n",
      "Learner:2\tReward total:0\tRunning mean: 0.001698\n",
      "Learner:3\tReward total:1\tRunning mean: 8.06\n",
      "Learner:4\tReward total:4\tRunning mean: 8.171\n",
      "Learner:5\tReward total:41\tRunning mean: 23.25\n",
      "Learner:6\tReward total:9\tRunning mean: 16.08\n",
      "Learner:7\tReward total:54\tRunning mean: 34.89\n",
      "Learner:8\tReward total:29\tRunning mean: 21.53\n",
      "Max Norms =  ['58.28', '0.00', '0.00', '14.95', '64.92', '84.73', '207.25', '201.90', '155.82']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2600 complete\n",
      "Learner:0\tReward total:1\tRunning mean: 1.104\n",
      "Learner:1\tReward total:5\tRunning mean: 3.659\n",
      "Learner:2\tReward total:2\tRunning mean: 0.6758\n",
      "Learner:3\tReward total:21\tRunning mean: 8.985\n",
      "Learner:4\tReward total:13\tRunning mean: 9.59\n",
      "Learner:5\tReward total:13\tRunning mean: 26.73\n",
      "Learner:6\tReward total:8\tRunning mean: 15.46\n",
      "Learner:7\tReward total:40\tRunning mean: 28.28\n",
      "Learner:8\tReward total:37\tRunning mean: 25.18\n",
      "Max Norms =  ['67.42', '42.73', '61.28', '26.19', '53.78', '281.79', '178.05', '80.74', '76.09']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2800 complete\n",
      "Learner:0\tReward total:3\tRunning mean: 1.632\n",
      "Learner:1\tReward total:7\tRunning mean: 3.501\n",
      "Learner:2\tReward total:4\tRunning mean: 4.684\n",
      "Learner:3\tReward total:15\tRunning mean: 14.2\n",
      "Learner:4\tReward total:3\tRunning mean: 8.504\n",
      "Learner:5\tReward total:32\tRunning mean: 22.92\n",
      "Learner:6\tReward total:14\tRunning mean: 14.81\n",
      "Learner:7\tReward total:27\tRunning mean: 29.09\n",
      "Learner:8\tReward total:27\tRunning mean: 24.63\n",
      "Max Norms =  ['56.07', '65.92', '28.26', '37.87', '20.12', '117.88', '158.60', '61.94', '73.87']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3000 complete\n",
      "Learner:0\tReward total:3\tRunning mean: 1.551\n",
      "Learner:1\tReward total:5\tRunning mean: 3.335\n",
      "Learner:2\tReward total:13\tRunning mean: 7.632\n",
      "Learner:3\tReward total:29\tRunning mean: 14.96\n",
      "Learner:4\tReward total:12\tRunning mean: 8.103\n",
      "Learner:5\tReward total:14\tRunning mean: 17.11\n",
      "Learner:6\tReward total:1\tRunning mean: 18.54\n",
      "Learner:7\tReward total:31\tRunning mean: 28.22\n",
      "Learner:8\tReward total:16\tRunning mean: 21.11\n",
      "Max Norms =  ['13.64', '32.69', '6.80', '91.16', '120.72', '96.70', '93.83', '115.89', '216.80']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3200 complete\n",
      "Learner:0\tReward total:3\tRunning mean: 2.263\n",
      "Learner:1\tReward total:1\tRunning mean: 3.527\n",
      "Learner:2\tReward total:9\tRunning mean: 10.23\n",
      "Learner:3\tReward total:15\tRunning mean: 13.69\n",
      "Learner:4\tReward total:3\tRunning mean: 8.724\n",
      "Learner:5\tReward total:7\tRunning mean: 12.97\n",
      "Learner:6\tReward total:22\tRunning mean: 18.67\n",
      "Learner:7\tReward total:22\tRunning mean: 26.46\n",
      "Learner:8\tReward total:20\tRunning mean: 21.51\n",
      "Max Norms =  ['24.13', '49.69', '20.02', '98.03', '96.21', '105.23', '93.87', '43.42', '57.69']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3400 complete\n",
      "Learner:0\tReward total:3\tRunning mean: 2.724\n",
      "Learner:1\tReward total:4\tRunning mean: 4.166\n",
      "Learner:2\tReward total:10\tRunning mean: 13.0\n",
      "Learner:3\tReward total:6\tRunning mean: 12.86\n",
      "Learner:4\tReward total:6\tRunning mean: 6.392\n",
      "Learner:5\tReward total:18\tRunning mean: 13.57\n",
      "Learner:6\tReward total:17\tRunning mean: 18.42\n",
      "Learner:7\tReward total:16\tRunning mean: 26.39\n",
      "Learner:8\tReward total:35\tRunning mean: 23.37\n",
      "Max Norms =  ['32.79', '35.39', '16.76', '67.68', '69.77', '47.16', '125.02', '60.38', '108.97']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3600 complete\n",
      "Learner:0\tReward total:2\tRunning mean: 2.566\n",
      "Learner:1\tReward total:0\tRunning mean: 3.764\n",
      "Learner:2\tReward total:2\tRunning mean: 8.187\n",
      "Learner:3\tReward total:5\tRunning mean: 10.92\n",
      "Learner:4\tReward total:0\tRunning mean: 6.495\n",
      "Learner:5\tReward total:15\tRunning mean: 15.5\n",
      "Learner:6\tReward total:44\tRunning mean: 23.12\n",
      "Learner:7\tReward total:26\tRunning mean: 22.67\n",
      "Learner:8\tReward total:16\tRunning mean: 19.85\n",
      "Max Norms =  ['34.67', '0.00', '30.60', '47.63', '0.00', '101.50', '87.11', '55.17', '101.07']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3800 complete\n",
      "Learner:0\tReward total:4\tRunning mean: 2.752\n",
      "Learner:1\tReward total:9\tRunning mean: 3.689\n",
      "Learner:2\tReward total:12\tRunning mean: 10.68\n",
      "Learner:3\tReward total:14\tRunning mean: 9.585\n",
      "Learner:4\tReward total:2\tRunning mean: 4.699\n",
      "Learner:5\tReward total:15\tRunning mean: 14.69\n",
      "Learner:6\tReward total:28\tRunning mean: 21.04\n",
      "Learner:7\tReward total:24\tRunning mean: 26.47\n",
      "Learner:8\tReward total:19\tRunning mean: 19.7\n",
      "Max Norms =  ['31.47', '38.44', '98.01', '64.10', '1.37', '93.52', '72.11', '89.50', '153.44']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 4000 complete\n",
      "Learner:0\tReward total:3\tRunning mean: 2.384\n",
      "Learner:1\tReward total:3\tRunning mean: 4.259\n",
      "Learner:2\tReward total:9\tRunning mean: 11.38\n",
      "Learner:3\tReward total:2\tRunning mean: 9.155\n",
      "Learner:4\tReward total:7\tRunning mean: 6.357\n",
      "Learner:5\tReward total:6\tRunning mean: 12.96\n",
      "Learner:6\tReward total:29\tRunning mean: 22.95\n",
      "Learner:7\tReward total:38\tRunning mean: 29.68\n",
      "Learner:8\tReward total:27\tRunning mean: 19.74\n",
      "Max Norms =  ['5.75', '24.73', '35.04', '42.41', '84.13', '66.62', '148.90', '98.18', '181.98']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 4200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.106\n",
      "Learner:1\tReward total:2\tRunning mean: 4.385\n",
      "Learner:2\tReward total:23\tRunning mean: 13.78\n",
      "Learner:3\tReward total:19\tRunning mean: 10.64\n",
      "Learner:4\tReward total:3\tRunning mean: 6.944\n",
      "Learner:5\tReward total:15\tRunning mean: 10.68\n",
      "Learner:6\tReward total:29\tRunning mean: 24.09\n",
      "Learner:7\tReward total:13\tRunning mean: 25.97\n",
      "Learner:8\tReward total:11\tRunning mean: 19.19\n",
      "Max Norms =  ['0.00', '61.47', '35.76', '72.59', '48.94', '66.00', '146.35', '99.84', '69.44']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 4400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.1481\n",
      "Learner:1\tReward total:5\tRunning mean: 5.007\n",
      "Learner:2\tReward total:12\tRunning mean: 11.64\n",
      "Learner:3\tReward total:6\tRunning mean: 9.808\n",
      "Learner:4\tReward total:15\tRunning mean: 7.298\n",
      "Learner:5\tReward total:19\tRunning mean: 11.36\n",
      "Learner:6\tReward total:18\tRunning mean: 25.71\n",
      "Learner:7\tReward total:28\tRunning mean: 26.92\n",
      "Learner:8\tReward total:14\tRunning mean: 18.5\n",
      "Max Norms =  ['0.00', '27.56', '13.86', '109.66', '63.59', '70.25', '116.55', '75.18', '55.61']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "........................................................................................................................................................................................................\n",
      "Episode 4600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.01985\n",
      "Learner:1\tReward total:1\tRunning mean: 4.344\n",
      "Learner:2\tReward total:11\tRunning mean: 12.15\n",
      "Learner:3\tReward total:3\tRunning mean: 8.514\n",
      "Learner:4\tReward total:6\tRunning mean: 7.473\n",
      "Learner:5\tReward total:5\tRunning mean: 8.926\n",
      "Learner:6\tReward total:25\tRunning mean: 24.33\n",
      "Learner:7\tReward total:37\tRunning mean: 29.67\n",
      "Learner:8\tReward total:36\tRunning mean: 20.93\n",
      "Max Norms =  ['0.00', '32.03', '41.48', '121.91', '33.75', '51.20', '69.83', '198.34', '143.55']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 4800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.002659\n",
      "Learner:1\tReward total:6\tRunning mean: 4.005\n",
      "Learner:2\tReward total:7\tRunning mean: 8.954\n",
      "Learner:3\tReward total:5\tRunning mean: 7.712\n",
      "Learner:4\tReward total:11\tRunning mean: 7.704\n",
      "Learner:5\tReward total:21\tRunning mean: 13.27\n",
      "Learner:6\tReward total:14\tRunning mean: 25.73\n",
      "Learner:7\tReward total:36\tRunning mean: 33.46\n",
      "Learner:8\tReward total:19\tRunning mean: 18.31\n",
      "Max Norms =  ['0.00', '39.76', '39.21', '32.37', '43.53', '111.20', '130.68', '119.73', '86.15']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 5000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0003563\n",
      "Learner:1\tReward total:13\tRunning mean: 4.312\n",
      "Learner:2\tReward total:11\tRunning mean: 7.843\n",
      "Learner:3\tReward total:10\tRunning mean: 7.846\n",
      "Learner:4\tReward total:1\tRunning mean: 10.13\n",
      "Learner:5\tReward total:14\tRunning mean: 14.38\n",
      "Learner:6\tReward total:22\tRunning mean: 26.7\n",
      "Learner:7\tReward total:23\tRunning mean: 29.4\n",
      "Learner:8\tReward total:16\tRunning mean: 19.3\n",
      "Max Norms =  ['0.00', '113.04', '64.19', '48.48', '15.20', '86.57', '140.45', '123.45', '76.07']\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import sys\n",
    "import pickle\n",
    "from collections import deque\n",
    "\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import numpy as np\n",
    "\n",
    "# Initialize agents parameters\n",
    "#   10 agents - 9 learning agents, 0 trained agent, 0 random agent\n",
    "num_learners = 9\n",
    "num_trained = 0\n",
    "num_rdn = 0\n",
    "num_statics = num_trained + num_rdn\n",
    "num_agents = num_learners + num_statics  \n",
    "\n",
    "# Initialize environment\n",
    "game = \"Gather\"\n",
    "num_actions = 8                       # 8 actions in Gathering\n",
    "\n",
    "# Initialize training parameters\n",
    "warm_start = False\n",
    "num_frames = 4      # environ observation consists of a list of 4 stacked frames per agent\n",
    "max_episodes = 5000\n",
    "max_frames = 300\n",
    "max_frames_ep = 0   # track highest number of frames an episode can last\n",
    "\n",
    "# These trainer parameters works for Atari Breakout\n",
    "gamma = 0.99  \n",
    "lr = 1e-3\n",
    "temp_start = 1.8  # Temperature for explore/exploit\n",
    "temp_end = 1.0\n",
    "log_interval = 200\n",
    "save_interval = 500\n",
    "\n",
    "def unpack_env_obs(env_obs):\n",
    "    \"\"\"\n",
    "    Gathering is a partially-observable Markov Game. env_obs returned by GatheringEnv is a numpy \n",
    "    array of dimension (num_agent, 800), which represents the agents' observations of the game.\n",
    "\n",
    "    The 800 elements (view_box) encodes 4 layers of 10x20 pixels frames in the format:\n",
    "    (viewbox_width, viewbox_depth, 4).\n",
    "    \n",
    "    This code reshapes the above into stacked frames that can be accepted by the Policy class:\n",
    "    (batch_idx, in_channel, width, height)\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    num_agents = len(env_obs)  # environ observations is a list of agents' observations\n",
    "    \n",
    "    obs = []\n",
    "    for i in range(num_agents):\n",
    "        x = env_obs[i]   # take the indexed agent's observation\n",
    "        x = torch.Tensor(x)   # Convert to tensor\n",
    "        \n",
    "        # Policy is a 3-layer CNN\n",
    "        x = x.view(1, 10, 20, -1)  # reshape into environment defined stacked frames\n",
    "        x = x.permute(0, 3, 1, 2)  # permute to Policy accepted stacked frames\n",
    "        obs.append(x)\n",
    "        \n",
    "    return obs  # return a list of Tensors\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "For now, we do not implement LSTM            \n",
    "# LSTM Change: Need to cycle hx and cx thru function\n",
    "def select_action(model, state, lstm_hc, cuda):\n",
    "    hx , cx = lstm_hc \n",
    "    num_frames, height, width = state.shape\n",
    "    state = torch.FloatTensor(state.reshape(-1, num_frames, height, width))\n",
    "\n",
    "    if cuda:\n",
    "        state = state.cuda()\n",
    "\n",
    "    probs, value, (hx, cx) = model((Variable(state), (hx, cx)))\n",
    "\n",
    "    m = torch.distributions.Categorical(probs)\n",
    "    action = m.sample()\n",
    "    log_prob = m.log_prob(action)\n",
    "    # LSTM Change: Need to cycle hx and cx thru function\n",
    "    return action.data[0], log_prob, value, (hx, cx)\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "def select_learner_action(model, obs, cuda):\n",
    "    \"\"\"\n",
    "    This code expects obs to be an array of stacked frames of the following dim:\n",
    "    (batch_idx, in_channel, width, height)\n",
    "    \n",
    "    This is inputted into model - the agent's Policy, which outputs a probability \n",
    "    distribution over available actions.\n",
    "    \n",
    "    Policy gradient is implemented using torch.distributions.Categorical. \n",
    "    \"\"\"\n",
    "    \n",
    "    # Policy is a 3-layer CNN\n",
    "    # _, num_frames, width, height = obs.shape\n",
    "    # obs = torch.FloatTensor(obs.reshape(-1, num_frames, width, height))\n",
    "    \n",
    "    # Policy is a 2-layer NN for now\n",
    "    # obs = obs.view(1, -1)\n",
    "   \n",
    "    if cuda:\n",
    "        obs = obs.cuda()\n",
    "      \n",
    "    probs = model(obs)\n",
    "    m = torch.distributions.Categorical(probs)\n",
    "    action = m.sample()\n",
    "    log_prob = m.log_prob(action)\n",
    "\n",
    "    return action.item(), log_prob \n",
    "\n",
    "\n",
    "def load_info(agents, narrate=False):\n",
    "    for i in range(num_agents):    \n",
    "        agents[i].load_info(info[i])\n",
    "        if narrate:\n",
    "            if agents[i].tagged:\n",
    "                print('frame {}, agent{} is tagged'.format(frame,i))\n",
    "            if agents[i].laser_fired:\n",
    "                print('frame {}, agent{} fires its laser'.format(frame,i))\n",
    "                print('and hit {} US and {} THEM'.format(agents[i].US_hit, agents[i].THEM_hit))\n",
    "    return\n",
    "\n",
    "\n",
    "# The main code starts here!!!\n",
    "\n",
    "# Cultural parameter search\n",
    "cultures =[ {'name':'individualist'}]\n",
    "\n",
    "for culture in cultures:   # Go down the list of cultures\n",
    "\n",
    "    # Data structure for agents\n",
    "    agents = []\n",
    "    actions = []\n",
    "    log_probs = []\n",
    "    tags = []\n",
    "    rewards = []\n",
    "    optimizers = []\n",
    "\n",
    "    # Cold start\n",
    "    if warm_start is False:\n",
    "   \n",
    "        # Initialize learner agents, then load static agents (trained followed by random)\n",
    "        for i in range(num_learners):\n",
    "            print(\"Learner agent {}\".format(i))\n",
    "            agents.append(Policy(num_frames, num_actions, i)) # No weights loaded for learning agent\n",
    "            optimizers.append(optim.Adam(agents[i].parameters(), lr=lr))\n",
    "        \n",
    "            # set up optimizer - this works for Atari Breakout\n",
    "            # optimizers.append(optim.RMSprop(agents[i].parameters(), lr=lr, weight_decay=0.1)) \n",
    "        \n",
    "        for i in range(num_learners, num_learners+num_trained):\n",
    "            print (\"Learning with trained agents - not implemented yet!\")\n",
    "            raise\n",
    "            \"\"\"\n",
    "            Disable for now! No need to train with trained agents.\n",
    "            agents.append(Policy(num_frames, num_actions, i))\n",
    "            agents[i].load_weights()         # load weight for static agent        \n",
    "            \"\"\"\n",
    "        for i in range(num_learners+num_trained, num_agents):\n",
    "            print(\"Load random agent {}\".format(i))\n",
    "            agents.append(Rdn_Policy())\n",
    "\n",
    "    \n",
    "        # Initialize all agent data\n",
    "        actions = [0 for i in range(num_agents)]\n",
    "        log_probs = [0 for i in range(num_agents)]\n",
    "        tags = [0 for i in range(num_agents)]\n",
    "        rewards = [0 for i in range(num_agents)]\n",
    "\n",
    "        # Keep track of rewards learned by learners\n",
    "        episode_reward = [0 for i in range(num_learners)]   # reward for an episode\n",
    "        running_reward = [None for i in range(num_learners)]   # running average\n",
    "        running_rewards = [[] for i in range(num_learners)]   # history of running averages\n",
    "        best_reward = [0 for i in range(num_learners)]    # best running average (for storing best_model)\n",
    "\n",
    "        # This is to support warm start for training\n",
    "        prior_eps = 0\n",
    "\n",
    "    # Warm start\n",
    "    if warm_start:\n",
    "        print (\"Cannot warm start\")\n",
    "        raise\n",
    "    \n",
    "        \"\"\"\n",
    "        # Disable for now!  Need to ensure model can support training on GPU and game playing\n",
    "        # on both CPU and GPU.\n",
    "    \n",
    "        data_file = 'results/{}.p'.format(game)\n",
    "\n",
    "        try:\n",
    "            with open(data_file, 'rb') as f:\n",
    "                running_rewards = pickle.load(f)\n",
    "                running_reward = running_rewards[-1]\n",
    "\n",
    "            prior_eps = len(running_rewards)\n",
    "\n",
    "            model_file = 'saved_models/actor_critic_{}_ep_{}.p'.format(game, prior_eps)\n",
    "            with open(model_file, 'rb') as f:\n",
    "                # Model Save and Load Update: Include both model and optim parameters\n",
    "                saved_model = pickle.load(f)\n",
    "                model, optimizer = saved_model\n",
    "\n",
    "        except OSError:\n",
    "            print('Saved file not found. Creating new cold start model.')\n",
    "            model = Policy(input_channels=num_frames, num_actions=num_actions)\n",
    "            optimizer = optim.RMSprop(model.parameters(), lr=lr,\n",
    "                                      weight_decay=0.1)\n",
    "            running_rewards = []\n",
    "            prior_eps = 0\n",
    "        \"\"\"\n",
    "\n",
    "    # Establish tribal association\n",
    "\n",
    "    tribes = []\n",
    "    tribes.append(Tribe(name='Vikings',color='blue', culture=culture, \\\n",
    "                    agents=[agents[0], agents[1], agents[2]]))\n",
    "    tribes.append(Tribe(name='Saxons', color='red', culture=culture, \\\n",
    "                    agents=[agents[3], agents[4], agents[5]]))\n",
    "    tribes.append(Tribe(name='Franks', color='purple', culture=culture, \\\n",
    "                    agents=[agents[6], agents[7], agents[8]]))\n",
    "    # tribes.append(Tribe(name='Crazies', color='yellow', agents=[agents[9]]))   # random agents are crazy!!!\n",
    "\n",
    "\n",
    "    # 9 agents in 4 tribes, used map defined in default.txt\n",
    "    agent_colors = [agent.color for agent in agents]\n",
    "    agent_tribes = [agent.tribe for agent in agents]\n",
    "    env = GatheringEnv(n_agents=num_agents,agent_colors=agent_colors, agent_tribes=agent_tribes, \\\n",
    "                       map_name='default')    \n",
    "   \n",
    "    \n",
    "    cuda = torch.cuda.is_available()\n",
    "\n",
    "    if cuda:\n",
    "        for i in range(num_learners):    # Learning agents need to utilize GPU\n",
    "            agents[i].cuda()\n",
    "\n",
    "        \n",
    "    for ep in range(max_episodes):\n",
    "    \n",
    "        print('.', end='')  # To show progress\n",
    "    \n",
    "        # Anneal temperature from temp_start to temp_end\n",
    "        for i in range(num_learners):    # For learning agents\n",
    "            agents[i].temperature = max(temp_end, temp_start - (temp_start - temp_end) * (ep / max_episodes))\n",
    "\n",
    "        env_obs = env.reset()  # Env return observations\n",
    "\n",
    "        # For Debug only\n",
    "        # print (len(env_obs))\n",
    "        # print (env_obs[0].shape)\n",
    "    \n",
    "        # Unpack observations into data structure compatible with agent Policy\n",
    "        agents_obs = unpack_env_obs(env_obs)\n",
    "\n",
    "        for i in range(num_learners):    # Reset agent info - laser tag statistics\n",
    "            agents[i].reset_info()   \n",
    "\n",
    "        # For Debug only\n",
    "        # print (len(agents_obs))\n",
    "        # print (agents_obs[0].shape)\n",
    "    \n",
    "        \"\"\"\n",
    "        For now, we do not stack observations, and we do not implement LSTM\n",
    "    \n",
    "        state = np.stack([state]*num_frames)\n",
    "\n",
    "        # LSTM change - reset LSTM hidden units when episode begins\n",
    "        cx = Variable(torch.zeros(1, 256))\n",
    "        hx = Variable(torch.zeros(1, 256))\n",
    "        if cuda:\n",
    "            cx = cx.cuda()\n",
    "            hx = hx.cuda()\n",
    "        \"\"\"\n",
    "\n",
    "        episode_reward = [0 for i in range(num_learners)]   # reward for an episode\n",
    "    \n",
    "        for frame in range(max_frames):\n",
    "\n",
    "            \"\"\"\n",
    "            For now, we do not implement LSTM\n",
    "            # Select action\n",
    "            # LSTM Change: Need to cycle hx and cx thru select_action\n",
    "            action, log_prob, value, (hx,cx)  = select_action(model, state, (hx,cx), cuda)        \n",
    "            \"\"\"\n",
    "\n",
    "            for i in range(num_learners):    # For learning agents\n",
    "                actions[i], log_probs[i] = select_learner_action(agents[i], agents_obs[i], cuda)\n",
    "                if actions[i] is 6:\n",
    "                    tags[i] += 1   # record a tag for accessing aggressiveness\n",
    "                agents[i].saved_actions.append((log_probs[i]))\n",
    "            \n",
    "                # Do not implement LSTM for now\n",
    "                # actions[i].saved_actions.append((log_prob, value))\n",
    "            \n",
    "            for i in range(num_learners, num_learners+num_trained):\n",
    "                print (\"No trained agent exist yet!\")\n",
    "                raise\n",
    "            for i in range(num_learners+num_trained, num_agents):   # For random agents\n",
    "                actions[i] = agents[i].select_action(agents_obs[i])\n",
    "                if actions[i] is 6:\n",
    "                    tags[i] += 1   # record a tag for accessing aggressiveness\n",
    "\n",
    "            # For Debug only\n",
    "            # if frame % 20 == 0:\n",
    "            #    print (actions) \n",
    "            #    print (log_probs)\n",
    "            \n",
    "            # Perform step        \n",
    "            env_obs, reward, done, info = env.step(actions)\n",
    "        \n",
    "            \"\"\"\n",
    "            For Debug only\n",
    "            print (env_obs)\n",
    "            print (reward)\n",
    "            print (done) \n",
    "            \"\"\"\n",
    "       \n",
    "            # Unpack observations into data structure compatible with agent Policy\n",
    "            agents_obs = unpack_env_obs(env_obs)\n",
    "            load_info(agents, narrate=False)   # Load agent info for AI agents\n",
    "\n",
    "            # For learner agents only, generate reward statistics and reward stack for policy gradient\n",
    "            for i in range(num_learners):\n",
    "                agents[i].rewards.append(reward[i])  # Stack rewards (for policy gradient)\n",
    "                episode_reward[i] += reward[i]   # accumulate episode reward \n",
    "            \n",
    "            \"\"\"\n",
    "            For now, we do not stack observation, may come in handy later on\n",
    "        \n",
    "            # Evict oldest diff add new diff to state\n",
    "            next_state = np.stack([next_state]*num_frames)\n",
    "            next_state[1:, :, :] = state[:-1, :, :]\n",
    "            state = next_state\n",
    "            \"\"\"\n",
    "\n",
    "            if any(done):\n",
    "                print(\"Done after {} frames\".format(frame))\n",
    "                break\n",
    "            \n",
    "        if frame > max_frames_ep:\n",
    "            max_frames_ep = frame    # Keep track of highest frames/episode\n",
    "\n",
    "        # Update reward statistics for learners\n",
    "        for i in range(num_learners):\n",
    "            if running_reward[i] is None:\n",
    "                running_reward[i] = episode_reward[i]\n",
    "            running_reward[i] = running_reward[i] * 0.99 + episode_reward[i] * 0.01\n",
    "            running_rewards[i].append(running_reward[i])\n",
    "\n",
    "        # Track Episode #, temp and highest frames/episode\n",
    "        if (ep+prior_eps+1) % log_interval == 0: \n",
    "            verbose_str = '\\nEpisode {} complete'.format(ep+prior_eps+1)\n",
    "            # verbose_str += '\\tTemp = {:.4}'.format(model.temperature)\n",
    "            # verbose_str += '\\tMax frames = {}'.format(max_frames_ep+1)\n",
    "            print(verbose_str)\n",
    "    \n",
    "            # Display rewards and running rewards for learning agents\n",
    "            for i in range(num_learners):\n",
    "                verbose_str = 'Learner:{}'.format(i)\n",
    "                verbose_str += '\\tReward total:{}'.format(episode_reward[i])\n",
    "                verbose_str += '\\tRunning mean: {:.4}'.format(running_reward[i])\n",
    "                print(verbose_str)\n",
    "    \n",
    "        # Update model\n",
    "        total_norms = finish_episode(agents[0:num_learners], optimizers[0:num_learners], gamma, cuda)\n",
    "\n",
    "        if (ep+prior_eps+1) % log_interval == 0:\n",
    "            print('Max Norms = ',[\"%0.2f\" % i for i in total_norms])\n",
    "        \n",
    "        if (ep+prior_eps+1) % save_interval == 0: \n",
    "            for i in range(num_learners):\n",
    "                model_dir = 'MA_models/3T-9L/'\n",
    "                results_dir = 'results/3T-9L/'\n",
    "\n",
    "                model_file = model_dir+'{}/MA{}_{}_ep{}.p'.format(culture['name'], i, game, ep+prior_eps+1)\n",
    "                data_file = results_dir+'{}/MA{}_{}.p'.format(culture['name'], i, game)\n",
    "                \n",
    "                os.makedirs(os.path.dirname(model_file), exist_ok=True)\n",
    "                os.makedirs(os.path.dirname(data_file), exist_ok=True)\n",
    "                \n",
    "                with open(model_file, 'wb') as f:\n",
    "                    # Model Save and Load Update: Include both model and optim parameters \n",
    "                    pickle.dump((agents[i].cpu(), optimizers[i]), f)\n",
    "\n",
    "                if cuda:\n",
    "                    agents[i] = agents[i].cuda()\n",
    "\n",
    "                with open(data_file, 'wb') as f:\n",
    "                    pickle.dump(running_rewards[i], f)    \n",
    "            \n",
    "    env.close()  # Close the environment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Culture = Warlike\n",
    "\n",
    "Run finish_episode() specific to this culture."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "def finish_episode(learners, optimizers, gamma, cuda):\n",
    "    \"\"\" \n",
    "    Note that in RL, policy gradient is calculated at the end of an episode and only then used to \n",
    "    update the weights of an agent's policy. This is very different compared to image recog.\n",
    "    \n",
    "    The code will perform policy update on each learning agent independently. Reward for each time \n",
    "    step is stored in the list policy.rewards[] --> r(t)\n",
    "    \"\"\"  \n",
    "    \n",
    "    num_learners = len(learners)\n",
    "    total_norms = [0 for i in range(num_learners)]\n",
    "    policy_losses = [[] for i in range(num_learners)]\n",
    "    losses = [[] for i in range(num_learners)]\n",
    "    T_reward = []\n",
    "\n",
    "   \n",
    "    for i in range(num_learners):\n",
    "\n",
    "        R = 0\n",
    "        saved_actions = learners[i].saved_actions\n",
    "        \n",
    "        for t in tribes:\n",
    "            if t.name is learners[i].tribe:\n",
    "                T_reward = t.tribal_awards(US_hits = learners[i].US_hits,THEM_hits = learners[i].THEM_hits)\n",
    " \n",
    "                # For debug only\n",
    "                # print('Agent{} receives tribal award from Tribe{}'.format(i,t.name))\n",
    "                # print (T_reward)\n",
    "                # print (learners[i].rewards)\n",
    "                \n",
    "        # Do not implement actor-critic for now\n",
    "        # value_losses = []\n",
    "        \n",
    "        rewards = deque()\n",
    "\n",
    "        for r,T in zip(learners[i].rewards[::-1],T_reward[::-1]):\n",
    "            # The agent is incentivized to cooperate by an award of 30% of what the tribe takes\n",
    "            # in by all its members\n",
    "            R = r + T + gamma * R\n",
    "            rewards.appendleft(R)\n",
    "            \n",
    "        rewards = list(rewards)\n",
    "        rewards = torch.Tensor(rewards)\n",
    "        if cuda:\n",
    "            rewards = rewards.cuda()\n",
    "\n",
    "        # z-score rewards\n",
    "        rewards = (rewards - rewards.mean()) / (1.1e-7+rewards.std())\n",
    "        \n",
    "        #Debug     \n",
    "        #print (rewards)       \n",
    "        \n",
    "        \"\"\"\n",
    "        Do not implement actor-critic for now!!!\n",
    "        for (log_prob, state_value), r in zip(saved_actions, rewards):\n",
    "            reward = r - state_value.data[0]\n",
    "            policy_losses.append(-log_prob * Variable(reward))\n",
    "            r = torch.Tensor([r])\n",
    "            if cuda:\n",
    "                r = r.cuda()\n",
    "            value_losses.append(torch.nn.functional.smooth_l1_loss(state_value,\n",
    "                                                               Variable(r)))\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss = torch.stack(policy_losses).sum() + torch.stack(value_losses).sum()\n",
    "        loss.backward()        \n",
    "        \n",
    "        \n",
    "        \"\"\"\n",
    "        for log_prob, r in zip(saved_actions, rewards):\n",
    "            r = torch.Tensor([r])\n",
    "            if cuda:\n",
    "                r = r.cuda()\n",
    "            policy_losses[i].append(-log_prob * Variable(r))\n",
    "\n",
    "        optimizers[i].zero_grad()\n",
    "        losses[i] = torch.stack(policy_losses[i]).sum()\n",
    "        losses[i].backward()\n",
    "        \n",
    "        # Gradient Clipping Update: prevent exploding gradient\n",
    "        total_norms[i] = torch.nn.utils.clip_grad_norm_(learners[i].parameters(), 8000)\n",
    "        \n",
    "        optimizers[i].step()\n",
    "        learners[i].clear_history()   # clear an agent's history at the end of episode\n",
    "\n",
    "\n",
    "    return total_norms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Special Case - Warlike Culture\n",
    "\n",
    "We discovered that if we run training of only 300 game steps, the agents with Warlike culture will spend the 1st hundreds of steps in mutually assured destruction - a field of deadly crossfire which they cannot escape out of.\n",
    "\n",
    "So we lengthen the game step to 1000 instead."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Load random agent 8\n",
      "....................................................................................................\n",
      "Episode 100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.006894\n",
      "Learner:1\tReward total:0\tRunning mean: 0.2255\n",
      "Learner:2\tReward total:1\tRunning mean: 1.172\n",
      "Learner:3\tReward total:0\tRunning mean: 0.292\n",
      "Learner:4\tReward total:0\tRunning mean: 0.7711\n",
      "Learner:5\tReward total:0\tRunning mean: 0.2664\n",
      "Learner:6\tReward total:6\tRunning mean: 1.922\n",
      "Learner:7\tReward total:15\tRunning mean: 8.157\n",
      "Max Norms =  ['153.63', '390.66', '179.70', '382.83', '177.09', '329.05', '250.73', '422.81']\n",
      "....................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:3\tRunning mean: 0.1015\n",
      "Learner:1\tReward total:1\tRunning mean: 0.558\n",
      "Learner:2\tReward total:50\tRunning mean: 15.79\n",
      "Learner:3\tReward total:16\tRunning mean: 4.78\n",
      "Learner:4\tReward total:31\tRunning mean: 12.5\n",
      "Learner:5\tReward total:0\tRunning mean: 0.5677\n",
      "Learner:6\tReward total:31\tRunning mean: 13.67\n",
      "Learner:7\tReward total:110\tRunning mean: 63.93\n",
      "Max Norms =  ['203.85', '262.33', '157.86', '256.05', '144.46', '230.61', '142.16', '303.94']\n",
      "....................................................................................................\n",
      "Episode 300 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.07372\n",
      "Learner:1\tReward total:16\tRunning mean: 4.839\n",
      "Learner:2\tReward total:58\tRunning mean: 41.51\n",
      "Learner:3\tReward total:39\tRunning mean: 18.11\n",
      "Learner:4\tReward total:48\tRunning mean: 37.81\n",
      "Learner:5\tReward total:0\tRunning mean: 1.058\n",
      "Learner:6\tReward total:54\tRunning mean: 43.63\n",
      "Learner:7\tReward total:172\tRunning mean: 112.1\n",
      "Max Norms =  ['136.69', '219.48', '214.09', '238.34', '233.95', '213.04', '185.63', '328.10']\n",
      "....................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.04208\n",
      "Learner:1\tReward total:14\tRunning mean: 18.89\n",
      "Learner:2\tReward total:30\tRunning mean: 46.16\n",
      "Learner:3\tReward total:30\tRunning mean: 23.19\n",
      "Learner:4\tReward total:55\tRunning mean: 58.69\n",
      "Learner:5\tReward total:0\tRunning mean: 0.4521\n",
      "Learner:6\tReward total:132\tRunning mean: 60.4\n",
      "Learner:7\tReward total:108\tRunning mean: 117.4\n",
      "Max Norms =  ['0.00', '166.06', '274.50', '109.75', '138.80', '163.73', '163.00', '202.57']\n",
      "....................................................................................................\n",
      "Episode 500 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0154\n",
      "Learner:1\tReward total:31\tRunning mean: 17.93\n",
      "Learner:2\tReward total:60\tRunning mean: 35.07\n",
      "Learner:3\tReward total:1\tRunning mean: 18.17\n",
      "Learner:4\tReward total:3\tRunning mean: 61.92\n",
      "Learner:5\tReward total:0\tRunning mean: 0.1655\n",
      "Learner:6\tReward total:112\tRunning mean: 71.48\n",
      "Learner:7\tReward total:153\tRunning mean: 141.6\n",
      "Max Norms =  ['0.00', '172.62', '242.48', '158.68', '380.04', '122.54', '205.86', '325.59']\n",
      "....................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.02819\n",
      "Learner:1\tReward total:13\tRunning mean: 13.06\n",
      "Learner:2\tReward total:18\tRunning mean: 26.6\n",
      "Learner:3\tReward total:7\tRunning mean: 14.85\n",
      "Learner:4\tReward total:46\tRunning mean: 45.11\n",
      "Learner:5\tReward total:0\tRunning mean: 0.06057\n",
      "Learner:6\tReward total:76\tRunning mean: 75.74\n",
      "Learner:7\tReward total:235\tRunning mean: 185.5\n",
      "Max Norms =  ['0.00', '220.99', '227.90', '169.10', '202.27', '38.11', '132.49', '274.51']\n",
      "....................................................................................................\n",
      "Episode 700 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.01032\n",
      "Learner:1\tReward total:0\tRunning mean: 9.597\n",
      "Learner:2\tReward total:41\tRunning mean: 24.68\n",
      "Learner:3\tReward total:18\tRunning mean: 12.3\n",
      "Learner:4\tReward total:11\tRunning mean: 31.19\n",
      "Learner:5\tReward total:0\tRunning mean: 0.02217\n",
      "Learner:6\tReward total:84\tRunning mean: 77.55\n",
      "Learner:7\tReward total:205\tRunning mean: 207.1\n",
      "Max Norms =  ['0.00', '0.00', '193.89', '75.69', '166.35', '15.26', '223.45', '229.44']\n",
      "....................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.003777\n",
      "Learner:1\tReward total:25\tRunning mean: 8.244\n",
      "Learner:2\tReward total:4\tRunning mean: 21.92\n",
      "Learner:3\tReward total:6\tRunning mean: 10.49\n",
      "Learner:4\tReward total:0\tRunning mean: 26.43\n",
      "Learner:5\tReward total:0\tRunning mean: 0.008116\n",
      "Learner:6\tReward total:59\tRunning mean: 89.71\n",
      "Learner:7\tReward total:349\tRunning mean: 217.0\n",
      "Max Norms =  ['0.00', '147.48', '243.81', '87.93', '164.57', '8.62', '147.62', '316.50']\n",
      "....................................................................................................\n",
      "Episode 900 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.001383\n",
      "Learner:1\tReward total:0\tRunning mean: 7.138\n",
      "Learner:2\tReward total:0\tRunning mean: 14.35\n",
      "Learner:3\tReward total:7\tRunning mean: 9.12\n",
      "Learner:4\tReward total:15\tRunning mean: 25.8\n",
      "Learner:5\tReward total:0\tRunning mean: 0.002971\n",
      "Learner:6\tReward total:143\tRunning mean: 103.3\n",
      "Learner:7\tReward total:166\tRunning mean: 205.7\n",
      "Max Norms =  ['0.00', '0.00', '255.78', '63.94', '137.68', '15.87', '162.60', '156.92']\n",
      "....................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0005061\n",
      "Learner:1\tReward total:21\tRunning mean: 9.443\n",
      "Learner:2\tReward total:16\tRunning mean: 13.19\n",
      "Learner:3\tReward total:5\tRunning mean: 8.559\n",
      "Learner:4\tReward total:44\tRunning mean: 23.83\n",
      "Learner:5\tReward total:0\tRunning mean: 0.001087\n",
      "Learner:6\tReward total:90\tRunning mean: 113.9\n",
      "Learner:7\tReward total:163\tRunning mean: 187.6\n",
      "Max Norms =  ['0.00', '272.99', '189.94', '116.42', '290.18', '8.78', '166.91', '181.02']\n",
      "....................................................................................................\n",
      "Episode 1100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.1458\n",
      "Learner:1\tReward total:8\tRunning mean: 10.38\n",
      "Learner:2\tReward total:20\tRunning mean: 12.95\n",
      "Learner:3\tReward total:13\tRunning mean: 6.42\n",
      "Learner:4\tReward total:29\tRunning mean: 25.97\n",
      "Learner:5\tReward total:0\tRunning mean: 0.000398\n",
      "Learner:6\tReward total:130\tRunning mean: 124.8\n",
      "Learner:7\tReward total:165\tRunning mean: 181.5\n",
      "Max Norms =  ['0.00', '173.15', '221.75', '54.93', '143.10', '27.08', '131.41', '273.37']\n",
      "....................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.05856\n",
      "Learner:1\tReward total:11\tRunning mean: 11.18\n",
      "Learner:2\tReward total:8\tRunning mean: 15.8\n",
      "Learner:3\tReward total:8\tRunning mean: 7.135\n",
      "Learner:4\tReward total:33\tRunning mean: 26.18\n",
      "Learner:5\tReward total:0\tRunning mean: 0.0001457\n",
      "Learner:6\tReward total:181\tRunning mean: 134.0\n",
      "Learner:7\tReward total:156\tRunning mean: 180.3\n",
      "Max Norms =  ['0.00', '175.05', '134.14', '80.94', '170.36', '0.52', '139.21', '170.94']\n",
      "....................................................................................................\n",
      "Episode 1300 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.02143\n",
      "Learner:1\tReward total:11\tRunning mean: 12.05\n",
      "Learner:2\tReward total:19\tRunning mean: 13.38\n",
      "Learner:3\tReward total:2\tRunning mean: 7.642\n",
      "Learner:4\tReward total:31\tRunning mean: 25.61\n",
      "Learner:5\tReward total:0\tRunning mean: 5.332e-05\n",
      "Learner:6\tReward total:183\tRunning mean: 128.9\n",
      "Learner:7\tReward total:135\tRunning mean: 178.1\n",
      "Max Norms =  ['0.00', '214.05', '112.21', '77.97', '197.02', '0.13', '200.75', '167.21']\n",
      "....................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.007846\n",
      "Learner:1\tReward total:25\tRunning mean: 14.45\n",
      "Learner:2\tReward total:23\tRunning mean: 12.93\n",
      "Learner:3\tReward total:6\tRunning mean: 8.123\n",
      "Learner:4\tReward total:1\tRunning mean: 23.22\n",
      "Learner:5\tReward total:0\tRunning mean: 1.952e-05\n",
      "Learner:6\tReward total:138\tRunning mean: 135.6\n",
      "Learner:7\tReward total:186\tRunning mean: 185.3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max Norms =  ['0.00', '188.34', '130.70', '28.47', '120.10', '12.57', '174.64', '209.38']\n",
      "....................................................................................................\n",
      "Episode 1500 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.002872\n",
      "Learner:1\tReward total:21\tRunning mean: 13.42\n",
      "Learner:2\tReward total:12\tRunning mean: 11.73\n",
      "Learner:3\tReward total:4\tRunning mean: 7.923\n",
      "Learner:4\tReward total:24\tRunning mean: 19.67\n",
      "Learner:5\tReward total:0\tRunning mean: 7.144e-06\n",
      "Learner:6\tReward total:154\tRunning mean: 118.7\n",
      "Learner:7\tReward total:173\tRunning mean: 178.7\n",
      "Max Norms =  ['0.00', '176.14', '147.63', '13.15', '272.13', '0.58', '236.86', '356.46']\n",
      "....................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.001051\n",
      "Learner:1\tReward total:14\tRunning mean: 11.31\n",
      "Learner:2\tReward total:4\tRunning mean: 12.23\n",
      "Learner:3\tReward total:0\tRunning mean: 6.768\n",
      "Learner:4\tReward total:50\tRunning mean: 20.97\n",
      "Learner:5\tReward total:0\tRunning mean: 2.615e-06\n",
      "Learner:6\tReward total:117\tRunning mean: 119.5\n",
      "Learner:7\tReward total:126\tRunning mean: 185.1\n",
      "Max Norms =  ['0.00', '104.12', '149.63', '0.00', '202.97', '3.84', '165.86', '227.41']\n",
      "....................................................................................................\n",
      "Episode 1700 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0003848\n",
      "Learner:1\tReward total:21\tRunning mean: 10.77\n",
      "Learner:2\tReward total:11\tRunning mean: 8.99\n",
      "Learner:3\tReward total:4\tRunning mean: 5.799\n",
      "Learner:4\tReward total:51\tRunning mean: 19.91\n",
      "Learner:5\tReward total:0\tRunning mean: 9.572e-07\n",
      "Learner:6\tReward total:105\tRunning mean: 112.1\n",
      "Learner:7\tReward total:124\tRunning mean: 178.8\n",
      "Max Norms =  ['0.00', '120.32', '183.85', '120.85', '192.79', '0.06', '114.73', '259.66']\n",
      "....................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0001408\n",
      "Learner:1\tReward total:0\tRunning mean: 5.349\n",
      "Learner:2\tReward total:0\tRunning mean: 3.682\n",
      "Learner:3\tReward total:0\tRunning mean: 2.55\n",
      "Learner:4\tReward total:0\tRunning mean: 8.973\n",
      "Learner:5\tReward total:0\tRunning mean: 3.504e-07\n",
      "Learner:6\tReward total:0\tRunning mean: 48.48\n",
      "Learner:7\tReward total:0\tRunning mean: 77.64\n",
      "Max Norms =  ['0.00', '0.00', '0.24', '0.00', '6.08', '0.01', '0.00', '0.43']\n",
      "....................................................................................................\n",
      "Episode 1900 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 5.155e-05\n",
      "Learner:1\tReward total:0\tRunning mean: 2.818\n",
      "Learner:2\tReward total:0\tRunning mean: 2.107\n",
      "Learner:3\tReward total:0\tRunning mean: 1.069\n",
      "Learner:4\tReward total:0\tRunning mean: 3.584\n",
      "Learner:5\tReward total:0\tRunning mean: 1.282e-07\n",
      "Learner:6\tReward total:0\tRunning mean: 19.84\n",
      "Learner:7\tReward total:0\tRunning mean: 29.86\n",
      "Max Norms =  ['0.00', '0.00', '0.18', '0.00', '4.33', '0.01', '0.00', '0.43']\n",
      "....................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.887e-05\n",
      "Learner:1\tReward total:0\tRunning mean: 1.031\n",
      "Learner:2\tReward total:0\tRunning mean: 0.7714\n",
      "Learner:3\tReward total:0\tRunning mean: 0.3913\n",
      "Learner:4\tReward total:0\tRunning mean: 1.312\n",
      "Learner:5\tReward total:0\tRunning mean: 4.694e-08\n",
      "Learner:6\tReward total:0\tRunning mean: 7.261\n",
      "Learner:7\tReward total:0\tRunning mean: 10.93\n",
      "Max Norms =  ['0.00', '0.00', '0.13', '0.00', '3.29', '0.01', '0.00', '0.28']\n",
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Load random agent 8\n",
      "....................................................................................................\n",
      "Episode 100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.1788\n",
      "Learner:1\tReward total:0\tRunning mean: 0.1507\n",
      "Learner:2\tReward total:0\tRunning mean: 0.639\n",
      "Learner:3\tReward total:1\tRunning mean: 0.4623\n",
      "Learner:4\tReward total:0\tRunning mean: 1.141\n",
      "Learner:5\tReward total:0\tRunning mean: 0.3843\n",
      "Learner:6\tReward total:2\tRunning mean: 2.436\n",
      "Learner:7\tReward total:13\tRunning mean: 2.749\n",
      "Max Norms =  ['217.33', '0.00', '237.51', '317.60', '294.42', '184.58', '198.46', '217.03']\n",
      "....................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:22\tRunning mean: 4.101\n",
      "Learner:1\tReward total:0\tRunning mean: 0.2759\n",
      "Learner:2\tReward total:18\tRunning mean: 4.308\n",
      "Learner:3\tReward total:28\tRunning mean: 3.746\n",
      "Learner:4\tReward total:21\tRunning mean: 7.804\n",
      "Learner:5\tReward total:0\tRunning mean: 1.757\n",
      "Learner:6\tReward total:74\tRunning mean: 19.04\n",
      "Learner:7\tReward total:33\tRunning mean: 17.93\n",
      "Max Norms =  ['142.00', '0.00', '224.80', '187.10', '236.74', '203.86', '226.96', '279.19']\n",
      "....................................................................................................\n",
      "Episode 300 complete\n",
      "Learner:0\tReward total:8\tRunning mean: 13.11\n",
      "Learner:1\tReward total:25\tRunning mean: 6.413\n",
      "Learner:2\tReward total:34\tRunning mean: 17.15\n",
      "Learner:3\tReward total:21\tRunning mean: 25.66\n",
      "Learner:4\tReward total:14\tRunning mean: 16.94\n",
      "Learner:5\tReward total:16\tRunning mean: 8.162\n",
      "Learner:6\tReward total:99\tRunning mean: 65.63\n",
      "Learner:7\tReward total:99\tRunning mean: 56.44\n",
      "Max Norms =  ['177.12', '189.74', '287.50', '412.38', '247.94', '383.97', '272.51', '235.98']\n",
      "....................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:28\tRunning mean: 23.66\n",
      "Learner:1\tReward total:41\tRunning mean: 30.61\n",
      "Learner:2\tReward total:55\tRunning mean: 30.44\n",
      "Learner:3\tReward total:66\tRunning mean: 53.39\n",
      "Learner:4\tReward total:17\tRunning mean: 32.71\n",
      "Learner:5\tReward total:78\tRunning mean: 34.71\n",
      "Learner:6\tReward total:92\tRunning mean: 83.48\n",
      "Learner:7\tReward total:116\tRunning mean: 80.06\n",
      "Max Norms =  ['231.19', '273.23', '203.95', '290.17', '241.65', '614.70', '257.91', '245.98']\n",
      "....................................................................................................\n",
      "Episode 500 complete\n",
      "Learner:0\tReward total:50\tRunning mean: 30.69\n",
      "Learner:1\tReward total:46\tRunning mean: 34.11\n",
      "Learner:2\tReward total:15\tRunning mean: 32.56\n",
      "Learner:3\tReward total:162\tRunning mean: 98.32\n",
      "Learner:4\tReward total:27\tRunning mean: 34.64\n",
      "Learner:5\tReward total:30\tRunning mean: 43.22\n",
      "Learner:6\tReward total:48\tRunning mean: 71.27\n",
      "Learner:7\tReward total:72\tRunning mean: 77.48\n",
      "Max Norms =  ['202.11', '291.73', '276.10', '340.98', '156.40', '241.68', '236.11', '204.46']\n",
      "....................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:22\tRunning mean: 25.51\n",
      "Learner:1\tReward total:73\tRunning mean: 33.98\n",
      "Learner:2\tReward total:25\tRunning mean: 31.3\n",
      "Learner:3\tReward total:145\tRunning mean: 136.5\n",
      "Learner:4\tReward total:25\tRunning mean: 34.89\n",
      "Learner:5\tReward total:10\tRunning mean: 41.23\n",
      "Learner:6\tReward total:61\tRunning mean: 60.08\n",
      "Learner:7\tReward total:65\tRunning mean: 68.29\n",
      "Max Norms =  ['209.06', '206.48', '210.88', '217.30', '454.26', '523.97', '246.00', '134.76']\n",
      "....................................................................................................\n",
      "Episode 700 complete\n",
      "Learner:0\tReward total:32\tRunning mean: 24.43\n",
      "Learner:1\tReward total:78\tRunning mean: 67.58\n",
      "Learner:2\tReward total:19\tRunning mean: 29.54\n",
      "Learner:3\tReward total:112\tRunning mean: 108.1\n",
      "Learner:4\tReward total:29\tRunning mean: 33.85\n",
      "Learner:5\tReward total:47\tRunning mean: 46.31\n",
      "Learner:6\tReward total:62\tRunning mean: 58.71\n",
      "Learner:7\tReward total:59\tRunning mean: 66.65\n",
      "Max Norms =  ['207.03', '176.27', '179.00', '306.55', '168.60', '162.74', '232.87', '136.00']\n",
      "....................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:31\tRunning mean: 20.83\n",
      "Learner:1\tReward total:79\tRunning mean: 79.02\n",
      "Learner:2\tReward total:20\tRunning mean: 24.72\n",
      "Learner:3\tReward total:76\tRunning mean: 76.68\n",
      "Learner:4\tReward total:59\tRunning mean: 31.87\n",
      "Learner:5\tReward total:68\tRunning mean: 61.17\n",
      "Learner:6\tReward total:66\tRunning mean: 51.98\n",
      "Learner:7\tReward total:50\tRunning mean: 65.85\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max Norms =  ['151.70', '114.28', '190.42', '184.54', '185.38', '177.19', '322.28', '126.96']\n",
      "....................................................................................................\n",
      "Episode 900 complete\n",
      "Learner:0\tReward total:6\tRunning mean: 18.54\n",
      "Learner:1\tReward total:78\tRunning mean: 79.66\n",
      "Learner:2\tReward total:6\tRunning mean: 21.82\n",
      "Learner:3\tReward total:76\tRunning mean: 64.01\n",
      "Learner:4\tReward total:19\tRunning mean: 31.32\n",
      "Learner:5\tReward total:29\tRunning mean: 63.12\n",
      "Learner:6\tReward total:68\tRunning mean: 53.39\n",
      "Learner:7\tReward total:32\tRunning mean: 53.0\n",
      "Max Norms =  ['172.01', '113.01', '168.59', '243.65', '102.74', '211.08', '255.89', '151.12']\n",
      "....................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:26\tRunning mean: 20.97\n",
      "Learner:1\tReward total:59\tRunning mean: 79.98\n",
      "Learner:2\tReward total:9\tRunning mean: 20.48\n",
      "Learner:3\tReward total:54\tRunning mean: 60.03\n",
      "Learner:4\tReward total:29\tRunning mean: 31.03\n",
      "Learner:5\tReward total:88\tRunning mean: 65.82\n",
      "Learner:6\tReward total:48\tRunning mean: 50.16\n",
      "Learner:7\tReward total:59\tRunning mean: 50.87\n",
      "Max Norms =  ['211.59', '121.17', '111.88', '147.54', '294.35', '154.09', '157.77', '229.55']\n",
      "....................................................................................................\n",
      "Episode 1100 complete\n",
      "Learner:0\tReward total:32\tRunning mean: 22.58\n",
      "Learner:1\tReward total:75\tRunning mean: 77.82\n",
      "Learner:2\tReward total:31\tRunning mean: 21.57\n",
      "Learner:3\tReward total:59\tRunning mean: 60.06\n",
      "Learner:4\tReward total:23\tRunning mean: 31.17\n",
      "Learner:5\tReward total:74\tRunning mean: 70.09\n",
      "Learner:6\tReward total:25\tRunning mean: 41.9\n",
      "Learner:7\tReward total:36\tRunning mean: 50.68\n",
      "Max Norms =  ['239.74', '198.94', '107.95', '216.07', '95.51', '201.33', '165.20', '87.78']\n",
      "....................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:26\tRunning mean: 23.53\n",
      "Learner:1\tReward total:96\tRunning mean: 76.29\n",
      "Learner:2\tReward total:21\tRunning mean: 20.64\n",
      "Learner:3\tReward total:62\tRunning mean: 64.3\n",
      "Learner:4\tReward total:67\tRunning mean: 34.23\n",
      "Learner:5\tReward total:74\tRunning mean: 70.6\n",
      "Learner:6\tReward total:29\tRunning mean: 43.9\n",
      "Learner:7\tReward total:36\tRunning mean: 52.18\n",
      "Max Norms =  ['201.50', '202.16', '195.78', '145.40', '125.83', '193.47', '206.23', '187.44']\n",
      "....................................................................................................\n",
      "Episode 1300 complete\n",
      "Learner:0\tReward total:15\tRunning mean: 25.77\n",
      "Learner:1\tReward total:44\tRunning mean: 81.21\n",
      "Learner:2\tReward total:13\tRunning mean: 22.7\n",
      "Learner:3\tReward total:15\tRunning mean: 48.83\n",
      "Learner:4\tReward total:109\tRunning mean: 40.37\n",
      "Learner:5\tReward total:59\tRunning mean: 70.03\n",
      "Learner:6\tReward total:58\tRunning mean: 54.32\n",
      "Learner:7\tReward total:55\tRunning mean: 57.01\n",
      "Max Norms =  ['172.03', '330.31', '70.64', '185.28', '198.44', '186.64', '346.58', '160.44']\n",
      "....................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:35\tRunning mean: 26.37\n",
      "Learner:1\tReward total:88\tRunning mean: 80.7\n",
      "Learner:2\tReward total:11\tRunning mean: 20.85\n",
      "Learner:3\tReward total:30\tRunning mean: 40.68\n",
      "Learner:4\tReward total:32\tRunning mean: 35.03\n",
      "Learner:5\tReward total:99\tRunning mean: 91.92\n",
      "Learner:6\tReward total:42\tRunning mean: 52.48\n",
      "Learner:7\tReward total:61\tRunning mean: 55.45\n",
      "Max Norms =  ['162.30', '174.17', '279.06', '139.09', '195.61', '250.63', '232.25', '197.88']\n",
      "....................................................................................................\n",
      "Episode 1500 complete\n",
      "Learner:0\tReward total:14\tRunning mean: 26.34\n",
      "Learner:1\tReward total:83\tRunning mean: 77.95\n",
      "Learner:2\tReward total:14\tRunning mean: 18.43\n",
      "Learner:3\tReward total:59\tRunning mean: 38.73\n",
      "Learner:4\tReward total:23\tRunning mean: 31.92\n",
      "Learner:5\tReward total:111\tRunning mean: 94.01\n",
      "Learner:6\tReward total:48\tRunning mean: 52.55\n",
      "Learner:7\tReward total:59\tRunning mean: 61.59\n",
      "Max Norms =  ['131.32', '185.97', '205.87', '146.75', '154.78', '257.37', '213.31', '195.62']\n",
      "....................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:19\tRunning mean: 25.3\n",
      "Learner:1\tReward total:84\tRunning mean: 74.46\n",
      "Learner:2\tReward total:14\tRunning mean: 18.34\n",
      "Learner:3\tReward total:53\tRunning mean: 50.12\n",
      "Learner:4\tReward total:27\tRunning mean: 30.85\n",
      "Learner:5\tReward total:84\tRunning mean: 80.42\n",
      "Learner:6\tReward total:47\tRunning mean: 47.06\n",
      "Learner:7\tReward total:34\tRunning mean: 60.74\n",
      "Max Norms =  ['152.31', '192.89', '117.31', '296.86', '184.45', '184.24', '283.19', '188.77']\n",
      "....................................................................................................\n",
      "Episode 1700 complete\n",
      "Learner:0\tReward total:23\tRunning mean: 26.2\n",
      "Learner:1\tReward total:59\tRunning mean: 77.3\n",
      "Learner:2\tReward total:25\tRunning mean: 17.97\n",
      "Learner:3\tReward total:41\tRunning mean: 51.81\n",
      "Learner:4\tReward total:37\tRunning mean: 34.47\n",
      "Learner:5\tReward total:86\tRunning mean: 84.12\n",
      "Learner:6\tReward total:57\tRunning mean: 45.24\n",
      "Learner:7\tReward total:66\tRunning mean: 53.52\n",
      "Max Norms =  ['132.56', '88.64', '243.99', '218.56', '163.98', '277.07', '230.74', '258.87']\n",
      "....................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:17\tRunning mean: 26.28\n",
      "Learner:1\tReward total:80\tRunning mean: 75.15\n",
      "Learner:2\tReward total:23\tRunning mean: 18.01\n",
      "Learner:3\tReward total:52\tRunning mean: 54.41\n",
      "Learner:4\tReward total:24\tRunning mean: 32.22\n",
      "Learner:5\tReward total:81\tRunning mean: 90.09\n",
      "Learner:6\tReward total:49\tRunning mean: 47.44\n",
      "Learner:7\tReward total:60\tRunning mean: 51.39\n",
      "Max Norms =  ['117.34', '214.30', '91.49', '209.72', '192.74', '308.92', '188.86', '379.21']\n",
      "....................................................................................................\n",
      "Episode 1900 complete\n",
      "Learner:0\tReward total:38\tRunning mean: 25.5\n",
      "Learner:1\tReward total:31\tRunning mean: 72.71\n",
      "Learner:2\tReward total:11\tRunning mean: 18.72\n",
      "Learner:3\tReward total:102\tRunning mean: 57.35\n",
      "Learner:4\tReward total:25\tRunning mean: 34.03\n",
      "Learner:5\tReward total:87\tRunning mean: 82.5\n",
      "Learner:6\tReward total:40\tRunning mean: 47.9\n",
      "Learner:7\tReward total:43\tRunning mean: 54.43\n",
      "Max Norms =  ['170.36', '283.52', '131.10', '216.19', '130.51', '179.40', '216.20', '270.19']\n",
      "....................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:50\tRunning mean: 29.03\n",
      "Learner:1\tReward total:75\tRunning mean: 70.15\n",
      "Learner:2\tReward total:22\tRunning mean: 17.34\n",
      "Learner:3\tReward total:61\tRunning mean: 54.46\n",
      "Learner:4\tReward total:14\tRunning mean: 31.85\n",
      "Learner:5\tReward total:83\tRunning mean: 85.71\n",
      "Learner:6\tReward total:78\tRunning mean: 50.04\n",
      "Learner:7\tReward total:45\tRunning mean: 47.31\n",
      "Max Norms =  ['138.86', '301.55', '150.38', '292.74', '97.74', '339.89', '207.48', '472.07']\n",
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Load random agent 8\n",
      "....................................................................................................\n",
      "Episode 100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.3867\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.7131\n",
      "Learner:3\tReward total:0\tRunning mean: 0.2691\n",
      "Learner:4\tReward total:0\tRunning mean: 0.2161\n",
      "Learner:5\tReward total:8\tRunning mean: 0.331\n",
      "Learner:6\tReward total:7\tRunning mean: 1.617\n",
      "Learner:7\tReward total:1\tRunning mean: 5.482\n",
      "Max Norms =  ['179.84', '300.97', '302.58', '225.99', '160.22', '195.43', '221.00', '333.40']\n",
      "....................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:20\tRunning mean: 3.035\n",
      "Learner:1\tReward total:0\tRunning mean: 0.04101\n",
      "Learner:2\tReward total:10\tRunning mean: 2.904\n",
      "Learner:3\tReward total:2\tRunning mean: 1.635\n",
      "Learner:4\tReward total:0\tRunning mean: 1.225\n",
      "Learner:5\tReward total:0\tRunning mean: 5.391\n",
      "Learner:6\tReward total:27\tRunning mean: 13.62\n",
      "Learner:7\tReward total:93\tRunning mean: 57.91\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max Norms =  ['156.75', '0.00', '198.85', '202.58', '163.86', '200.80', '179.78', '356.85']\n",
      "....................................................................................................\n",
      "Episode 300 complete\n",
      "Learner:0\tReward total:56\tRunning mean: 18.5\n",
      "Learner:1\tReward total:0\tRunning mean: 0.1352\n",
      "Learner:2\tReward total:64\tRunning mean: 18.39\n",
      "Learner:3\tReward total:23\tRunning mean: 7.133\n",
      "Learner:4\tReward total:11\tRunning mean: 5.869\n",
      "Learner:5\tReward total:17\tRunning mean: 18.54\n",
      "Learner:6\tReward total:89\tRunning mean: 57.02\n",
      "Learner:7\tReward total:114\tRunning mean: 110.1\n",
      "Max Norms =  ['475.06', '0.00', '186.40', '191.49', '356.62', '284.23', '174.26', '377.16']\n",
      "....................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:72\tRunning mean: 46.3\n",
      "Learner:1\tReward total:0\tRunning mean: 0.2357\n",
      "Learner:2\tReward total:39\tRunning mean: 37.37\n",
      "Learner:3\tReward total:58\tRunning mean: 22.66\n",
      "Learner:4\tReward total:51\tRunning mean: 26.47\n",
      "Learner:5\tReward total:48\tRunning mean: 39.93\n",
      "Learner:6\tReward total:107\tRunning mean: 84.61\n",
      "Learner:7\tReward total:116\tRunning mean: 105.6\n",
      "Max Norms =  ['205.41', '164.68', '219.15', '167.85', '215.81', '215.86', '296.54', '344.13']\n",
      "....................................................................................................\n",
      "Episode 500 complete\n",
      "Learner:0\tReward total:23\tRunning mean: 43.7\n",
      "Learner:1\tReward total:26\tRunning mean: 3.291\n",
      "Learner:2\tReward total:28\tRunning mean: 38.88\n",
      "Learner:3\tReward total:42\tRunning mean: 33.51\n",
      "Learner:4\tReward total:60\tRunning mean: 35.58\n",
      "Learner:5\tReward total:71\tRunning mean: 49.57\n",
      "Learner:6\tReward total:175\tRunning mean: 123.6\n",
      "Learner:7\tReward total:71\tRunning mean: 81.85\n",
      "Max Norms =  ['124.59', '171.37', '213.19', '186.11', '179.61', '245.89', '317.67', '272.77']\n",
      "....................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:42\tRunning mean: 43.52\n",
      "Learner:1\tReward total:31\tRunning mean: 18.81\n",
      "Learner:2\tReward total:59\tRunning mean: 40.57\n",
      "Learner:3\tReward total:19\tRunning mean: 32.03\n",
      "Learner:4\tReward total:70\tRunning mean: 48.88\n",
      "Learner:5\tReward total:25\tRunning mean: 46.89\n",
      "Learner:6\tReward total:66\tRunning mean: 124.3\n",
      "Learner:7\tReward total:16\tRunning mean: 73.54\n",
      "Max Norms =  ['135.81', '192.90', '174.96', '149.91', '249.61', '292.99', '197.75', '312.31']\n",
      "....................................................................................................\n",
      "Episode 700 complete\n",
      "Learner:0\tReward total:32\tRunning mean: 36.95\n",
      "Learner:1\tReward total:51\tRunning mean: 27.83\n",
      "Learner:2\tReward total:81\tRunning mean: 57.81\n",
      "Learner:3\tReward total:25\tRunning mean: 23.03\n",
      "Learner:4\tReward total:69\tRunning mean: 67.91\n",
      "Learner:5\tReward total:38\tRunning mean: 31.54\n",
      "Learner:6\tReward total:56\tRunning mean: 68.45\n",
      "Learner:7\tReward total:42\tRunning mean: 51.97\n",
      "Max Norms =  ['189.07', '141.20', '257.86', '139.75', '204.68', '358.39', '182.39', '237.63']\n",
      "....................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:62\tRunning mean: 43.17\n",
      "Learner:1\tReward total:58\tRunning mean: 37.87\n",
      "Learner:2\tReward total:63\tRunning mean: 74.85\n",
      "Learner:3\tReward total:3\tRunning mean: 22.1\n",
      "Learner:4\tReward total:71\tRunning mean: 71.86\n",
      "Learner:5\tReward total:54\tRunning mean: 29.22\n",
      "Learner:6\tReward total:31\tRunning mean: 46.12\n",
      "Learner:7\tReward total:33\tRunning mean: 53.98\n",
      "Max Norms =  ['181.73', '203.90', '240.74', '108.19', '220.46', '182.80', '268.61', '209.83']\n",
      "....................................................................................................\n",
      "Episode 900 complete\n",
      "Learner:0\tReward total:64\tRunning mean: 44.25\n",
      "Learner:1\tReward total:44\tRunning mean: 39.54\n",
      "Learner:2\tReward total:87\tRunning mean: 77.85\n",
      "Learner:3\tReward total:9\tRunning mean: 18.93\n",
      "Learner:4\tReward total:68\tRunning mean: 78.25\n",
      "Learner:5\tReward total:30\tRunning mean: 24.77\n",
      "Learner:6\tReward total:40\tRunning mean: 41.12\n",
      "Learner:7\tReward total:59\tRunning mean: 48.41\n",
      "Max Norms =  ['137.17', '114.13', '330.46', '141.56', '152.59', '273.66', '272.16', '189.39']\n",
      "....................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:38\tRunning mean: 46.56\n",
      "Learner:1\tReward total:49\tRunning mean: 42.99\n",
      "Learner:2\tReward total:69\tRunning mean: 82.13\n",
      "Learner:3\tReward total:29\tRunning mean: 16.47\n",
      "Learner:4\tReward total:84\tRunning mean: 80.66\n",
      "Learner:5\tReward total:27\tRunning mean: 18.5\n",
      "Learner:6\tReward total:84\tRunning mean: 41.22\n",
      "Learner:7\tReward total:18\tRunning mean: 47.36\n",
      "Max Norms =  ['147.25', '136.50', '226.26', '53.04', '229.66', '123.64', '159.92', '177.19']\n",
      "....................................................................................................\n",
      "Episode 1100 complete\n",
      "Learner:0\tReward total:40\tRunning mean: 45.68\n",
      "Learner:1\tReward total:40\tRunning mean: 48.08\n",
      "Learner:2\tReward total:98\tRunning mean: 84.23\n",
      "Learner:3\tReward total:12\tRunning mean: 15.5\n",
      "Learner:4\tReward total:93\tRunning mean: 83.99\n",
      "Learner:5\tReward total:13\tRunning mean: 17.99\n",
      "Learner:6\tReward total:38\tRunning mean: 37.46\n",
      "Learner:7\tReward total:36\tRunning mean: 49.16\n",
      "Max Norms =  ['162.38', '238.50', '262.66', '96.69', '254.33', '197.04', '237.91', '213.63']\n",
      "....................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:28\tRunning mean: 46.74\n",
      "Learner:1\tReward total:56\tRunning mean: 53.63\n",
      "Learner:2\tReward total:74\tRunning mean: 81.72\n",
      "Learner:3\tReward total:16\tRunning mean: 13.57\n",
      "Learner:4\tReward total:99\tRunning mean: 76.59\n",
      "Learner:5\tReward total:27\tRunning mean: 16.56\n",
      "Learner:6\tReward total:37\tRunning mean: 35.91\n",
      "Learner:7\tReward total:40\tRunning mean: 43.74\n",
      "Max Norms =  ['148.93', '228.41', '164.92', '54.06', '191.02', '91.53', '256.41', '219.30']\n",
      "....................................................................................................\n",
      "Episode 1300 complete\n",
      "Learner:0\tReward total:71\tRunning mean: 48.68\n",
      "Learner:1\tReward total:37\tRunning mean: 50.78\n",
      "Learner:2\tReward total:92\tRunning mean: 80.45\n",
      "Learner:3\tReward total:1\tRunning mean: 12.33\n",
      "Learner:4\tReward total:122\tRunning mean: 73.75\n",
      "Learner:5\tReward total:7\tRunning mean: 14.47\n",
      "Learner:6\tReward total:20\tRunning mean: 39.0\n",
      "Learner:7\tReward total:18\tRunning mean: 39.32\n",
      "Max Norms =  ['197.80', '143.18', '220.72', '52.41', '202.92', '313.09', '216.88', '162.72']\n",
      "....................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:54\tRunning mean: 48.25\n",
      "Learner:1\tReward total:62\tRunning mean: 53.99\n",
      "Learner:2\tReward total:88\tRunning mean: 88.09\n",
      "Learner:3\tReward total:9\tRunning mean: 13.06\n",
      "Learner:4\tReward total:81\tRunning mean: 73.84\n",
      "Learner:5\tReward total:16\tRunning mean: 13.73\n",
      "Learner:6\tReward total:31\tRunning mean: 36.48\n",
      "Learner:7\tReward total:68\tRunning mean: 40.43\n",
      "Max Norms =  ['196.39', '157.41', '200.44', '45.72', '185.21', '105.74', '236.69', '223.43']\n",
      "....................................................................................................\n",
      "Episode 1500 complete\n",
      "Learner:0\tReward total:60\tRunning mean: 51.72\n",
      "Learner:1\tReward total:86\tRunning mean: 64.81\n",
      "Learner:2\tReward total:103\tRunning mean: 91.36\n",
      "Learner:3\tReward total:19\tRunning mean: 11.76\n",
      "Learner:4\tReward total:57\tRunning mean: 73.07\n",
      "Learner:5\tReward total:16\tRunning mean: 14.56\n",
      "Learner:6\tReward total:34\tRunning mean: 35.22\n",
      "Learner:7\tReward total:41\tRunning mean: 43.72\n",
      "Max Norms =  ['193.61', '234.94', '247.73', '108.21', '219.35', '246.21', '201.25', '237.54']\n",
      "....................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:38\tRunning mean: 47.64\n",
      "Learner:1\tReward total:81\tRunning mean: 65.56\n",
      "Learner:2\tReward total:84\tRunning mean: 82.29\n",
      "Learner:3\tReward total:19\tRunning mean: 12.35\n",
      "Learner:4\tReward total:95\tRunning mean: 80.06\n",
      "Learner:5\tReward total:13\tRunning mean: 16.23\n",
      "Learner:6\tReward total:26\tRunning mean: 35.52\n",
      "Learner:7\tReward total:44\tRunning mean: 39.52\n",
      "Max Norms =  ['168.08', '186.83', '214.81', '144.93', '283.13', '87.97', '396.01', '265.48']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "....................................................................................................\n",
      "Episode 1700 complete\n",
      "Learner:0\tReward total:50\tRunning mean: 41.51\n",
      "Learner:1\tReward total:96\tRunning mean: 75.14\n",
      "Learner:2\tReward total:70\tRunning mean: 76.79\n",
      "Learner:3\tReward total:11\tRunning mean: 13.32\n",
      "Learner:4\tReward total:51\tRunning mean: 80.99\n",
      "Learner:5\tReward total:9\tRunning mean: 17.22\n",
      "Learner:6\tReward total:51\tRunning mean: 41.05\n",
      "Learner:7\tReward total:53\tRunning mean: 39.49\n",
      "Max Norms =  ['328.90', '223.80', '176.67', '50.75', '216.14', '130.33', '268.03', '214.34']\n",
      "....................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:61\tRunning mean: 41.04\n",
      "Learner:1\tReward total:95\tRunning mean: 78.32\n",
      "Learner:2\tReward total:80\tRunning mean: 81.57\n",
      "Learner:3\tReward total:12\tRunning mean: 12.32\n",
      "Learner:4\tReward total:81\tRunning mean: 72.06\n",
      "Learner:5\tReward total:3\tRunning mean: 15.8\n",
      "Learner:6\tReward total:35\tRunning mean: 41.11\n",
      "Learner:7\tReward total:57\tRunning mean: 35.9\n",
      "Max Norms =  ['216.12', '274.09', '226.50', '104.91', '170.22', '164.96', '201.77', '305.46']\n",
      "....................................................................................................\n",
      "Episode 1900 complete\n",
      "Learner:0\tReward total:37\tRunning mean: 45.12\n",
      "Learner:1\tReward total:89\tRunning mean: 79.04\n",
      "Learner:2\tReward total:107\tRunning mean: 90.62\n",
      "Learner:3\tReward total:3\tRunning mean: 9.796\n",
      "Learner:4\tReward total:73\tRunning mean: 72.46\n",
      "Learner:5\tReward total:25\tRunning mean: 16.08\n",
      "Learner:6\tReward total:36\tRunning mean: 37.99\n",
      "Learner:7\tReward total:31\tRunning mean: 39.78\n",
      "Max Norms =  ['107.11', '277.07', '390.50', '91.74', '202.08', '237.95', '401.05', '183.10']\n",
      "....................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:28\tRunning mean: 46.93\n",
      "Learner:1\tReward total:77\tRunning mean: 78.83\n",
      "Learner:2\tReward total:70\tRunning mean: 86.25\n",
      "Learner:3\tReward total:17\tRunning mean: 11.95\n",
      "Learner:4\tReward total:101\tRunning mean: 79.36\n",
      "Learner:5\tReward total:8\tRunning mean: 16.48\n",
      "Learner:6\tReward total:26\tRunning mean: 32.51\n",
      "Learner:7\tReward total:50\tRunning mean: 43.35\n",
      "Max Norms =  ['205.81', '235.08', '149.20', '74.00', '339.52', '132.77', '342.62', '332.07']\n",
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Load random agent 8\n",
      "....................................................................................................\n",
      "Episode 100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.3956\n",
      "Learner:1\tReward total:0\tRunning mean: 0.72\n",
      "Learner:2\tReward total:0\tRunning mean: 0.8289\n",
      "Learner:3\tReward total:0\tRunning mean: 0.1116\n",
      "Learner:4\tReward total:7\tRunning mean: 0.4718\n",
      "Learner:5\tReward total:7\tRunning mean: 0.4335\n",
      "Learner:6\tReward total:8\tRunning mean: 1.644\n",
      "Learner:7\tReward total:10\tRunning mean: 7.7\n",
      "Max Norms =  ['175.18', '227.38', '147.79', '276.13', '245.65', '212.72', '187.10', '233.27']\n",
      "....................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:27\tRunning mean: 5.638\n",
      "Learner:1\tReward total:26\tRunning mean: 3.801\n",
      "Learner:2\tReward total:11\tRunning mean: 5.32\n",
      "Learner:3\tReward total:6\tRunning mean: 1.605\n",
      "Learner:4\tReward total:19\tRunning mean: 3.09\n",
      "Learner:5\tReward total:9\tRunning mean: 5.398\n",
      "Learner:6\tReward total:52\tRunning mean: 20.83\n",
      "Learner:7\tReward total:48\tRunning mean: 67.65\n",
      "Max Norms =  ['164.52', '192.43', '199.68', '204.10', '194.23', '208.01', '232.67', '240.22']\n",
      "....................................................................................................\n",
      "Episode 300 complete\n",
      "Learner:0\tReward total:43\tRunning mean: 22.58\n",
      "Learner:1\tReward total:20\tRunning mean: 16.83\n",
      "Learner:2\tReward total:35\tRunning mean: 13.96\n",
      "Learner:3\tReward total:13\tRunning mean: 5.065\n",
      "Learner:4\tReward total:11\tRunning mean: 16.83\n",
      "Learner:5\tReward total:41\tRunning mean: 15.21\n",
      "Learner:6\tReward total:153\tRunning mean: 78.09\n",
      "Learner:7\tReward total:99\tRunning mean: 101.3\n",
      "Max Norms =  ['268.41', '247.73', '206.14', '203.37', '253.87', '253.28', '324.52', '264.06']\n",
      "....................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:40\tRunning mean: 36.26\n",
      "Learner:1\tReward total:30\tRunning mean: 25.52\n",
      "Learner:2\tReward total:23\tRunning mean: 20.84\n",
      "Learner:3\tReward total:19\tRunning mean: 14.42\n",
      "Learner:4\tReward total:147\tRunning mean: 75.94\n",
      "Learner:5\tReward total:25\tRunning mean: 26.59\n",
      "Learner:6\tReward total:29\tRunning mean: 70.51\n",
      "Learner:7\tReward total:85\tRunning mean: 82.46\n",
      "Max Norms =  ['205.23', '204.39', '191.66', '144.21', '301.79', '228.10', '249.31', '314.85']\n",
      "....................................................................................................\n",
      "Episode 500 complete\n",
      "Learner:0\tReward total:29\tRunning mean: 41.01\n",
      "Learner:1\tReward total:18\tRunning mean: 30.14\n",
      "Learner:2\tReward total:37\tRunning mean: 27.87\n",
      "Learner:3\tReward total:23\tRunning mean: 28.35\n",
      "Learner:4\tReward total:155\tRunning mean: 113.7\n",
      "Learner:5\tReward total:18\tRunning mean: 35.78\n",
      "Learner:6\tReward total:32\tRunning mean: 61.79\n",
      "Learner:7\tReward total:69\tRunning mean: 71.91\n",
      "Max Norms =  ['126.76', '166.55', '211.32', '156.99', '322.59', '161.73', '133.14', '190.44']\n",
      "....................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:47\tRunning mean: 38.08\n",
      "Learner:1\tReward total:38\tRunning mean: 33.1\n",
      "Learner:2\tReward total:26\tRunning mean: 27.69\n",
      "Learner:3\tReward total:38\tRunning mean: 30.3\n",
      "Learner:4\tReward total:171\tRunning mean: 146.2\n",
      "Learner:5\tReward total:36\tRunning mean: 33.5\n",
      "Learner:6\tReward total:61\tRunning mean: 49.25\n",
      "Learner:7\tReward total:43\tRunning mean: 64.01\n",
      "Max Norms =  ['253.33', '156.30', '193.28', '178.44', '300.55', '164.74', '220.92', '272.16']\n",
      "....................................................................................................\n",
      "Episode 700 complete\n",
      "Learner:0\tReward total:44\tRunning mean: 37.77\n",
      "Learner:1\tReward total:46\tRunning mean: 33.53\n",
      "Learner:2\tReward total:24\tRunning mean: 26.97\n",
      "Learner:3\tReward total:35\tRunning mean: 28.84\n",
      "Learner:4\tReward total:139\tRunning mean: 169.6\n",
      "Learner:5\tReward total:31\tRunning mean: 29.78\n",
      "Learner:6\tReward total:45\tRunning mean: 44.7\n",
      "Learner:7\tReward total:99\tRunning mean: 68.19\n",
      "Max Norms =  ['274.23', '138.83', '212.92', '84.50', '338.37', '127.48', '190.25', '344.68']\n",
      "....................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:47\tRunning mean: 35.33\n",
      "Learner:1\tReward total:26\tRunning mean: 32.12\n",
      "Learner:2\tReward total:29\tRunning mean: 28.6\n",
      "Learner:3\tReward total:23\tRunning mean: 30.91\n",
      "Learner:4\tReward total:173\tRunning mean: 143.6\n",
      "Learner:5\tReward total:21\tRunning mean: 28.44\n",
      "Learner:6\tReward total:25\tRunning mean: 40.99\n",
      "Learner:7\tReward total:75\tRunning mean: 92.32\n",
      "Max Norms =  ['160.75', '64.24', '134.41', '108.89', '326.32', '115.71', '187.06', '300.73']\n",
      "....................................................................................................\n",
      "Episode 900 complete\n",
      "Learner:0\tReward total:15\tRunning mean: 29.61\n",
      "Learner:1\tReward total:31\tRunning mean: 28.78\n",
      "Learner:2\tReward total:13\tRunning mean: 26.74\n",
      "Learner:3\tReward total:100\tRunning mean: 41.19\n",
      "Learner:4\tReward total:123\tRunning mean: 137.4\n",
      "Learner:5\tReward total:12\tRunning mean: 24.05\n",
      "Learner:6\tReward total:32\tRunning mean: 37.72\n",
      "Learner:7\tReward total:63\tRunning mean: 91.44\n",
      "Max Norms =  ['126.53', '98.27', '157.89', '173.53', '223.07', '61.87', '269.34', '277.25']\n",
      "....................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:34\tRunning mean: 30.15\n",
      "Learner:1\tReward total:33\tRunning mean: 29.44\n",
      "Learner:2\tReward total:11\tRunning mean: 23.21\n",
      "Learner:3\tReward total:32\tRunning mean: 65.0\n",
      "Learner:4\tReward total:169\tRunning mean: 114.8\n",
      "Learner:5\tReward total:13\tRunning mean: 19.48\n",
      "Learner:6\tReward total:33\tRunning mean: 33.65\n",
      "Learner:7\tReward total:66\tRunning mean: 81.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max Norms =  ['176.75', '84.17', '139.82', '138.96', '308.97', '66.52', '154.82', '212.29']\n",
      "....................................................................................................\n",
      "Episode 1100 complete\n",
      "Learner:0\tReward total:16\tRunning mean: 28.12\n",
      "Learner:1\tReward total:14\tRunning mean: 27.88\n",
      "Learner:2\tReward total:13\tRunning mean: 22.72\n",
      "Learner:3\tReward total:92\tRunning mean: 81.14\n",
      "Learner:4\tReward total:123\tRunning mean: 110.9\n",
      "Learner:5\tReward total:2\tRunning mean: 18.26\n",
      "Learner:6\tReward total:46\tRunning mean: 33.18\n",
      "Learner:7\tReward total:83\tRunning mean: 75.92\n",
      "Max Norms =  ['129.14', '54.71', '136.10', '129.72', '309.87', '37.07', '147.62', '287.20']\n",
      "....................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:28\tRunning mean: 29.14\n",
      "Learner:1\tReward total:31\tRunning mean: 27.82\n",
      "Learner:2\tReward total:15\tRunning mean: 20.83\n",
      "Learner:3\tReward total:40\tRunning mean: 84.79\n",
      "Learner:4\tReward total:149\tRunning mean: 112.5\n",
      "Learner:5\tReward total:19\tRunning mean: 16.9\n",
      "Learner:6\tReward total:38\tRunning mean: 29.88\n",
      "Learner:7\tReward total:82\tRunning mean: 73.69\n",
      "Max Norms =  ['180.88', '49.63', '138.19', '185.97', '284.14', '118.84', '234.07', '253.98']\n",
      "....................................................................................................\n",
      "Episode 1300 complete\n",
      "Learner:0\tReward total:34\tRunning mean: 26.9\n",
      "Learner:1\tReward total:24\tRunning mean: 25.28\n",
      "Learner:2\tReward total:24\tRunning mean: 22.18\n",
      "Learner:3\tReward total:59\tRunning mean: 87.52\n",
      "Learner:4\tReward total:159\tRunning mean: 112.6\n",
      "Learner:5\tReward total:13\tRunning mean: 17.22\n",
      "Learner:6\tReward total:20\tRunning mean: 29.19\n",
      "Learner:7\tReward total:77\tRunning mean: 76.15\n",
      "Max Norms =  ['191.63', '46.13', '72.21', '86.36', '462.93', '73.15', '141.53', '324.67']\n",
      "....................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:20\tRunning mean: 27.94\n",
      "Learner:1\tReward total:29\tRunning mean: 25.32\n",
      "Learner:2\tReward total:17\tRunning mean: 22.62\n",
      "Learner:3\tReward total:94\tRunning mean: 94.91\n",
      "Learner:4\tReward total:100\tRunning mean: 104.2\n",
      "Learner:5\tReward total:12\tRunning mean: 16.35\n",
      "Learner:6\tReward total:24\tRunning mean: 31.17\n",
      "Learner:7\tReward total:48\tRunning mean: 86.25\n",
      "Max Norms =  ['189.86', '54.79', '88.45', '120.78', '338.27', '63.66', '182.59', '198.53']\n",
      "....................................................................................................\n",
      "Episode 1500 complete\n",
      "Learner:0\tReward total:50\tRunning mean: 26.02\n",
      "Learner:1\tReward total:30\tRunning mean: 25.64\n",
      "Learner:2\tReward total:23\tRunning mean: 21.65\n",
      "Learner:3\tReward total:127\tRunning mean: 89.47\n",
      "Learner:4\tReward total:83\tRunning mean: 105.9\n",
      "Learner:5\tReward total:10\tRunning mean: 16.33\n",
      "Learner:6\tReward total:22\tRunning mean: 29.33\n",
      "Learner:7\tReward total:40\tRunning mean: 84.03\n",
      "Max Norms =  ['140.34', '73.51', '290.82', '152.26', '390.66', '89.58', '156.46', '233.88']\n",
      "....................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:25\tRunning mean: 26.82\n",
      "Learner:1\tReward total:13\tRunning mean: 25.94\n",
      "Learner:2\tReward total:14\tRunning mean: 22.48\n",
      "Learner:3\tReward total:59\tRunning mean: 84.01\n",
      "Learner:4\tReward total:148\tRunning mean: 106.0\n",
      "Learner:5\tReward total:10\tRunning mean: 17.86\n",
      "Learner:6\tReward total:46\tRunning mean: 28.78\n",
      "Learner:7\tReward total:92\tRunning mean: 87.08\n",
      "Max Norms =  ['175.42', '40.82', '135.98', '65.92', '496.01', '133.32', '188.31', '274.80']\n",
      "....................................................................................................\n",
      "Episode 1700 complete\n",
      "Learner:0\tReward total:19\tRunning mean: 25.76\n",
      "Learner:1\tReward total:29\tRunning mean: 23.68\n",
      "Learner:2\tReward total:17\tRunning mean: 22.82\n",
      "Learner:3\tReward total:164\tRunning mean: 92.09\n",
      "Learner:4\tReward total:58\tRunning mean: 108.6\n",
      "Learner:5\tReward total:9\tRunning mean: 18.22\n",
      "Learner:6\tReward total:15\tRunning mean: 27.97\n",
      "Learner:7\tReward total:89\tRunning mean: 88.27\n",
      "Max Norms =  ['212.06', '196.75', '170.60', '414.31', '297.88', '67.00', '269.32', '393.00']\n",
      "....................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:36\tRunning mean: 27.32\n",
      "Learner:1\tReward total:18\tRunning mean: 22.94\n",
      "Learner:2\tReward total:15\tRunning mean: 24.75\n",
      "Learner:3\tReward total:88\tRunning mean: 97.4\n",
      "Learner:4\tReward total:167\tRunning mean: 106.1\n",
      "Learner:5\tReward total:15\tRunning mean: 18.45\n",
      "Learner:6\tReward total:17\tRunning mean: 28.91\n",
      "Learner:7\tReward total:73\tRunning mean: 84.66\n",
      "Max Norms =  ['248.02', '123.81', '173.48', '140.22', '425.77', '35.20', '124.41', '276.64']\n",
      "....................................................................................................\n",
      "Episode 1900 complete\n",
      "Learner:0\tReward total:32\tRunning mean: 27.72\n",
      "Learner:1\tReward total:25\tRunning mean: 22.6\n",
      "Learner:2\tReward total:33\tRunning mean: 27.2\n",
      "Learner:3\tReward total:55\tRunning mean: 78.74\n",
      "Learner:4\tReward total:152\tRunning mean: 114.4\n",
      "Learner:5\tReward total:16\tRunning mean: 18.65\n",
      "Learner:6\tReward total:29\tRunning mean: 26.53\n",
      "Learner:7\tReward total:93\tRunning mean: 89.27\n",
      "Max Norms =  ['131.14', '63.63', '196.90', '111.25', '433.68', '126.76', '163.47', '377.64']\n",
      "....................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:33\tRunning mean: 26.35\n",
      "Learner:1\tReward total:35\tRunning mean: 22.71\n",
      "Learner:2\tReward total:51\tRunning mean: 30.29\n",
      "Learner:3\tReward total:65\tRunning mean: 74.39\n",
      "Learner:4\tReward total:101\tRunning mean: 117.9\n",
      "Learner:5\tReward total:29\tRunning mean: 19.96\n",
      "Learner:6\tReward total:35\tRunning mean: 27.5\n",
      "Learner:7\tReward total:109\tRunning mean: 85.26\n",
      "Max Norms =  ['197.02', '136.33', '302.71', '152.44', '343.66', '303.24', '322.00', '252.19']\n",
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Load random agent 8\n",
      "....................................................................................................\n",
      "Episode 100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.219\n",
      "Learner:1\tReward total:0\tRunning mean: 0.02572\n",
      "Learner:2\tReward total:0\tRunning mean: 0.5616\n",
      "Learner:3\tReward total:1\tRunning mean: 0.2186\n",
      "Learner:4\tReward total:0\tRunning mean: 0.5373\n",
      "Learner:5\tReward total:3\tRunning mean: 0.3639\n",
      "Learner:6\tReward total:0\tRunning mean: 0.4161\n",
      "Learner:7\tReward total:15\tRunning mean: 2.139\n",
      "Max Norms =  ['247.89', '224.04', '227.29', '203.39', '176.13', '165.77', '312.54', '190.42']\n",
      "....................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:8\tRunning mean: 1.723\n",
      "Learner:1\tReward total:3\tRunning mean: 0.5675\n",
      "Learner:2\tReward total:4\tRunning mean: 1.595\n",
      "Learner:3\tReward total:0\tRunning mean: 1.208\n",
      "Learner:4\tReward total:13\tRunning mean: 5.835\n",
      "Learner:5\tReward total:14\tRunning mean: 4.305\n",
      "Learner:6\tReward total:17\tRunning mean: 4.462\n",
      "Learner:7\tReward total:49\tRunning mean: 18.12\n",
      "Max Norms =  ['252.89', '188.52', '155.44', '180.20', '194.34', '184.74', '213.78', '302.31']\n",
      "....................................................................................................\n",
      "Episode 300 complete\n",
      "Learner:0\tReward total:34\tRunning mean: 9.803\n",
      "Learner:1\tReward total:11\tRunning mean: 5.056\n",
      "Learner:2\tReward total:43\tRunning mean: 15.17\n",
      "Learner:3\tReward total:13\tRunning mean: 2.97\n",
      "Learner:4\tReward total:73\tRunning mean: 71.28\n",
      "Learner:5\tReward total:88\tRunning mean: 30.46\n",
      "Learner:6\tReward total:13\tRunning mean: 10.29\n",
      "Learner:7\tReward total:70\tRunning mean: 57.1\n",
      "Max Norms =  ['207.16', '200.98', '207.03', '226.12', '206.74', '256.84', '177.88', '178.19']\n",
      "....................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:33\tRunning mean: 30.0\n",
      "Learner:1\tReward total:32\tRunning mean: 15.38\n",
      "Learner:2\tReward total:134\tRunning mean: 54.77\n",
      "Learner:3\tReward total:16\tRunning mean: 7.139\n",
      "Learner:4\tReward total:71\tRunning mean: 98.52\n",
      "Learner:5\tReward total:38\tRunning mean: 44.5\n",
      "Learner:6\tReward total:48\tRunning mean: 25.46\n",
      "Learner:7\tReward total:44\tRunning mean: 55.86\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max Norms =  ['120.51', '246.55', '243.80', '167.24', '316.38', '286.87', '314.73', '344.48']\n",
      "....................................................................................................\n",
      "Episode 500 complete\n",
      "Learner:0\tReward total:22\tRunning mean: 32.3\n",
      "Learner:1\tReward total:38\tRunning mean: 22.07\n",
      "Learner:2\tReward total:64\tRunning mean: 85.31\n",
      "Learner:3\tReward total:12\tRunning mean: 12.65\n",
      "Learner:4\tReward total:119\tRunning mean: 89.41\n",
      "Learner:5\tReward total:19\tRunning mean: 37.92\n",
      "Learner:6\tReward total:32\tRunning mean: 26.7\n",
      "Learner:7\tReward total:69\tRunning mean: 63.04\n",
      "Max Norms =  ['135.98', '217.41', '264.24', '212.71', '338.97', '159.11', '218.48', '301.26']\n",
      "....................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:30\tRunning mean: 29.95\n",
      "Learner:1\tReward total:44\tRunning mean: 23.36\n",
      "Learner:2\tReward total:65\tRunning mean: 84.54\n",
      "Learner:3\tReward total:22\tRunning mean: 16.15\n",
      "Learner:4\tReward total:122\tRunning mean: 84.39\n",
      "Learner:5\tReward total:37\tRunning mean: 33.46\n",
      "Learner:6\tReward total:20\tRunning mean: 25.32\n",
      "Learner:7\tReward total:51\tRunning mean: 65.04\n",
      "Max Norms =  ['97.64', '268.58', '183.95', '262.16', '232.82', '169.08', '227.85', '253.00']\n",
      "....................................................................................................\n",
      "Episode 700 complete\n",
      "Learner:0\tReward total:42\tRunning mean: 31.06\n",
      "Learner:1\tReward total:40\tRunning mean: 24.99\n",
      "Learner:2\tReward total:65\tRunning mean: 84.93\n",
      "Learner:3\tReward total:1\tRunning mean: 17.16\n",
      "Learner:4\tReward total:93\tRunning mean: 85.4\n",
      "Learner:5\tReward total:52\tRunning mean: 41.64\n",
      "Learner:6\tReward total:57\tRunning mean: 24.86\n",
      "Learner:7\tReward total:56\tRunning mean: 56.58\n",
      "Max Norms =  ['137.83', '230.12', '221.02', '184.95', '283.56', '216.26', '224.86', '172.93']\n",
      "....................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:33\tRunning mean: 34.03\n",
      "Learner:1\tReward total:14\tRunning mean: 23.5\n",
      "Learner:2\tReward total:50\tRunning mean: 85.06\n",
      "Learner:3\tReward total:25\tRunning mean: 20.23\n",
      "Learner:4\tReward total:73\tRunning mean: 76.53\n",
      "Learner:5\tReward total:42\tRunning mean: 49.25\n",
      "Learner:6\tReward total:36\tRunning mean: 24.07\n",
      "Learner:7\tReward total:89\tRunning mean: 51.5\n",
      "Max Norms =  ['95.25', '108.28', '196.51', '167.40', '217.27', '276.85', '262.98', '285.19']\n",
      "....................................................................................................\n",
      "Episode 900 complete\n",
      "Learner:0\tReward total:23\tRunning mean: 36.01\n",
      "Learner:1\tReward total:30\tRunning mean: 25.37\n",
      "Learner:2\tReward total:56\tRunning mean: 83.48\n",
      "Learner:3\tReward total:8\tRunning mean: 20.54\n",
      "Learner:4\tReward total:80\tRunning mean: 75.71\n",
      "Learner:5\tReward total:84\tRunning mean: 52.24\n",
      "Learner:6\tReward total:14\tRunning mean: 21.02\n",
      "Learner:7\tReward total:51\tRunning mean: 56.04\n",
      "Max Norms =  ['77.01', '444.72', '175.63', '143.97', '369.09', '170.32', '469.01', '210.92']\n",
      "....................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:34\tRunning mean: 36.05\n",
      "Learner:1\tReward total:16\tRunning mean: 29.1\n",
      "Learner:2\tReward total:123\tRunning mean: 87.09\n",
      "Learner:3\tReward total:22\tRunning mean: 22.31\n",
      "Learner:4\tReward total:37\tRunning mean: 72.16\n",
      "Learner:5\tReward total:68\tRunning mean: 49.01\n",
      "Learner:6\tReward total:21\tRunning mean: 19.15\n",
      "Learner:7\tReward total:59\tRunning mean: 56.12\n",
      "Max Norms =  ['78.52', '280.34', '315.36', '199.64', '304.36', '335.59', '180.57', '245.18']\n",
      "....................................................................................................\n",
      "Episode 1100 complete\n",
      "Learner:0\tReward total:36\tRunning mean: 35.08\n",
      "Learner:1\tReward total:26\tRunning mean: 30.29\n",
      "Learner:2\tReward total:105\tRunning mean: 84.83\n",
      "Learner:3\tReward total:24\tRunning mean: 24.09\n",
      "Learner:4\tReward total:120\tRunning mean: 64.67\n",
      "Learner:5\tReward total:34\tRunning mean: 53.32\n",
      "Learner:6\tReward total:1\tRunning mean: 17.04\n",
      "Learner:7\tReward total:36\tRunning mean: 54.18\n",
      "Max Norms =  ['121.16', '141.26', '190.75', '339.93', '418.56', '115.37', '97.96', '236.34']\n",
      "....................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:31\tRunning mean: 37.47\n",
      "Learner:1\tReward total:15\tRunning mean: 26.12\n",
      "Learner:2\tReward total:131\tRunning mean: 85.42\n",
      "Learner:3\tReward total:16\tRunning mean: 24.33\n",
      "Learner:4\tReward total:49\tRunning mean: 63.92\n",
      "Learner:5\tReward total:36\tRunning mean: 65.22\n",
      "Learner:6\tReward total:18\tRunning mean: 18.21\n",
      "Learner:7\tReward total:33\tRunning mean: 44.14\n",
      "Max Norms =  ['130.53', '94.87', '296.09', '136.36', '227.93', '192.78', '171.01', '144.37']\n",
      "....................................................................................................\n",
      "Episode 1300 complete\n",
      "Learner:0\tReward total:44\tRunning mean: 38.9\n",
      "Learner:1\tReward total:10\tRunning mean: 22.74\n",
      "Learner:2\tReward total:47\tRunning mean: 82.95\n",
      "Learner:3\tReward total:21\tRunning mean: 22.3\n",
      "Learner:4\tReward total:62\tRunning mean: 60.16\n",
      "Learner:5\tReward total:39\tRunning mean: 53.83\n",
      "Learner:6\tReward total:7\tRunning mean: 17.08\n",
      "Learner:7\tReward total:38\tRunning mean: 52.0\n",
      "Max Norms =  ['226.61', '76.51', '209.61', '55.02', '192.49', '139.69', '96.62', '194.19']\n",
      "....................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:41\tRunning mean: 36.91\n",
      "Learner:1\tReward total:31\tRunning mean: 21.34\n",
      "Learner:2\tReward total:107\tRunning mean: 84.74\n",
      "Learner:3\tReward total:18\tRunning mean: 22.56\n",
      "Learner:4\tReward total:31\tRunning mean: 51.41\n",
      "Learner:5\tReward total:86\tRunning mean: 55.11\n",
      "Learner:6\tReward total:22\tRunning mean: 15.53\n",
      "Learner:7\tReward total:41\tRunning mean: 45.66\n",
      "Max Norms =  ['147.80', '192.71', '319.33', '74.97', '207.53', '221.73', '169.79', '326.39']\n",
      "....................................................................................................\n",
      "Episode 1500 complete\n",
      "Learner:0\tReward total:30\tRunning mean: 32.07\n",
      "Learner:1\tReward total:30\tRunning mean: 27.51\n",
      "Learner:2\tReward total:62\tRunning mean: 67.29\n",
      "Learner:3\tReward total:43\tRunning mean: 20.92\n",
      "Learner:4\tReward total:58\tRunning mean: 61.17\n",
      "Learner:5\tReward total:32\tRunning mean: 57.18\n",
      "Learner:6\tReward total:17\tRunning mean: 16.17\n",
      "Learner:7\tReward total:59\tRunning mean: 55.48\n",
      "Max Norms =  ['201.98', '86.75', '170.35', '206.13', '274.44', '188.03', '82.62', '169.58']\n",
      "....................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:52\tRunning mean: 35.95\n",
      "Learner:1\tReward total:17\tRunning mean: 27.57\n",
      "Learner:2\tReward total:44\tRunning mean: 64.42\n",
      "Learner:3\tReward total:30\tRunning mean: 22.7\n",
      "Learner:4\tReward total:77\tRunning mean: 64.74\n",
      "Learner:5\tReward total:23\tRunning mean: 53.44\n",
      "Learner:6\tReward total:20\tRunning mean: 15.7\n",
      "Learner:7\tReward total:84\tRunning mean: 65.55\n",
      "Max Norms =  ['144.35', '134.87', '227.39', '135.92', '228.22', '149.04', '189.00', '132.54']\n",
      "....................................................................................................\n",
      "Episode 1700 complete\n",
      "Learner:0\tReward total:50\tRunning mean: 34.06\n",
      "Learner:1\tReward total:17\tRunning mean: 26.52\n",
      "Learner:2\tReward total:57\tRunning mean: 65.89\n",
      "Learner:3\tReward total:39\tRunning mean: 23.85\n",
      "Learner:4\tReward total:97\tRunning mean: 70.74\n",
      "Learner:5\tReward total:45\tRunning mean: 52.4\n",
      "Learner:6\tReward total:1\tRunning mean: 17.78\n",
      "Learner:7\tReward total:66\tRunning mean: 70.49\n",
      "Max Norms =  ['129.49', '113.83', '372.42', '168.11', '199.90', '100.99', '67.30', '228.81']\n",
      "....................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:22\tRunning mean: 30.46\n",
      "Learner:1\tReward total:30\tRunning mean: 20.63\n",
      "Learner:2\tReward total:69\tRunning mean: 68.44\n",
      "Learner:3\tReward total:25\tRunning mean: 25.55\n",
      "Learner:4\tReward total:70\tRunning mean: 65.24\n",
      "Learner:5\tReward total:82\tRunning mean: 64.38\n",
      "Learner:6\tReward total:26\tRunning mean: 16.41\n",
      "Learner:7\tReward total:65\tRunning mean: 71.85\n",
      "Max Norms =  ['93.25', '321.80', '179.94', '96.85', '223.89', '249.42', '272.56', '190.25']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "....................................................................................................\n",
      "Episode 1900 complete\n",
      "Learner:0\tReward total:34\tRunning mean: 26.36\n",
      "Learner:1\tReward total:34\tRunning mean: 25.53\n",
      "Learner:2\tReward total:54\tRunning mean: 60.75\n",
      "Learner:3\tReward total:44\tRunning mean: 25.34\n",
      "Learner:4\tReward total:56\tRunning mean: 64.24\n",
      "Learner:5\tReward total:52\tRunning mean: 64.26\n",
      "Learner:6\tReward total:19\tRunning mean: 15.32\n",
      "Learner:7\tReward total:81\tRunning mean: 81.0\n",
      "Max Norms =  ['133.56', '93.41', '169.74', '193.53', '257.74', '187.74', '115.09', '241.58']\n",
      "....................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:58\tRunning mean: 29.89\n",
      "Learner:1\tReward total:25\tRunning mean: 24.39\n",
      "Learner:2\tReward total:102\tRunning mean: 68.87\n",
      "Learner:3\tReward total:17\tRunning mean: 25.89\n",
      "Learner:4\tReward total:119\tRunning mean: 64.89\n",
      "Learner:5\tReward total:63\tRunning mean: 59.15\n",
      "Learner:6\tReward total:6\tRunning mean: 14.38\n",
      "Learner:7\tReward total:42\tRunning mean: 79.74\n",
      "Max Norms =  ['213.79', '76.06', '312.13', '57.80', '247.38', '161.69', '245.33', '241.43']\n",
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Load random agent 8\n",
      "....................................................................................................\n",
      "Episode 100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.02647\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.05212\n",
      "Learner:4\tReward total:0\tRunning mean: 0.01867\n",
      "Learner:5\tReward total:0\tRunning mean: 0.1765\n",
      "Learner:6\tReward total:0\tRunning mean: 0.05708\n",
      "Learner:7\tReward total:0\tRunning mean: 0.6564\n",
      "Max Norms =  ['182.95', '362.16', '246.36', '237.95', '168.85', '201.50', '145.22', '304.24']\n",
      "....................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.271\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.01908\n",
      "Learner:4\tReward total:0\tRunning mean: 0.01061\n",
      "Learner:5\tReward total:0\tRunning mean: 0.0646\n",
      "Learner:6\tReward total:0\tRunning mean: 0.02089\n",
      "Learner:7\tReward total:0\tRunning mean: 3.407\n",
      "Max Norms =  ['0.00', '208.53', '85.50', '47.47', '24.50', '62.16', '94.58', '176.35']\n",
      "....................................................................................................\n",
      "Episode 300 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0992\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.006984\n",
      "Learner:4\tReward total:0\tRunning mean: 0.003883\n",
      "Learner:5\tReward total:0\tRunning mean: 0.02364\n",
      "Learner:6\tReward total:0\tRunning mean: 0.007648\n",
      "Learner:7\tReward total:0\tRunning mean: 1.247\n",
      "Max Norms =  ['101.70', '32.64', '12.05', '14.31', '19.26', '18.23', '5.15', '40.84']\n",
      "....................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.03631\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.002556\n",
      "Learner:4\tReward total:0\tRunning mean: 0.001421\n",
      "Learner:5\tReward total:0\tRunning mean: 0.008655\n",
      "Learner:6\tReward total:0\tRunning mean: 0.002799\n",
      "Learner:7\tReward total:0\tRunning mean: 0.4565\n",
      "Max Norms =  ['21.01', '3.90', '14.29', '9.28', '11.85', '31.88', '9.64', '3.52']\n",
      "....................................................................................................\n",
      "Episode 500 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.01329\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0009357\n",
      "Learner:4\tReward total:0\tRunning mean: 0.0005202\n",
      "Learner:5\tReward total:0\tRunning mean: 0.003168\n",
      "Learner:6\tReward total:0\tRunning mean: 0.001025\n",
      "Learner:7\tReward total:0\tRunning mean: 0.1671\n",
      "Max Norms =  ['16.92', '7.97', '15.32', '16.42', '18.32', '1.52', '0.04', '0.16']\n",
      "....................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.004865\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0003425\n",
      "Learner:4\tReward total:0\tRunning mean: 0.0001904\n",
      "Learner:5\tReward total:0\tRunning mean: 0.00116\n",
      "Learner:6\tReward total:0\tRunning mean: 0.0003751\n",
      "Learner:7\tReward total:0\tRunning mean: 0.06116\n",
      "Max Norms =  ['8.55', '2.14', '0.07', '47.33', '6.03', '3.91', '13.27', '14.10']\n",
      "....................................................................................................\n",
      "Episode 700 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.001781\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0001254\n",
      "Learner:4\tReward total:0\tRunning mean: 6.97e-05\n",
      "Learner:5\tReward total:0\tRunning mean: 0.0004244\n",
      "Learner:6\tReward total:0\tRunning mean: 0.0001373\n",
      "Learner:7\tReward total:0\tRunning mean: 0.02238\n",
      "Max Norms =  ['8.34', '0.03', '0.56', '9.34', '0.28', '25.82', '3.35', '16.47']\n",
      "....................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0006518\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 4.589e-05\n",
      "Learner:4\tReward total:0\tRunning mean: 2.551e-05\n",
      "Learner:5\tReward total:0\tRunning mean: 0.0001554\n",
      "Learner:6\tReward total:0\tRunning mean: 5.025e-05\n",
      "Learner:7\tReward total:0\tRunning mean: 0.008194\n",
      "Max Norms =  ['0.16', '0.02', '0.02', '12.17', '0.03', '0.52', '0.03', '0.06']\n",
      "....................................................................................................\n",
      "Episode 900 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0002386\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 1.68e-05\n",
      "Learner:4\tReward total:0\tRunning mean: 9.338e-06\n",
      "Learner:5\tReward total:0\tRunning mean: 5.687e-05\n",
      "Learner:6\tReward total:0\tRunning mean: 1.839e-05\n",
      "Learner:7\tReward total:0\tRunning mean: 0.002999\n",
      "Max Norms =  ['0.05', '0.02', '12.50', '3.98', '0.02', '1.51', '0.02', '0.04']\n",
      "....................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 8.733e-05\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 6.148e-06\n",
      "Learner:4\tReward total:0\tRunning mean: 3.418e-06\n",
      "Learner:5\tReward total:0\tRunning mean: 2.081e-05\n",
      "Learner:6\tReward total:0\tRunning mean: 6.733e-06\n",
      "Learner:7\tReward total:0\tRunning mean: 0.001098\n",
      "Max Norms =  ['0.04', '0.02', '0.01', '0.04', '0.01', '0.01', '0.01', '12.25']\n",
      "....................................................................................................\n",
      "Episode 1100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 3.196e-05\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 2.25e-06\n",
      "Learner:4\tReward total:0\tRunning mean: 1.251e-06\n",
      "Learner:5\tReward total:0\tRunning mean: 7.619e-06\n",
      "Learner:6\tReward total:0\tRunning mean: 2.464e-06\n",
      "Learner:7\tReward total:0\tRunning mean: 0.0004018\n",
      "Max Norms =  ['0.03', '0.01', '0.01', '0.02', '0.12', '0.00', '0.01', '0.05']\n",
      "....................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.17e-05\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 8.237e-07\n",
      "Learner:4\tReward total:0\tRunning mean: 4.579e-07\n",
      "Learner:5\tReward total:0\tRunning mean: 2.789e-06\n",
      "Learner:6\tReward total:0\tRunning mean: 9.021e-07\n",
      "Learner:7\tReward total:0\tRunning mean: 0.0001471\n",
      "Max Norms =  ['0.02', '0.01', '0.01', '0.01', '0.01', '0.00', '0.01', '0.03']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "....................................................................................................\n",
      "Episode 1300 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 4.283e-06\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 3.015e-07\n",
      "Learner:4\tReward total:0\tRunning mean: 1.676e-07\n",
      "Learner:5\tReward total:0\tRunning mean: 1.021e-06\n",
      "Learner:6\tReward total:0\tRunning mean: 3.302e-07\n",
      "Learner:7\tReward total:0\tRunning mean: 5.384e-05\n",
      "Max Norms =  ['0.02', '0.01', '0.00', '0.01', '0.01', '0.00', '0.00', '0.02']\n",
      "....................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.568e-06\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 1.104e-07\n",
      "Learner:4\tReward total:0\tRunning mean: 6.135e-08\n",
      "Learner:5\tReward total:0\tRunning mean: 3.736e-07\n",
      "Learner:6\tReward total:0\tRunning mean: 1.209e-07\n",
      "Learner:7\tReward total:0\tRunning mean: 1.971e-05\n",
      "Max Norms =  ['0.01', '0.01', '0.00', '0.01', '0.00', '0.00', '0.00', '0.03']\n",
      "....................................................................................................\n",
      "Episode 1500 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 5.738e-07\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 4.039e-08\n",
      "Learner:4\tReward total:0\tRunning mean: 2.246e-08\n",
      "Learner:5\tReward total:0\tRunning mean: 1.368e-07\n",
      "Learner:6\tReward total:0\tRunning mean: 4.424e-08\n",
      "Learner:7\tReward total:0\tRunning mean: 7.213e-06\n",
      "Max Norms =  ['0.01', '0.01', '0.00', '0.01', '0.02', '0.00', '0.00', '2.49']\n",
      "....................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 2.1e-07\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 1.479e-08\n",
      "Learner:4\tReward total:0\tRunning mean: 8.22e-09\n",
      "Learner:5\tReward total:0\tRunning mean: 5.006e-08\n",
      "Learner:6\tReward total:0\tRunning mean: 1.619e-08\n",
      "Learner:7\tReward total:0\tRunning mean: 2.64e-06\n",
      "Max Norms =  ['11.41', '0.00', '0.00', '0.00', '0.03', '0.00', '0.00', '0.01']\n",
      "....................................................................................................\n",
      "Episode 1700 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 7.688e-08\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 5.412e-09\n",
      "Learner:4\tReward total:0\tRunning mean: 3.009e-09\n",
      "Learner:5\tReward total:0\tRunning mean: 1.832e-08\n",
      "Learner:6\tReward total:0\tRunning mean: 5.927e-09\n",
      "Learner:7\tReward total:0\tRunning mean: 9.664e-07\n",
      "Max Norms =  ['0.01', '0.00', '0.00', '0.00', '0.02', '0.00', '0.00', '0.00']\n",
      "....................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 2.814e-08\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 1.981e-09\n",
      "Learner:4\tReward total:0\tRunning mean: 1.101e-09\n",
      "Learner:5\tReward total:0\tRunning mean: 6.707e-09\n",
      "Learner:6\tReward total:0\tRunning mean: 2.169e-09\n",
      "Learner:7\tReward total:0\tRunning mean: 3.537e-07\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.01', '0.00', '0.00', '0.00']\n",
      "....................................................................................................\n",
      "Episode 1900 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.03e-08\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 7.251e-10\n",
      "Learner:4\tReward total:0\tRunning mean: 4.031e-10\n",
      "Learner:5\tReward total:0\tRunning mean: 2.455e-09\n",
      "Learner:6\tReward total:0\tRunning mean: 7.941e-10\n",
      "Learner:7\tReward total:0\tRunning mean: 1.295e-07\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.01', '0.00', '0.00', '0.00']\n",
      "....................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 3.77e-09\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 2.654e-10\n",
      "Learner:4\tReward total:0\tRunning mean: 1.476e-10\n",
      "Learner:5\tReward total:0\tRunning mean: 8.986e-10\n",
      "Learner:6\tReward total:0\tRunning mean: 2.907e-10\n",
      "Learner:7\tReward total:0\tRunning mean: 4.739e-08\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '0.00']\n",
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Load random agent 8\n",
      "....................................................................................................\n",
      "Episode 100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.006173\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 0.003772\n",
      "Learner:5\tReward total:0\tRunning mean: 0.04722\n",
      "Learner:6\tReward total:0\tRunning mean: 0.06727\n",
      "Learner:7\tReward total:0\tRunning mean: 0.1932\n",
      "Max Norms =  ['322.50', '365.88', '64.22', '356.15', '277.73', '0.00', '317.29', '205.81']\n",
      "....................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.002259\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 0.001381\n",
      "Learner:5\tReward total:0\tRunning mean: 0.01728\n",
      "Learner:6\tReward total:0\tRunning mean: 0.02462\n",
      "Learner:7\tReward total:0\tRunning mean: 0.0707\n",
      "Max Norms =  ['44.67', '46.74', '69.72', '296.85', '30.25', '35.60', '163.52', '88.02']\n",
      "....................................................................................................\n",
      "Episode 300 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.000827\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 0.0005054\n",
      "Learner:5\tReward total:0\tRunning mean: 0.006326\n",
      "Learner:6\tReward total:0\tRunning mean: 0.009012\n",
      "Learner:7\tReward total:0\tRunning mean: 0.02588\n",
      "Max Norms =  ['61.83', '34.08', '25.57', '75.11', '16.63', '15.00', '57.39', '63.18']\n",
      "....................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0003027\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 0.000185\n",
      "Learner:5\tReward total:0\tRunning mean: 0.002316\n",
      "Learner:6\tReward total:0\tRunning mean: 0.003299\n",
      "Learner:7\tReward total:0\tRunning mean: 0.009472\n",
      "Max Norms =  ['11.40', '13.87', '37.78', '49.15', '56.87', '27.60', '6.22', '18.03']\n",
      "....................................................................................................\n",
      "Episode 500 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0001108\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 6.772e-05\n",
      "Learner:5\tReward total:0\tRunning mean: 0.0008476\n",
      "Learner:6\tReward total:0\tRunning mean: 0.001207\n",
      "Learner:7\tReward total:0\tRunning mean: 0.003467\n",
      "Max Norms =  ['17.02', '35.52', '1.31', '49.98', '5.30', '13.08', '0.15', '6.04']\n",
      "....................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 4.056e-05\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 2.479e-05\n",
      "Learner:5\tReward total:0\tRunning mean: 0.0003102\n",
      "Learner:6\tReward total:0\tRunning mean: 0.000442\n",
      "Learner:7\tReward total:0\tRunning mean: 0.001269\n",
      "Max Norms =  ['0.07', '0.05', '10.06', '15.55', '12.61', '10.26', '0.11', '14.72']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "....................................................................................................\n",
      "Episode 700 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.485e-05\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 9.073e-06\n",
      "Learner:5\tReward total:0\tRunning mean: 0.0001136\n",
      "Learner:6\tReward total:0\tRunning mean: 0.0001618\n",
      "Learner:7\tReward total:0\tRunning mean: 0.0004645\n",
      "Max Norms =  ['0.04', '0.23', '5.51', '32.72', '10.49', '1.19', '5.46', '0.04']\n",
      "....................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 5.434e-06\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 3.321e-06\n",
      "Learner:5\tReward total:0\tRunning mean: 4.157e-05\n",
      "Learner:6\tReward total:0\tRunning mean: 5.921e-05\n",
      "Learner:7\tReward total:0\tRunning mean: 0.00017\n",
      "Max Norms =  ['0.03', '40.07', '0.01', '39.29', '0.07', '37.05', '0.04', '0.02']\n",
      "....................................................................................................\n",
      "Episode 900 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.989e-06\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 1.216e-06\n",
      "Learner:5\tReward total:0\tRunning mean: 1.521e-05\n",
      "Learner:6\tReward total:0\tRunning mean: 2.167e-05\n",
      "Learner:7\tReward total:0\tRunning mean: 6.224e-05\n",
      "Max Norms =  ['0.02', '9.17', '0.01', '7.46', '0.04', '182.01', '0.03', '0.01']\n",
      "....................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 7.281e-07\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 4.449e-07\n",
      "Learner:5\tReward total:0\tRunning mean: 5.569e-06\n",
      "Learner:6\tReward total:0\tRunning mean: 7.934e-06\n",
      "Learner:7\tReward total:0\tRunning mean: 2.278e-05\n",
      "Max Norms =  ['0.02', '0.01', '0.01', '16.29', '0.03', '0.02', '0.02', '0.03']\n",
      "....................................................................................................\n",
      "Episode 1100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 2.665e-07\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 1.629e-07\n",
      "Learner:5\tReward total:0\tRunning mean: 2.038e-06\n",
      "Learner:6\tReward total:0\tRunning mean: 2.904e-06\n",
      "Learner:7\tReward total:0\tRunning mean: 8.339e-06\n",
      "Max Norms =  ['135.01', '0.01', '0.00', '2.11', '0.02', '0.02', '26.56', '0.03']\n",
      "....................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 9.754e-08\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 5.961e-08\n",
      "Learner:5\tReward total:0\tRunning mean: 7.461e-07\n",
      "Learner:6\tReward total:0\tRunning mean: 1.063e-06\n",
      "Learner:7\tReward total:0\tRunning mean: 3.052e-06\n",
      "Max Norms =  ['0.01', '0.00', '0.00', '0.04', '0.01', '12.64', '0.01', '0.03']\n",
      "....................................................................................................\n",
      "Episode 1300 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 3.57e-08\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 2.182e-08\n",
      "Learner:5\tReward total:0\tRunning mean: 2.731e-07\n",
      "Learner:6\tReward total:0\tRunning mean: 3.891e-07\n",
      "Learner:7\tReward total:0\tRunning mean: 1.117e-06\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '13.30', '0.01', '0.02', '0.01', '2.76']\n",
      "....................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.307e-08\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 7.987e-09\n",
      "Learner:5\tReward total:0\tRunning mean: 9.997e-08\n",
      "Learner:6\tReward total:0\tRunning mean: 1.424e-07\n",
      "Learner:7\tReward total:0\tRunning mean: 4.089e-07\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '9.85', '0.01', '0.01', '0.01', '0.01']\n",
      "....................................................................................................\n",
      "Episode 1500 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 4.784e-09\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 2.923e-09\n",
      "Learner:5\tReward total:0\tRunning mean: 3.659e-08\n",
      "Learner:6\tReward total:0\tRunning mean: 5.213e-08\n",
      "Learner:7\tReward total:0\tRunning mean: 1.497e-07\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '2.81', '0.01', '0.02', '0.00', '0.01']\n",
      "....................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.751e-09\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 1.07e-09\n",
      "Learner:5\tReward total:0\tRunning mean: 1.339e-08\n",
      "Learner:6\tReward total:0\tRunning mean: 1.908e-08\n",
      "Learner:7\tReward total:0\tRunning mean: 5.479e-08\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.03', '0.00', '0.01', '0.00', '0.01']\n",
      "....................................................................................................\n",
      "Episode 1700 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 6.409e-10\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 3.917e-10\n",
      "Learner:5\tReward total:0\tRunning mean: 4.902e-09\n",
      "Learner:6\tReward total:0\tRunning mean: 6.984e-09\n",
      "Learner:7\tReward total:0\tRunning mean: 2.005e-08\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '14.52', '0.00', '0.01', '0.00', '0.01']\n",
      "....................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 2.346e-10\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 1.434e-10\n",
      "Learner:5\tReward total:0\tRunning mean: 1.794e-09\n",
      "Learner:6\tReward total:0\tRunning mean: 2.556e-09\n",
      "Learner:7\tReward total:0\tRunning mean: 7.341e-09\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.02', '0.00', '0.00', '0.00', '0.00']\n",
      "....................................................................................................\n",
      "Episode 1900 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 8.587e-11\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 5.248e-11\n",
      "Learner:5\tReward total:0\tRunning mean: 6.568e-10\n",
      "Learner:6\tReward total:0\tRunning mean: 9.357e-10\n",
      "Learner:7\tReward total:0\tRunning mean: 2.687e-09\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '11.26', '0.00', '0.00', '0.00', '0.00']\n",
      "....................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 3.143e-11\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 1.921e-11\n",
      "Learner:5\tReward total:0\tRunning mean: 2.404e-10\n",
      "Learner:6\tReward total:0\tRunning mean: 3.425e-10\n",
      "Learner:7\tReward total:0\tRunning mean: 9.835e-10\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '72.82', '0.00', '0.00', '0.00', '0.00']\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import sys\n",
    "import pickle\n",
    "from collections import deque\n",
    "\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import numpy as np\n",
    "\n",
    "# Initialize agents parameters\n",
    "#   9 agents - 9 learning agents, 0 trained agent, 0 random agent\n",
    "num_learners = 9\n",
    "num_trained = 0\n",
    "num_rdn = 0\n",
    "num_statics = num_trained + num_rdn\n",
    "num_agents = num_learners + num_statics  \n",
    "\n",
    "# Initialize environment\n",
    "game = \"Gather\"\n",
    "num_actions = 8                       # 8 actions in Gathering\n",
    "\n",
    "# Initialize training parameters\n",
    "warm_start = False\n",
    "num_frames = 4      # environ observation consists of a list of 4 stacked frames per agent\n",
    "max_episodes = 5000\n",
    "max_frames = 1000\n",
    "max_frames_ep = 0   # track highest number of frames an episode can last\n",
    "\n",
    "# These trainer parameters works for Atari Breakout\n",
    "gamma = 0.99  \n",
    "lr = 1e-3\n",
    "temp_start = 1.8  # Temperature for explore/exploit\n",
    "temp_end = 1.0\n",
    "log_interval = 200\n",
    "save_interval = 500\n",
    "\n",
    "\n",
    "def unpack_env_obs(env_obs):\n",
    "    \"\"\"\n",
    "    Gathering is a partially-observable Markov Game. env_obs returned by GatheringEnv is a numpy \n",
    "    array of dimension (num_agent, 800), which represents the agents' observations of the game.\n",
    "\n",
    "    The 800 elements (view_box) encodes 4 layers of 10x20 pixels frames in the format:\n",
    "    (viewbox_width, viewbox_depth, 4).\n",
    "    \n",
    "    This code reshapes the above into stacked frames that can be accepted by the Policy class:\n",
    "    (batch_idx, in_channel, width, height)\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    num_agents = len(env_obs)  # environ observations is a list of agents' observations\n",
    "    \n",
    "    obs = []\n",
    "    for i in range(num_agents):\n",
    "        x = env_obs[i]   # take the indexed agent's observation\n",
    "        x = torch.Tensor(x)   # Convert to tensor\n",
    "        \n",
    "        # Policy is a 3-layer CNN\n",
    "        x = x.view(1, 10, 20, -1)  # reshape into environment defined stacked frames\n",
    "        x = x.permute(0, 3, 1, 2)  # permute to Policy accepted stacked frames\n",
    "        obs.append(x)\n",
    "        \n",
    "    return obs  # return a list of Tensors\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "For now, we do not implement LSTM            \n",
    "# LSTM Change: Need to cycle hx and cx thru function\n",
    "def select_action(model, state, lstm_hc, cuda):\n",
    "    hx , cx = lstm_hc \n",
    "    num_frames, height, width = state.shape\n",
    "    state = torch.FloatTensor(state.reshape(-1, num_frames, height, width))\n",
    "\n",
    "    if cuda:\n",
    "        state = state.cuda()\n",
    "\n",
    "    probs, value, (hx, cx) = model((Variable(state), (hx, cx)))\n",
    "\n",
    "    m = torch.distributions.Categorical(probs)\n",
    "    action = m.sample()\n",
    "    log_prob = m.log_prob(action)\n",
    "    # LSTM Change: Need to cycle hx and cx thru function\n",
    "    return action.data[0], log_prob, value, (hx, cx)\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "def select_learner_action(model, obs, cuda):\n",
    "    \"\"\"\n",
    "    This code expects obs to be an array of stacked frames of the following dim:\n",
    "    (batch_idx, in_channel, width, height)\n",
    "    \n",
    "    This is inputted into model - the agent's Policy, which outputs a probability \n",
    "    distribution over available actions.\n",
    "    \n",
    "    Policy gradient is implemented using torch.distributions.Categorical. \n",
    "    \"\"\"\n",
    "    \n",
    "    # Policy is a 3-layer CNN\n",
    "    # _, num_frames, width, height = obs.shape\n",
    "    # obs = torch.FloatTensor(obs.reshape(-1, num_frames, width, height))\n",
    "    \n",
    "    # Policy is a 2-layer NN for now\n",
    "    # obs = obs.view(1, -1)\n",
    "   \n",
    "    if cuda:\n",
    "        obs = obs.cuda()\n",
    "      \n",
    "    probs = model(obs)\n",
    "    m = torch.distributions.Categorical(probs)\n",
    "    action = m.sample()\n",
    "    log_prob = m.log_prob(action)\n",
    "\n",
    "    return action.item(), log_prob \n",
    "\n",
    "\n",
    "def load_info(agents, narrate=False):\n",
    "    for i in range(num_agents):    \n",
    "        agents[i].load_info(info[i])\n",
    "        if narrate:\n",
    "            if agents[i].tagged:\n",
    "                print('frame {}, agent{} is tagged'.format(frame,i))\n",
    "            if agents[i].laser_fired:\n",
    "                print('frame {}, agent{} fires its laser'.format(frame,i))\n",
    "                print('and hit {} US and {} THEM'.format(agents[i].US_hit, agents[i].THEM_hit))\n",
    "    return\n",
    "\n",
    "\n",
    "# The main code starts here!!!\n",
    "\n",
    "cultures =[{'name':'warlike', 'penalty':-1.0, 'reward':0.001},\n",
    "           {'name':'warlike', 'penalty':-1.0, 'reward':0.005}\n",
    "          ]\n",
    "\n",
    "\"\"\"\n",
    "cultures =[\n",
    "           {'name':'warlike', 'penalty':-1.0, 'reward':0.001},\n",
    "           {'name':'warlike', 'penalty':-1.0, 'reward':0.005},\n",
    "           {'name':'warlike', 'penalty':-1.0, 'reward':0.01},\n",
    "           {'name':'warlike', 'penalty':-1.0, 'reward':0.05},\n",
    "           {'name':'warlike', 'penalty':-1.0, 'reward':0.1},\n",
    "           {'name':'warlike', 'penalty':-1.0, 'reward':0.5},\n",
    "           {'name':'warlike', 'penalty':-1.0, 'reward':1.0}\n",
    "          ]\n",
    "\"\"\"\n",
    "\n",
    "# Cultural parameter search\n",
    "for culture in cultures:   # Go down the list of cultures\n",
    "\n",
    "    # Data structure for agents\n",
    "    agents = []\n",
    "    actions = []\n",
    "    log_probs = []\n",
    "    tags = []\n",
    "    rewards = []\n",
    "    optimizers = []\n",
    "\n",
    "    # Cold start\n",
    "    if warm_start is False:\n",
    "   \n",
    "        # Initialize learner agents, then load static agents (trained followed by random)\n",
    "        for i in range(num_learners):\n",
    "            print(\"Learner agent {}\".format(i))\n",
    "            agents.append(Policy(num_frames, num_actions, i)) # No weights loaded for learning agent\n",
    "            optimizers.append(optim.Adam(agents[i].parameters(), lr=lr))\n",
    "        \n",
    "            # set up optimizer - this works for Atari Breakout\n",
    "            # optimizers.append(optim.RMSprop(agents[i].parameters(), lr=lr, weight_decay=0.1)) \n",
    "        \n",
    "        for i in range(num_learners, num_learners+num_trained):\n",
    "            print (\"No trained agent exist yet!\")\n",
    "            raise\n",
    "            \"\"\"\n",
    "            Disable for now! No trained model exist!!!\n",
    "            agents.append(Policy(num_frames, num_actions, i))\n",
    "            agents[i].load_weights()         # load weight for static agent        \n",
    "            \"\"\"\n",
    "        for i in range(num_learners+num_trained, num_agents):\n",
    "            print(\"Load random agent {}\".format(i))\n",
    "            agents.append(Rdn_Policy())\n",
    "\n",
    "    \n",
    "        # Initialize all agent data\n",
    "        actions = [0 for i in range(num_agents)]\n",
    "        log_probs = [0 for i in range(num_agents)]\n",
    "        tags = [0 for i in range(num_agents)]\n",
    "        rewards = [0 for i in range(num_agents)]\n",
    "\n",
    "        # Keep track of rewards learned by learners\n",
    "        episode_reward = [0 for i in range(num_learners)]   # reward for an episode\n",
    "        running_reward = [None for i in range(num_learners)]   # running average\n",
    "        running_rewards = [[] for i in range(num_learners)]   # history of running averages\n",
    "        best_reward = [0 for i in range(num_learners)]    # best running average (for storing best_model)\n",
    "\n",
    "        # This is to support warm start for training\n",
    "        prior_eps = 0\n",
    "\n",
    "    # Warm start\n",
    "    if warm_start:\n",
    "        print (\"Cannot warm start\")\n",
    "        raise\n",
    "    \n",
    "        \"\"\"\n",
    "        # Disable for now!  Need to ensure model can support training on GPU and game playing\n",
    "        # on both CPU and GPU.\n",
    "    \n",
    "        data_file = 'results/{}.p'.format(game)\n",
    "\n",
    "        try:\n",
    "            with open(data_file, 'rb') as f:\n",
    "                running_rewards = pickle.load(f)\n",
    "                running_reward = running_rewards[-1]\n",
    "\n",
    "            prior_eps = len(running_rewards)\n",
    "\n",
    "            model_file = 'saved_models/actor_critic_{}_ep_{}.p'.format(game, prior_eps)\n",
    "            with open(model_file, 'rb') as f:\n",
    "                # Model Save and Load Update: Include both model and optim parameters\n",
    "                saved_model = pickle.load(f)\n",
    "                model, optimizer = saved_model\n",
    "\n",
    "        except OSError:\n",
    "            print('Saved file not found. Creating new cold start model.')\n",
    "            model = Policy(input_channels=num_frames, num_actions=num_actions)\n",
    "            optimizer = optim.RMSprop(model.parameters(), lr=lr,\n",
    "                                      weight_decay=0.1)\n",
    "            running_rewards = []\n",
    "            prior_eps = 0\n",
    "        \"\"\"\n",
    "\n",
    "    # Establish tribal association\n",
    "\n",
    "    tribes = []\n",
    "    tribes.append(Tribe(name='Vikings',color='blue', culture=culture, \\\n",
    "                    agents=[agents[0], agents[1], agents[2]]))\n",
    "    tribes.append(Tribe(name='Saxons', color='red', culture=culture, \\\n",
    "                    agents=[agents[3], agents[4], agents[5]]))\n",
    "    tribes.append(Tribe(name='Franks', color='purple', culture=culture, \\\n",
    "                    agents=[agents[6], agents[7], agents[8]]))\n",
    "    # tribes.append(Tribe(name='Crazies', color='yellow', agents=[agents[9]]))   # random agents are crazy!!!\n",
    "\n",
    "    # 9 agents in 4 tribes, used map defined in default.txt\n",
    "    agent_colors = [agent.color for agent in agents]\n",
    "    agent_tribes = [agent.tribe for agent in agents]\n",
    "    \n",
    "    env = GatheringEnv(n_agents=num_agents,agent_colors=agent_colors, agent_tribes=agent_tribes, \\\n",
    "                       map_name='default')    \n",
    "    \n",
    "    cuda = torch.cuda.is_available()\n",
    "\n",
    "    if cuda:\n",
    "        for i in range(num_learners):    # Learning agents need to utilize GPU\n",
    "            agents[i].cuda()\n",
    "\n",
    "        \n",
    "    for ep in range(max_episodes):\n",
    "    \n",
    "        print('.', end='')  # To show progress\n",
    "    \n",
    "        # Anneal temperature from temp_start to temp_end\n",
    "        for i in range(num_learners):    # For learning agents\n",
    "            agents[i].temperature = max(temp_end, temp_start - (temp_start - temp_end) * (ep / max_episodes))\n",
    "\n",
    "        env_obs = env.reset()  # Env return observations\n",
    "\n",
    "        # For Debug only\n",
    "        # print (len(env_obs))\n",
    "        # print (env_obs[0].shape)\n",
    "    \n",
    "        # Unpack observations into data structure compatible with agent Policy\n",
    "        agents_obs = unpack_env_obs(env_obs)\n",
    "\n",
    "        for i in range(num_learners):    # Reset agent info - laser tag statistics\n",
    "            agents[i].reset_info()   \n",
    "\n",
    "        # For Debug only\n",
    "        # print (len(agents_obs))\n",
    "        # print (agents_obs[0].shape)\n",
    "    \n",
    "        \"\"\"\n",
    "        For now, we do not stack observations, and we do not implement LSTM\n",
    "    \n",
    "        state = np.stack([state]*num_frames)\n",
    "\n",
    "        # LSTM change - reset LSTM hidden units when episode begins\n",
    "        cx = Variable(torch.zeros(1, 256))\n",
    "        hx = Variable(torch.zeros(1, 256))\n",
    "        if cuda:\n",
    "            cx = cx.cuda()\n",
    "            hx = hx.cuda()\n",
    "        \"\"\"\n",
    "\n",
    "        episode_reward = [0 for i in range(num_learners)]   # reward for an episode\n",
    "    \n",
    "        for frame in range(max_frames):\n",
    "\n",
    "            \"\"\"\n",
    "            For now, we do not implement LSTM\n",
    "            # Select action\n",
    "            # LSTM Change: Need to cycle hx and cx thru select_action\n",
    "            action, log_prob, value, (hx,cx)  = select_action(model, state, (hx,cx), cuda)        \n",
    "            \"\"\"\n",
    "\n",
    "            for i in range(num_learners):    # For learning agents\n",
    "                actions[i], log_probs[i] = select_learner_action(agents[i], agents_obs[i], cuda)\n",
    "                if actions[i] is 6:\n",
    "                    tags[i] += 1   # record a tag for accessing aggressiveness\n",
    "                agents[i].saved_actions.append((log_probs[i]))\n",
    "            \n",
    "                # Do not implement LSTM for now\n",
    "                # actions[i].saved_actions.append((log_prob, value))\n",
    "            \n",
    "            for i in range(num_learners, num_learners+num_trained):\n",
    "                print (\"No trained agent exist yet!\")\n",
    "                raise\n",
    "            for i in range(num_learners+num_trained, num_agents):   # For random agents\n",
    "                actions[i] = agents[i].select_action(agents_obs[i])\n",
    "                if actions[i] is 6:\n",
    "                    tags[i] += 1   # record a tag for accessing aggressiveness\n",
    "\n",
    "            # For Debug only\n",
    "            # if frame % 20 == 0:\n",
    "            #    print (actions) \n",
    "            #    print (log_probs)\n",
    "            \n",
    "            # Perform step        \n",
    "            env_obs, reward, done, info = env.step(actions)\n",
    "        \n",
    "            \"\"\"\n",
    "            For Debug only\n",
    "            print (env_obs)\n",
    "            print (reward)\n",
    "            print (done) \n",
    "            \"\"\"\n",
    "       \n",
    "            # Unpack observations into data structure compatible with agent Policy\n",
    "            agents_obs = unpack_env_obs(env_obs)\n",
    "            load_info(agents, narrate=False)   # Load agent info for AI agents\n",
    "\n",
    "            # For learner agents only, generate reward statistics and reward stack for policy gradient\n",
    "            for i in range(num_learners):\n",
    "                agents[i].rewards.append(reward[i])  # Stack rewards (for policy gradient)\n",
    "                episode_reward[i] += reward[i]   # accumulate episode reward \n",
    "            \n",
    "            \"\"\"\n",
    "            For now, we do not stack observation, may come in handy later on\n",
    "        \n",
    "            # Evict oldest diff add new diff to state\n",
    "            next_state = np.stack([next_state]*num_frames)\n",
    "            next_state[1:, :, :] = state[:-1, :, :]\n",
    "            state = next_state\n",
    "            \"\"\"\n",
    "\n",
    "            if any(done):\n",
    "                print(\"Done after {} frames\".format(frame))\n",
    "                break\n",
    "            \n",
    "        if frame > max_frames_ep:\n",
    "            max_frames_ep = frame    # Keep track of highest frames/episode\n",
    "\n",
    "        # Update reward statistics for learners\n",
    "        for i in range(num_learners):\n",
    "            if running_reward[i] is None:\n",
    "                running_reward[i] = episode_reward[i]\n",
    "            running_reward[i] = running_reward[i] * 0.99 + episode_reward[i] * 0.01\n",
    "            running_rewards[i].append(running_reward[i])\n",
    "\n",
    "        # Track Episode #, temp and highest frames/episode\n",
    "        if (ep+prior_eps+1) % log_interval == 0: \n",
    "            verbose_str = '\\nEpisode {} complete'.format(ep+prior_eps+1)\n",
    "            # verbose_str += '\\tTemp = {:.4}'.format(model.temperature)\n",
    "            # verbose_str += '\\tMax frames = {}'.format(max_frames_ep+1)\n",
    "            print(verbose_str)\n",
    "    \n",
    "            # Display rewards and running rewards for learning agents\n",
    "            for i in range(num_learners):\n",
    "                verbose_str = 'Learner:{}'.format(i)\n",
    "                verbose_str += '\\tReward total:{}'.format(episode_reward[i])\n",
    "                verbose_str += '\\tRunning mean: {:.4}'.format(running_reward[i])\n",
    "                print(verbose_str)\n",
    "    \n",
    "        # Update model\n",
    "        total_norms = finish_episode(agents[0:num_learners], optimizers[0:num_learners], gamma, cuda)\n",
    "\n",
    "        if (ep+prior_eps+1) % log_interval == 0:\n",
    "            print('Max Norms = ',[\"%0.2f\" % i for i in total_norms])\n",
    "        \n",
    "        if (ep+prior_eps+1) % save_interval == 0: \n",
    "            for i in range(num_learners):\n",
    "                model_dir = 'MA_models/3T-9L/'\n",
    "                results_dir = 'results/3T-9L/'\n",
    "\n",
    "                model_file = model_dir+'{}/p{}_r{}/MA{}_{}_ep{}.p'.format(culture['name'], culture['penalty'],\\\n",
    "                                 culture['reward'], i, game, ep+prior_eps+1)\n",
    "                data_file = results_dir+'{}/p{}_r{}/MA{}_{}.p'.format(culture['name'], culture['penalty'],\\\n",
    "                                                      culture['reward'], i, game)\n",
    "                \n",
    "                os.makedirs(os.path.dirname(model_file), exist_ok=True)\n",
    "                os.makedirs(os.path.dirname(data_file), exist_ok=True)\n",
    "                \n",
    "                with open(model_file, 'wb') as f:\n",
    "                    # Model Save and Load Update: Include both model and optim parameters \n",
    "                    pickle.dump((agents[i].cpu(), optimizers[i]), f)\n",
    "\n",
    "                if cuda:\n",
    "                    agents[i] = agents[i].cuda()\n",
    "\n",
    "                with open(data_file, 'wb') as f:\n",
    "                    pickle.dump(running_rewards[i], f)    \n",
    "            \n",
    "    env.close()  # Close the environment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Culture = Cooperative\n",
    "\n",
    "Run finish_episode() specific to this culture."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def finish_episode(learners, optimizers, gamma, cuda):\n",
    "    \"\"\" \n",
    "    Note that in RL, policy gradient is calculated at the end of an episode and only then used to \n",
    "    update the weights of an agent's policy. This is very different compared to image recog.\n",
    "    \n",
    "    The code will perform policy update on each learning agent independently. Reward for each time \n",
    "    step is stored in the list policy.rewards[] --> r(t)\n",
    "    \"\"\"  \n",
    "    \n",
    "    num_learners = len(learners)\n",
    "    total_norms = [0 for i in range(num_learners)]\n",
    "    policy_losses = [[] for i in range(num_learners)]\n",
    "    losses = [[] for i in range(num_learners)]\n",
    "    T_reward = []\n",
    "\n",
    "   \n",
    "    for i in range(num_learners):\n",
    "\n",
    "        R = 0\n",
    "        saved_actions = learners[i].saved_actions\n",
    "        \n",
    "        for t in tribes:\n",
    "            if t.name is learners[i].tribe:\n",
    "                T_reward = t.tribal_awards()\n",
    "                \n",
    "                # For debug only\n",
    "                # print('Agent{} receives tribal award from Tribe{}'.format(i,t.name))\n",
    "                # print (T_reward)\n",
    "                # print (learners[i].rewards)\n",
    "        \n",
    "        # Do not implement actor-critic for now\n",
    "        # value_losses = []\n",
    "        \n",
    "        rewards = deque()\n",
    "\n",
    "        for r,T in zip(learners[i].rewards[::-1],T_reward[::-1]):\n",
    "            # The agent is incentivized to cooperate by an award of 30% of what the tribe takes\n",
    "            # in by all its members\n",
    "            R = r + T + gamma * R\n",
    "            rewards.appendleft(R)\n",
    "            \n",
    "        rewards = list(rewards)\n",
    "        rewards = torch.Tensor(rewards)\n",
    "        if cuda:\n",
    "            rewards = rewards.cuda()\n",
    "\n",
    "        # z-score rewards\n",
    "        rewards = (rewards - rewards.mean()) / (1.1e-7+rewards.std())\n",
    "        \n",
    "        #Debug     \n",
    "        #print (rewards)       \n",
    "        \n",
    "        \"\"\"\n",
    "        Do not implement actor-critic for now!!!\n",
    "        for (log_prob, state_value), r in zip(saved_actions, rewards):\n",
    "            reward = r - state_value.data[0]\n",
    "            policy_losses.append(-log_prob * Variable(reward))\n",
    "            r = torch.Tensor([r])\n",
    "            if cuda:\n",
    "                r = r.cuda()\n",
    "            value_losses.append(torch.nn.functional.smooth_l1_loss(state_value,\n",
    "                                                               Variable(r)))\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss = torch.stack(policy_losses).sum() + torch.stack(value_losses).sum()\n",
    "        loss.backward()        \n",
    "        \n",
    "        \n",
    "        \"\"\"\n",
    "        for log_prob, r in zip(saved_actions, rewards):\n",
    "            r = torch.Tensor([r])\n",
    "            if cuda:\n",
    "                r = r.cuda()\n",
    "            policy_losses[i].append(-log_prob * Variable(r))\n",
    "\n",
    "        optimizers[i].zero_grad()\n",
    "        losses[i] = torch.stack(policy_losses[i]).sum()\n",
    "        losses[i].backward()\n",
    "        \n",
    "        # Gradient Clipping Update: prevent exploding gradient\n",
    "        total_norms[i] = torch.nn.utils.clip_grad_norm_(learners[i].parameters(), 8000)\n",
    "        \n",
    "        optimizers[i].step()\n",
    "        \n",
    "    # clear all agent's history only at the end of episode; they are needed for tribal reward calc\n",
    "    for i in range(num_learners):\n",
    "        learners[i].clear_history()\n",
    "\n",
    "\n",
    "    return total_norms\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Learner agent 8\n",
      "........................................................................................................................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0596\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.1163\n",
      "Learner:4\tReward total:0\tRunning mean: 0.03945\n",
      "Learner:5\tReward total:0\tRunning mean: 0.02259\n",
      "Learner:6\tReward total:0\tRunning mean: 0.4338\n",
      "Learner:7\tReward total:0\tRunning mean: 0.261\n",
      "Learner:8\tReward total:0\tRunning mean: 2.055\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '0.00']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:0\tRunning mean: 0.007985\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.01892\n",
      "Learner:4\tReward total:0\tRunning mean: 0.01695\n",
      "Learner:5\tReward total:0\tRunning mean: 0.01732\n",
      "Learner:6\tReward total:0\tRunning mean: 0.08251\n",
      "Learner:7\tReward total:0\tRunning mean: 0.9305\n",
      "Learner:8\tReward total:37\tRunning mean: 19.91\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '128.18', '93.92', '129.88']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.1832\n",
      "Learner:1\tReward total:0\tRunning mean: 0.1486\n",
      "Learner:2\tReward total:10\tRunning mean: 1.818\n",
      "Learner:3\tReward total:0\tRunning mean: 0.002535\n",
      "Learner:4\tReward total:3\tRunning mean: 0.513\n",
      "Learner:5\tReward total:0\tRunning mean: 0.1991\n",
      "Learner:6\tReward total:0\tRunning mean: 0.1147\n",
      "Learner:7\tReward total:0\tRunning mean: 0.6423\n",
      "Learner:8\tReward total:53\tRunning mean: 43.79\n",
      "Max Norms =  ['44.30', '85.17', '76.69', '68.29', '101.45', '65.32', '64.56', '72.19', '110.46']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.5641\n",
      "Learner:1\tReward total:2\tRunning mean: 0.8225\n",
      "Learner:2\tReward total:7\tRunning mean: 7.429\n",
      "Learner:3\tReward total:0\tRunning mean: 0.02725\n",
      "Learner:4\tReward total:4\tRunning mean: 6.205\n",
      "Learner:5\tReward total:14\tRunning mean: 7.662\n",
      "Learner:6\tReward total:0\tRunning mean: 0.02517\n",
      "Learner:7\tReward total:2\tRunning mean: 0.5067\n",
      "Learner:8\tReward total:67\tRunning mean: 57.08\n",
      "Max Norms =  ['32.80', '62.20', '17.10', '51.30', '107.62', '81.67', '127.28', '65.29', '127.84']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.493\n",
      "Learner:1\tReward total:12\tRunning mean: 3.104\n",
      "Learner:2\tReward total:11\tRunning mean: 7.771\n",
      "Learner:3\tReward total:0\tRunning mean: 0.003651\n",
      "Learner:4\tReward total:2\tRunning mean: 2.554\n",
      "Learner:5\tReward total:11\tRunning mean: 21.98\n",
      "Learner:6\tReward total:0\tRunning mean: 0.03781\n",
      "Learner:7\tReward total:0\tRunning mean: 0.649\n",
      "Learner:8\tReward total:55\tRunning mean: 57.6\n",
      "Max Norms =  ['69.34', '85.83', '72.11', '46.54', '80.65', '103.97', '17.99', '58.95', '116.49']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.74\n",
      "Learner:1\tReward total:1\tRunning mean: 4.985\n",
      "Learner:2\tReward total:3\tRunning mean: 9.138\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0004891\n",
      "Learner:4\tReward total:10\tRunning mean: 3.072\n",
      "Learner:5\tReward total:40\tRunning mean: 32.1\n",
      "Learner:6\tReward total:0\tRunning mean: 0.02302\n",
      "Learner:7\tReward total:0\tRunning mean: 0.2931\n",
      "Learner:8\tReward total:65\tRunning mean: 56.88\n",
      "Max Norms =  ['23.34', '60.91', '36.98', '44.10', '60.86', '84.92', '86.27', '43.92', '163.06']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.73\n",
      "Learner:1\tReward total:1\tRunning mean: 4.592\n",
      "Learner:2\tReward total:7\tRunning mean: 9.433\n",
      "Learner:3\tReward total:0\tRunning mean: 6.553e-05\n",
      "Learner:4\tReward total:6\tRunning mean: 3.795\n",
      "Learner:5\tReward total:90\tRunning mean: 35.47\n",
      "Learner:6\tReward total:0\tRunning mean: 0.02154\n",
      "Learner:7\tReward total:0\tRunning mean: 0.04177\n",
      "Learner:8\tReward total:15\tRunning mean: 51.06\n",
      "Max Norms =  ['76.19', '59.42', '46.43', '58.17', '42.20', '129.29', '88.75', '33.21', '121.64']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:1\tRunning mean: 1.749\n",
      "Learner:1\tReward total:1\tRunning mean: 6.94\n",
      "Learner:2\tReward total:2\tRunning mean: 8.88\n",
      "Learner:3\tReward total:0\tRunning mean: 8.78e-06\n",
      "Learner:4\tReward total:5\tRunning mean: 4.452\n",
      "Learner:5\tReward total:39\tRunning mean: 40.54\n",
      "Learner:6\tReward total:0\tRunning mean: 0.2109\n",
      "Learner:7\tReward total:0\tRunning mean: 0.005596\n",
      "Learner:8\tReward total:53\tRunning mean: 42.67\n",
      "Max Norms =  ['13.40', '42.61', '69.32', '29.87', '75.17', '112.41', '74.36', '16.68', '73.10']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:2\tRunning mean: 1.299\n",
      "Learner:1\tReward total:0\tRunning mean: 9.682\n",
      "Learner:2\tReward total:5\tRunning mean: 7.084\n",
      "Learner:3\tReward total:0\tRunning mean: 1.176e-06\n",
      "Learner:4\tReward total:0\tRunning mean: 2.562\n",
      "Learner:5\tReward total:10\tRunning mean: 27.63\n",
      "Learner:6\tReward total:0\tRunning mean: 0.08961\n",
      "Learner:7\tReward total:0\tRunning mean: 0.0007497\n",
      "Learner:8\tReward total:110\tRunning mean: 61.11\n",
      "Max Norms =  ['14.24', '34.26', '24.63', '6.24', '25.83', '105.31', '33.21', '5.06', '165.46']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.258\n",
      "Learner:1\tReward total:0\tRunning mean: 4.488\n",
      "Learner:2\tReward total:0\tRunning mean: 4.554\n",
      "Learner:3\tReward total:0\tRunning mean: 1.576e-07\n",
      "Learner:4\tReward total:0\tRunning mean: 1.178\n",
      "Learner:5\tReward total:32\tRunning mean: 23.16\n",
      "Learner:6\tReward total:0\tRunning mean: 0.01986\n",
      "Learner:7\tReward total:0\tRunning mean: 0.0001005\n",
      "Learner:8\tReward total:102\tRunning mean: 80.82\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '44.77', '27.11', '70.02', '49.78', '32.70', '149.21']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.6563\n",
      "Learner:1\tReward total:0\tRunning mean: 2.482\n",
      "Learner:2\tReward total:0\tRunning mean: 3.9\n",
      "Learner:3\tReward total:0\tRunning mean: 2.112e-08\n",
      "Learner:4\tReward total:0\tRunning mean: 0.4297\n",
      "Learner:5\tReward total:0\tRunning mean: 11.64\n",
      "Learner:6\tReward total:0\tRunning mean: 0.002661\n",
      "Learner:7\tReward total:0\tRunning mean: 1.346e-05\n",
      "Learner:8\tReward total:115\tRunning mean: 96.53\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '1.77', '7.50', '210.18']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "........................................................................................................................................................................................................\n",
      "Episode 2400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.5089\n",
      "Learner:1\tReward total:0\tRunning mean: 1.73\n",
      "Learner:2\tReward total:0\tRunning mean: 1.244\n",
      "Learner:3\tReward total:0\tRunning mean: 2.829e-09\n",
      "Learner:4\tReward total:0\tRunning mean: 0.2031\n",
      "Learner:5\tReward total:0\tRunning mean: 6.277\n",
      "Learner:6\tReward total:0\tRunning mean: 0.0003565\n",
      "Learner:7\tReward total:0\tRunning mean: 0.01218\n",
      "Learner:8\tReward total:119\tRunning mean: 112.9\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '0.93', '8.76', '123.70']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.2654\n",
      "Learner:1\tReward total:0\tRunning mean: 1.724\n",
      "Learner:2\tReward total:0\tRunning mean: 1.402\n",
      "Learner:3\tReward total:0\tRunning mean: 3.79e-10\n",
      "Learner:4\tReward total:0\tRunning mean: 0.105\n",
      "Learner:5\tReward total:0\tRunning mean: 2.884\n",
      "Learner:6\tReward total:0\tRunning mean: 4.776e-05\n",
      "Learner:7\tReward total:0\tRunning mean: 0.1078\n",
      "Learner:8\tReward total:112\tRunning mean: 100.9\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '0.41', '66.46', '142.33']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.8962\n",
      "Learner:1\tReward total:0\tRunning mean: 1.194\n",
      "Learner:2\tReward total:0\tRunning mean: 1.538\n",
      "Learner:3\tReward total:0\tRunning mean: 5.078e-11\n",
      "Learner:4\tReward total:0\tRunning mean: 0.1278\n",
      "Learner:5\tReward total:0\tRunning mean: 3.11\n",
      "Learner:6\tReward total:0\tRunning mean: 6.399e-06\n",
      "Learner:7\tReward total:0\tRunning mean: 0.1212\n",
      "Learner:8\tReward total:131\tRunning mean: 108.9\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '0.43', '25.63', '156.03']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.4522\n",
      "Learner:1\tReward total:0\tRunning mean: 0.6173\n",
      "Learner:2\tReward total:0\tRunning mean: 0.8494\n",
      "Learner:3\tReward total:0\tRunning mean: 6.804e-12\n",
      "Learner:4\tReward total:0\tRunning mean: 0.2329\n",
      "Learner:5\tReward total:0\tRunning mean: 1.139\n",
      "Learner:6\tReward total:0\tRunning mean: 8.573e-07\n",
      "Learner:7\tReward total:0\tRunning mean: 0.01624\n",
      "Learner:8\tReward total:86\tRunning mean: 116.1\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '0.22', '30.17', '216.12']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 5.792\n",
      "Learner:1\tReward total:0\tRunning mean: 8.455\n",
      "Learner:2\tReward total:0\tRunning mean: 3.392\n",
      "Learner:3\tReward total:0\tRunning mean: 9.116e-13\n",
      "Learner:4\tReward total:0\tRunning mean: 0.1612\n",
      "Learner:5\tReward total:0\tRunning mean: 21.32\n",
      "Learner:6\tReward total:0\tRunning mean: 1.149e-07\n",
      "Learner:7\tReward total:0\tRunning mean: 0.002176\n",
      "Learner:8\tReward total:147\tRunning mean: 87.71\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '0.17', '16.98', '232.98']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 3.106\n",
      "Learner:1\tReward total:0\tRunning mean: 8.671\n",
      "Learner:2\tReward total:0\tRunning mean: 2.811\n",
      "Learner:3\tReward total:0\tRunning mean: 1.221e-13\n",
      "Learner:4\tReward total:0\tRunning mean: 0.6198\n",
      "Learner:5\tReward total:0\tRunning mean: 22.94\n",
      "Learner:6\tReward total:0\tRunning mean: 1.539e-08\n",
      "Learner:7\tReward total:0\tRunning mean: 0.0002915\n",
      "Learner:8\tReward total:150\tRunning mean: 93.06\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '0.29', '5.03', '148.85']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 2.556\n",
      "Learner:1\tReward total:0\tRunning mean: 2.324\n",
      "Learner:2\tReward total:0\tRunning mean: 1.306\n",
      "Learner:3\tReward total:0\tRunning mean: 1.636e-14\n",
      "Learner:4\tReward total:0\tRunning mean: 0.08305\n",
      "Learner:5\tReward total:0\tRunning mean: 4.549\n",
      "Learner:6\tReward total:0\tRunning mean: 2.062e-09\n",
      "Learner:7\tReward total:0\tRunning mean: 3.906e-05\n",
      "Learner:8\tReward total:156\tRunning mean: 126.8\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '0.22', '12.53', '106.48']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3800 complete\n",
      "Learner:0\tReward total:7\tRunning mean: 5.873\n",
      "Learner:1\tReward total:0\tRunning mean: 16.43\n",
      "Learner:2\tReward total:4\tRunning mean: 4.652\n",
      "Learner:3\tReward total:0\tRunning mean: 2.192e-15\n",
      "Learner:4\tReward total:0\tRunning mean: 0.1458\n",
      "Learner:5\tReward total:91\tRunning mean: 26.64\n",
      "Learner:6\tReward total:0\tRunning mean: 2.762e-10\n",
      "Learner:7\tReward total:0\tRunning mean: 5.233e-06\n",
      "Learner:8\tReward total:28\tRunning mean: 79.77\n",
      "Max Norms =  ['17.26', '12.63', '39.31', '0.49', '11.46', '122.82', '2.26', '3.92', '100.27']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 4000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.772\n",
      "Learner:1\tReward total:0\tRunning mean: 4.687\n",
      "Learner:2\tReward total:0\tRunning mean: 1.519\n",
      "Learner:3\tReward total:0\tRunning mean: 2.937e-16\n",
      "Learner:4\tReward total:0\tRunning mean: 0.07939\n",
      "Learner:5\tReward total:0\tRunning mean: 9.447\n",
      "Learner:6\tReward total:0\tRunning mean: 3.701e-11\n",
      "Learner:7\tReward total:0\tRunning mean: 7.011e-07\n",
      "Learner:8\tReward total:164\tRunning mean: 127.4\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '11.69', '0.00', '133.71']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 4200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.2474\n",
      "Learner:1\tReward total:0\tRunning mean: 0.6608\n",
      "Learner:2\tReward total:0\tRunning mean: 0.3523\n",
      "Learner:3\tReward total:0\tRunning mean: 3.935e-17\n",
      "Learner:4\tReward total:0\tRunning mean: 0.01311\n",
      "Learner:5\tReward total:0\tRunning mean: 2.164\n",
      "Learner:6\tReward total:0\tRunning mean: 4.959e-12\n",
      "Learner:7\tReward total:0\tRunning mean: 9.393e-08\n",
      "Learner:8\tReward total:158\tRunning mean: 147.0\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '0.29', '0.00', '117.31']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 4400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.03314\n",
      "Learner:1\tReward total:0\tRunning mean: 0.2268\n",
      "Learner:2\tReward total:0\tRunning mean: 0.1772\n",
      "Learner:3\tReward total:0\tRunning mean: 5.273e-18\n",
      "Learner:4\tReward total:0\tRunning mean: 0.001756\n",
      "Learner:5\tReward total:0\tRunning mean: 1.197\n",
      "Learner:6\tReward total:0\tRunning mean: 6.643e-13\n",
      "Learner:7\tReward total:0\tRunning mean: 1.259e-08\n",
      "Learner:8\tReward total:159\tRunning mean: 149.2\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '0.17', '0.00', '42.61']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "........................................................................................................................................................................................................\n",
      "Episode 4600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.00444\n",
      "Learner:1\tReward total:0\tRunning mean: 0.03039\n",
      "Learner:2\tReward total:0\tRunning mean: 0.05122\n",
      "Learner:3\tReward total:0\tRunning mean: 7.064e-19\n",
      "Learner:4\tReward total:0\tRunning mean: 0.0002353\n",
      "Learner:5\tReward total:0\tRunning mean: 0.1878\n",
      "Learner:6\tReward total:0\tRunning mean: 8.901e-14\n",
      "Learner:7\tReward total:0\tRunning mean: 1.686e-09\n",
      "Learner:8\tReward total:162\tRunning mean: 156.8\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '0.08', '9.06', '107.60']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 4800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0005949\n",
      "Learner:1\tReward total:0\tRunning mean: 0.07547\n",
      "Learner:2\tReward total:0\tRunning mean: 0.3697\n",
      "Learner:3\tReward total:0\tRunning mean: 9.465e-20\n",
      "Learner:4\tReward total:0\tRunning mean: 3.153e-05\n",
      "Learner:5\tReward total:0\tRunning mean: 0.1402\n",
      "Learner:6\tReward total:0\tRunning mean: 1.193e-14\n",
      "Learner:7\tReward total:0\tRunning mean: 2.259e-10\n",
      "Learner:8\tReward total:156\tRunning mean: 158.2\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '0.08', '0.00', '124.92']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 5000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 7.971e-05\n",
      "Learner:1\tReward total:0\tRunning mean: 0.1562\n",
      "Learner:2\tReward total:0\tRunning mean: 0.267\n",
      "Learner:3\tReward total:0\tRunning mean: 1.268e-20\n",
      "Learner:4\tReward total:0\tRunning mean: 4.224e-06\n",
      "Learner:5\tReward total:0\tRunning mean: 1.199\n",
      "Learner:6\tReward total:0\tRunning mean: 1.598e-15\n",
      "Learner:7\tReward total:0\tRunning mean: 3.027e-11\n",
      "Learner:8\tReward total:158\tRunning mean: 146.8\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '0.23', '0.02', '42.84']\n",
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Learner agent 8\n",
      "........................................................................................................................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.02841\n",
      "Learner:1\tReward total:0\tRunning mean: 0.02255\n",
      "Learner:2\tReward total:0\tRunning mean: 0.02309\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 0.01853\n",
      "Learner:5\tReward total:0\tRunning mean: 0.0\n",
      "Learner:6\tReward total:0\tRunning mean: 0.1472\n",
      "Learner:7\tReward total:5\tRunning mean: 0.6996\n",
      "Learner:8\tReward total:0\tRunning mean: 0.7756\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '96.01', '102.64', '175.67']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.006986\n",
      "Learner:1\tReward total:0\tRunning mean: 0.003021\n",
      "Learner:2\tReward total:0\tRunning mean: 0.003094\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 0.2926\n",
      "Learner:5\tReward total:0\tRunning mean: 0.0024\n",
      "Learner:6\tReward total:0\tRunning mean: 0.02283\n",
      "Learner:7\tReward total:0\tRunning mean: 0.85\n",
      "Learner:8\tReward total:40\tRunning mean: 19.79\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '78.93', '81.03', '99.88']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0009359\n",
      "Learner:1\tReward total:0\tRunning mean: 0.1118\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0004145\n",
      "Learner:3\tReward total:0\tRunning mean: 0.5466\n",
      "Learner:4\tReward total:36\tRunning mean: 7.278\n",
      "Learner:5\tReward total:0\tRunning mean: 0.0003215\n",
      "Learner:6\tReward total:0\tRunning mean: 0.1429\n",
      "Learner:7\tReward total:11\tRunning mean: 1.146\n",
      "Learner:8\tReward total:37\tRunning mean: 51.06\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '62.10', '131.15', '57.40', '64.05', '78.43', '104.61']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0001254\n",
      "Learner:1\tReward total:9\tRunning mean: 4.932\n",
      "Learner:2\tReward total:0\tRunning mean: 0.1563\n",
      "Learner:3\tReward total:0\tRunning mean: 0.3299\n",
      "Learner:4\tReward total:39\tRunning mean: 33.81\n",
      "Learner:5\tReward total:0\tRunning mean: 4.308e-05\n",
      "Learner:6\tReward total:0\tRunning mean: 0.4201\n",
      "Learner:7\tReward total:4\tRunning mean: 0.8959\n",
      "Learner:8\tReward total:37\tRunning mean: 50.08\n",
      "Max Norms =  ['67.58', '76.02', '26.09', '59.28', '120.30', '52.20', '76.57', '74.58', '117.50']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.02515\n",
      "Learner:1\tReward total:4\tRunning mean: 13.79\n",
      "Learner:2\tReward total:0\tRunning mean: 0.1318\n",
      "Learner:3\tReward total:0\tRunning mean: 0.1395\n",
      "Learner:4\tReward total:76\tRunning mean: 48.92\n",
      "Learner:5\tReward total:0\tRunning mean: 5.772e-06\n",
      "Learner:6\tReward total:0\tRunning mean: 0.5558\n",
      "Learner:7\tReward total:0\tRunning mean: 0.4162\n",
      "Learner:8\tReward total:11\tRunning mean: 40.88\n",
      "Max Norms =  ['75.02', '18.25', '46.23', '52.67', '79.21', '67.06', '68.05', '84.16', '95.75']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.02095\n",
      "Learner:1\tReward total:7\tRunning mean: 13.57\n",
      "Learner:2\tReward total:0\tRunning mean: 0.565\n",
      "Learner:3\tReward total:0\tRunning mean: 0.1402\n",
      "Learner:4\tReward total:86\tRunning mean: 56.25\n",
      "Learner:5\tReward total:0\tRunning mean: 0.02844\n",
      "Learner:6\tReward total:0\tRunning mean: 0.2326\n",
      "Learner:7\tReward total:0\tRunning mean: 0.3704\n",
      "Learner:8\tReward total:21\tRunning mean: 33.96\n",
      "Max Norms =  ['120.93', '19.54', '67.79', '25.68', '121.39', '71.53', '60.51', '78.76', '122.29']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.01003\n",
      "Learner:1\tReward total:12\tRunning mean: 13.1\n",
      "Learner:2\tReward total:3\tRunning mean: 1.963\n",
      "Learner:3\tReward total:0\tRunning mean: 0.08\n",
      "Learner:4\tReward total:91\tRunning mean: 62.86\n",
      "Learner:5\tReward total:0\tRunning mean: 0.00381\n",
      "Learner:6\tReward total:0\tRunning mean: 0.1719\n",
      "Learner:7\tReward total:0\tRunning mean: 0.2334\n",
      "Learner:8\tReward total:14\tRunning mean: 38.21\n",
      "Max Norms =  ['50.01', '37.78', '30.40', '15.54', '105.93', '29.76', '94.02', '95.80', '80.75']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.001343\n",
      "Learner:1\tReward total:9\tRunning mean: 11.34\n",
      "Learner:2\tReward total:3\tRunning mean: 1.603\n",
      "Learner:3\tReward total:0\tRunning mean: 0.06618\n",
      "Learner:4\tReward total:79\tRunning mean: 71.82\n",
      "Learner:5\tReward total:0\tRunning mean: 0.0005105\n",
      "Learner:6\tReward total:0\tRunning mean: 0.07782\n",
      "Learner:7\tReward total:0\tRunning mean: 0.1228\n",
      "Learner:8\tReward total:35\tRunning mean: 35.23\n",
      "Max Norms =  ['83.79', '5.24', '35.19', '23.59', '106.33', '26.29', '36.84', '67.96', '110.42']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "........................................................................................................................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.07334\n",
      "Learner:1\tReward total:32\tRunning mean: 20.02\n",
      "Learner:2\tReward total:1\tRunning mean: 1.843\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0125\n",
      "Learner:4\tReward total:52\tRunning mean: 66.65\n",
      "Learner:5\tReward total:0\tRunning mean: 6.839e-05\n",
      "Learner:6\tReward total:0\tRunning mean: 0.1131\n",
      "Learner:7\tReward total:0\tRunning mean: 0.3806\n",
      "Learner:8\tReward total:38\tRunning mean: 39.17\n",
      "Max Norms =  ['74.28', '67.17', '73.25', '24.96', '63.00', '24.38', '66.19', '214.57', '102.55']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.02853\n",
      "Learner:1\tReward total:11\tRunning mean: 43.11\n",
      "Learner:2\tReward total:0\tRunning mean: 0.6215\n",
      "Learner:3\tReward total:0\tRunning mean: 0.02175\n",
      "Learner:4\tReward total:44\tRunning mean: 32.76\n",
      "Learner:5\tReward total:0\tRunning mean: 9.163e-06\n",
      "Learner:6\tReward total:0\tRunning mean: 0.1012\n",
      "Learner:7\tReward total:0\tRunning mean: 0.3729\n",
      "Learner:8\tReward total:61\tRunning mean: 25.45\n",
      "Max Norms =  ['35.34', '148.39', '22.13', '56.99', '143.94', '28.42', '67.64', '88.54', '80.90']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.008796\n",
      "Learner:1\tReward total:70\tRunning mean: 51.98\n",
      "Learner:2\tReward total:0\tRunning mean: 0.5543\n",
      "Learner:3\tReward total:0\tRunning mean: 0.01924\n",
      "Learner:4\tReward total:1\tRunning mean: 22.55\n",
      "Learner:5\tReward total:0\tRunning mean: 1.228e-06\n",
      "Learner:6\tReward total:0\tRunning mean: 0.2232\n",
      "Learner:7\tReward total:0\tRunning mean: 0.4247\n",
      "Learner:8\tReward total:26\tRunning mean: 22.75\n",
      "Max Norms =  ['66.37', '94.15', '16.05', '46.44', '114.22', '24.59', '57.86', '74.79', '44.32']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.001178\n",
      "Learner:1\tReward total:44\tRunning mean: 53.43\n",
      "Learner:2\tReward total:0\tRunning mean: 0.1005\n",
      "Learner:3\tReward total:0\tRunning mean: 0.1071\n",
      "Learner:4\tReward total:17\tRunning mean: 23.62\n",
      "Learner:5\tReward total:0\tRunning mean: 1.645e-07\n",
      "Learner:6\tReward total:0\tRunning mean: 0.3191\n",
      "Learner:7\tReward total:1\tRunning mean: 0.7989\n",
      "Learner:8\tReward total:22\tRunning mean: 25.61\n",
      "Max Norms =  ['61.73', '91.76', '8.40', '26.18', '79.00', '52.57', '74.34', '120.99', '108.80']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0001579\n",
      "Learner:1\tReward total:20\tRunning mean: 60.02\n",
      "Learner:2\tReward total:0\tRunning mean: 0.03375\n",
      "Learner:3\tReward total:0\tRunning mean: 0.01912\n",
      "Learner:4\tReward total:20\tRunning mean: 21.19\n",
      "Learner:5\tReward total:0\tRunning mean: 0.005812\n",
      "Learner:6\tReward total:0\tRunning mean: 0.1285\n",
      "Learner:7\tReward total:0\tRunning mean: 0.2237\n",
      "Learner:8\tReward total:49\tRunning mean: 25.81\n",
      "Max Norms =  ['56.45', '65.47', '2.95', '0.83', '86.35', '52.09', '84.14', '77.79', '84.29']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 2.115e-05\n",
      "Learner:1\tReward total:104\tRunning mean: 51.43\n",
      "Learner:2\tReward total:0\tRunning mean: 0.004522\n",
      "Learner:3\tReward total:0\tRunning mean: 0.002562\n",
      "Learner:4\tReward total:0\tRunning mean: 27.14\n",
      "Learner:5\tReward total:0\tRunning mean: 0.0149\n",
      "Learner:6\tReward total:0\tRunning mean: 0.1135\n",
      "Learner:7\tReward total:0\tRunning mean: 0.2275\n",
      "Learner:8\tReward total:3\tRunning mean: 27.58\n",
      "Max Norms =  ['15.26', '47.66', '1.50', '0.00', '0.00', '0.00', '91.99', '123.11', '94.50']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 2.834e-06\n",
      "Learner:1\tReward total:62\tRunning mean: 48.03\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0006059\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0003432\n",
      "Learner:4\tReward total:11\tRunning mean: 19.02\n",
      "Learner:5\tReward total:0\tRunning mean: 0.001996\n",
      "Learner:6\tReward total:0\tRunning mean: 0.07306\n",
      "Learner:7\tReward total:0\tRunning mean: 0.2155\n",
      "Learner:8\tReward total:12\tRunning mean: 30.27\n",
      "Max Norms =  ['26.80', '102.70', '0.29', '4.14', '61.03', '19.85', '83.06', '99.47', '144.43']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 3.797e-07\n",
      "Learner:1\tReward total:53\tRunning mean: 57.64\n",
      "Learner:2\tReward total:0\tRunning mean: 8.117e-05\n",
      "Learner:3\tReward total:0\tRunning mean: 4.598e-05\n",
      "Learner:4\tReward total:17\tRunning mean: 13.34\n",
      "Learner:5\tReward total:0\tRunning mean: 0.0002674\n",
      "Learner:6\tReward total:0\tRunning mean: 0.1548\n",
      "Learner:7\tReward total:0\tRunning mean: 0.1829\n",
      "Learner:8\tReward total:36\tRunning mean: 33.04\n",
      "Max Norms =  ['13.69', '55.16', '0.16', '24.50', '73.02', '52.95', '68.22', '82.54', '155.60']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 5.088e-08\n",
      "Learner:1\tReward total:37\tRunning mean: 46.73\n",
      "Learner:2\tReward total:0\tRunning mean: 1.088e-05\n",
      "Learner:3\tReward total:0\tRunning mean: 6.161e-06\n",
      "Learner:4\tReward total:75\tRunning mean: 29.18\n",
      "Learner:5\tReward total:0\tRunning mean: 3.583e-05\n",
      "Learner:6\tReward total:0\tRunning mean: 0.02478\n",
      "Learner:7\tReward total:0\tRunning mean: 0.04153\n",
      "Learner:8\tReward total:25\tRunning mean: 51.91\n",
      "Max Norms =  ['13.74', '60.98', '0.08', '0.04', '138.30', '27.82', '31.21', '17.21', '79.66']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 6.817e-09\n",
      "Learner:1\tReward total:0\tRunning mean: 37.23\n",
      "Learner:2\tReward total:0\tRunning mean: 1.457e-06\n",
      "Learner:3\tReward total:0\tRunning mean: 8.254e-07\n",
      "Learner:4\tReward total:97\tRunning mean: 27.52\n",
      "Learner:5\tReward total:0\tRunning mean: 4.8e-06\n",
      "Learner:6\tReward total:0\tRunning mean: 0.00332\n",
      "Learner:7\tReward total:0\tRunning mean: 0.03617\n",
      "Learner:8\tReward total:25\tRunning mean: 69.12\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.18', '178.54', '24.92', '49.55', '47.30', '92.56']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 9.133e-10\n",
      "Learner:1\tReward total:23\tRunning mean: 51.08\n",
      "Learner:2\tReward total:0\tRunning mean: 1.952e-07\n",
      "Learner:3\tReward total:0\tRunning mean: 1.106e-07\n",
      "Learner:4\tReward total:60\tRunning mean: 32.99\n",
      "Learner:5\tReward total:0\tRunning mean: 6.432e-07\n",
      "Learner:6\tReward total:0\tRunning mean: 0.0004448\n",
      "Learner:7\tReward total:0\tRunning mean: 0.01239\n",
      "Learner:8\tReward total:56\tRunning mean: 52.7\n",
      "Max Norms =  ['0.85', '53.22', '0.11', '0.17', '172.75', '8.42', '63.32', '96.65', '88.86']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "........................................................................................................................................................................................................\n",
      "Episode 4000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.224e-10\n",
      "Learner:1\tReward total:81\tRunning mean: 43.79\n",
      "Learner:2\tReward total:0\tRunning mean: 2.616e-08\n",
      "Learner:3\tReward total:0\tRunning mean: 1.482e-08\n",
      "Learner:4\tReward total:26\tRunning mean: 37.9\n",
      "Learner:5\tReward total:0\tRunning mean: 8.617e-08\n",
      "Learner:6\tReward total:0\tRunning mean: 5.96e-05\n",
      "Learner:7\tReward total:0\tRunning mean: 0.1039\n",
      "Learner:8\tReward total:27\tRunning mean: 52.13\n",
      "Max Norms =  ['0.21', '54.07', '0.00', '0.55', '155.53', '34.56', '31.70', '78.47', '34.53']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 4200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.639e-11\n",
      "Learner:1\tReward total:71\tRunning mean: 28.76\n",
      "Learner:2\tReward total:0\tRunning mean: 3.504e-09\n",
      "Learner:3\tReward total:0\tRunning mean: 1.985e-09\n",
      "Learner:4\tReward total:4\tRunning mean: 38.49\n",
      "Learner:5\tReward total:0\tRunning mean: 1.154e-08\n",
      "Learner:6\tReward total:0\tRunning mean: 7.985e-06\n",
      "Learner:7\tReward total:0\tRunning mean: 0.2956\n",
      "Learner:8\tReward total:25\tRunning mean: 54.31\n",
      "Max Norms =  ['0.10', '87.62', '0.00', '0.06', '61.14', '29.12', '27.98', '67.71', '130.12']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 4400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 2.196e-12\n",
      "Learner:1\tReward total:77\tRunning mean: 48.58\n",
      "Learner:2\tReward total:0\tRunning mean: 4.695e-10\n",
      "Learner:3\tReward total:0\tRunning mean: 2.66e-10\n",
      "Learner:4\tReward total:31\tRunning mean: 31.42\n",
      "Learner:5\tReward total:0\tRunning mean: 1.547e-09\n",
      "Learner:6\tReward total:0\tRunning mean: 1.07e-06\n",
      "Learner:7\tReward total:0\tRunning mean: 0.08966\n",
      "Learner:8\tReward total:14\tRunning mean: 42.84\n",
      "Max Norms =  ['0.33', '83.59', '0.02', '0.23', '110.63', '13.16', '61.96', '67.14', '137.90']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 4600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 2.943e-13\n",
      "Learner:1\tReward total:0\tRunning mean: 47.14\n",
      "Learner:2\tReward total:0\tRunning mean: 6.291e-11\n",
      "Learner:3\tReward total:0\tRunning mean: 3.564e-11\n",
      "Learner:4\tReward total:96\tRunning mean: 27.28\n",
      "Learner:5\tReward total:0\tRunning mean: 2.072e-10\n",
      "Learner:6\tReward total:0\tRunning mean: 0.02824\n",
      "Learner:7\tReward total:0\tRunning mean: 0.01201\n",
      "Learner:8\tReward total:52\tRunning mean: 65.56\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.02', '111.85', '79.59', '73.07', '8.51', '107.64']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 4800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 3.943e-14\n",
      "Learner:1\tReward total:0\tRunning mean: 16.72\n",
      "Learner:2\tReward total:0\tRunning mean: 8.428e-12\n",
      "Learner:3\tReward total:0\tRunning mean: 4.774e-12\n",
      "Learner:4\tReward total:0\tRunning mean: 23.89\n",
      "Learner:5\tReward total:0\tRunning mean: 2.777e-11\n",
      "Learner:6\tReward total:0\tRunning mean: 0.003784\n",
      "Learner:7\tReward total:0\tRunning mean: 0.001609\n",
      "Learner:8\tReward total:161\tRunning mean: 105.1\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '9.43', '0.29', '227.06']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 5000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 5.282e-15\n",
      "Learner:1\tReward total:0\tRunning mean: 27.38\n",
      "Learner:2\tReward total:0\tRunning mean: 1.129e-12\n",
      "Learner:3\tReward total:0\tRunning mean: 6.397e-13\n",
      "Learner:4\tReward total:2\tRunning mean: 38.9\n",
      "Learner:5\tReward total:0\tRunning mean: 3.72e-12\n",
      "Learner:6\tReward total:0\tRunning mean: 0.000507\n",
      "Learner:7\tReward total:0\tRunning mean: 0.0002156\n",
      "Learner:8\tReward total:165\tRunning mean: 82.04\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '70.49', '43.87', '70.44', '0.06', '54.54']\n",
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Learner agent 8\n",
      "........................................................................................................................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:0\tRunning mean: 0.03979\n",
      "Learner:2\tReward total:0\tRunning mean: 0.005523\n",
      "Learner:3\tReward total:0\tRunning mean: 0.07891\n",
      "Learner:4\tReward total:0\tRunning mean: 0.242\n",
      "Learner:5\tReward total:0\tRunning mean: 0.01237\n",
      "Learner:6\tReward total:0\tRunning mean: 0.1603\n",
      "Learner:7\tReward total:5\tRunning mean: 0.6829\n",
      "Learner:8\tReward total:16\tRunning mean: 2.691\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '61.85', '116.03', '69.47']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.004444\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0343\n",
      "Learner:2\tReward total:0\tRunning mean: 0.00074\n",
      "Learner:3\tReward total:0\tRunning mean: 0.113\n",
      "Learner:4\tReward total:0\tRunning mean: 0.1909\n",
      "Learner:5\tReward total:3\tRunning mean: 0.2811\n",
      "Learner:6\tReward total:0\tRunning mean: 0.04726\n",
      "Learner:7\tReward total:0\tRunning mean: 1.259\n",
      "Learner:8\tReward total:62\tRunning mean: 19.67\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '75.20', '67.31', '86.64', '77.18', '80.76', '96.52']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0005955\n",
      "Learner:1\tReward total:0\tRunning mean: 0.1081\n",
      "Learner:2\tReward total:0\tRunning mean: 0.06607\n",
      "Learner:3\tReward total:0\tRunning mean: 0.02731\n",
      "Learner:4\tReward total:18\tRunning mean: 3.906\n",
      "Learner:5\tReward total:7\tRunning mean: 6.544\n",
      "Learner:6\tReward total:0\tRunning mean: 0.4726\n",
      "Learner:7\tReward total:10\tRunning mean: 1.307\n",
      "Learner:8\tReward total:49\tRunning mean: 42.22\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '51.86', '68.23', '105.38', '76.24', '76.25', '94.55']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 7.978e-05\n",
      "Learner:1\tReward total:0\tRunning mean: 0.06534\n",
      "Learner:2\tReward total:0\tRunning mean: 0.1554\n",
      "Learner:3\tReward total:0\tRunning mean: 0.05062\n",
      "Learner:4\tReward total:18\tRunning mean: 15.07\n",
      "Learner:5\tReward total:9\tRunning mean: 11.64\n",
      "Learner:6\tReward total:0\tRunning mean: 0.2828\n",
      "Learner:7\tReward total:11\tRunning mean: 2.838\n",
      "Learner:8\tReward total:48\tRunning mean: 49.68\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '66.91', '82.54', '68.73', '91.44', '97.57', '97.88']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.069e-05\n",
      "Learner:1\tReward total:0\tRunning mean: 0.07584\n",
      "Learner:2\tReward total:0\tRunning mean: 6.064\n",
      "Learner:3\tReward total:0\tRunning mean: 0.006782\n",
      "Learner:4\tReward total:19\tRunning mean: 21.59\n",
      "Learner:5\tReward total:22\tRunning mean: 12.96\n",
      "Learner:6\tReward total:2\tRunning mean: 0.3958\n",
      "Learner:7\tReward total:3\tRunning mean: 2.907\n",
      "Learner:8\tReward total:67\tRunning mean: 47.24\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '44.59', '54.12', '37.54', '59.53', '73.90', '117.36']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "........................................................................................................................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.432e-06\n",
      "Learner:1\tReward total:0\tRunning mean: 0.03067\n",
      "Learner:2\tReward total:2\tRunning mean: 8.771\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0009087\n",
      "Learner:4\tReward total:38\tRunning mean: 28.94\n",
      "Learner:5\tReward total:11\tRunning mean: 13.89\n",
      "Learner:6\tReward total:0\tRunning mean: 0.2041\n",
      "Learner:7\tReward total:0\tRunning mean: 1.286\n",
      "Learner:8\tReward total:29\tRunning mean: 41.71\n",
      "Max Norms =  ['0.22', '67.78', '79.76', '25.51', '78.28', '68.09', '51.44', '67.15', '99.39']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.919e-07\n",
      "Learner:1\tReward total:0\tRunning mean: 0.00411\n",
      "Learner:2\tReward total:37\tRunning mean: 15.96\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0001217\n",
      "Learner:4\tReward total:34\tRunning mean: 30.77\n",
      "Learner:5\tReward total:8\tRunning mean: 14.08\n",
      "Learner:6\tReward total:0\tRunning mean: 0.1796\n",
      "Learner:7\tReward total:0\tRunning mean: 0.8822\n",
      "Learner:8\tReward total:20\tRunning mean: 37.26\n",
      "Max Norms =  ['0.30', '53.93', '90.31', '28.94', '50.43', '33.23', '84.30', '89.25', '79.99']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 2.571e-08\n",
      "Learner:1\tReward total:0\tRunning mean: 0.1323\n",
      "Learner:2\tReward total:28\tRunning mean: 22.59\n",
      "Learner:3\tReward total:0\tRunning mean: 1.631e-05\n",
      "Learner:4\tReward total:52\tRunning mean: 34.73\n",
      "Learner:5\tReward total:9\tRunning mean: 11.27\n",
      "Learner:6\tReward total:0\tRunning mean: 0.04271\n",
      "Learner:7\tReward total:0\tRunning mean: 0.5612\n",
      "Learner:8\tReward total:17\tRunning mean: 27.18\n",
      "Max Norms =  ['0.24', '57.23', '129.22', '28.86', '38.49', '42.95', '79.37', '68.70', '66.05']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 3.444e-09\n",
      "Learner:1\tReward total:0\tRunning mean: 0.131\n",
      "Learner:2\tReward total:29\tRunning mean: 23.0\n",
      "Learner:3\tReward total:0\tRunning mean: 2.185e-06\n",
      "Learner:4\tReward total:42\tRunning mean: 35.16\n",
      "Learner:5\tReward total:16\tRunning mean: 13.32\n",
      "Learner:6\tReward total:0\tRunning mean: 0.005722\n",
      "Learner:7\tReward total:0\tRunning mean: 0.4677\n",
      "Learner:8\tReward total:18\tRunning mean: 24.79\n",
      "Max Norms =  ['8.86', '88.91', '90.64', '98.78', '65.49', '36.99', '39.61', '57.29', '59.03']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 4.614e-10\n",
      "Learner:1\tReward total:0\tRunning mean: 0.1161\n",
      "Learner:2\tReward total:21\tRunning mean: 25.68\n",
      "Learner:3\tReward total:0\tRunning mean: 2.928e-07\n",
      "Learner:4\tReward total:39\tRunning mean: 29.74\n",
      "Learner:5\tReward total:14\tRunning mean: 13.56\n",
      "Learner:6\tReward total:0\tRunning mean: 0.0007666\n",
      "Learner:7\tReward total:0\tRunning mean: 0.2879\n",
      "Learner:8\tReward total:39\tRunning mean: 28.63\n",
      "Max Norms =  ['0.14', '61.58', '69.17', '40.08', '42.02', '24.28', '39.83', '55.28', '85.45']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 6.182e-11\n",
      "Learner:1\tReward total:0\tRunning mean: 0.06304\n",
      "Learner:2\tReward total:41\tRunning mean: 31.99\n",
      "Learner:3\tReward total:0\tRunning mean: 3.923e-08\n",
      "Learner:4\tReward total:40\tRunning mean: 31.66\n",
      "Learner:5\tReward total:14\tRunning mean: 13.57\n",
      "Learner:6\tReward total:0\tRunning mean: 0.00933\n",
      "Learner:7\tReward total:1\tRunning mean: 0.5115\n",
      "Learner:8\tReward total:12\tRunning mean: 26.9\n",
      "Max Norms =  ['22.65', '62.69', '82.88', '67.07', '85.53', '32.63', '60.91', '55.07', '59.52']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 8.283e-12\n",
      "Learner:1\tReward total:0\tRunning mean: 0.02328\n",
      "Learner:2\tReward total:49\tRunning mean: 25.62\n",
      "Learner:3\tReward total:0\tRunning mean: 5.256e-09\n",
      "Learner:4\tReward total:19\tRunning mean: 30.2\n",
      "Learner:5\tReward total:14\tRunning mean: 13.86\n",
      "Learner:6\tReward total:7\tRunning mean: 0.3541\n",
      "Learner:7\tReward total:0\tRunning mean: 0.7371\n",
      "Learner:8\tReward total:21\tRunning mean: 33.97\n",
      "Max Norms =  ['0.76', '38.17', '85.23', '29.68', '70.85', '34.51', '77.66', '45.12', '62.36']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.11e-12\n",
      "Learner:1\tReward total:0\tRunning mean: 0.01106\n",
      "Learner:2\tReward total:36\tRunning mean: 29.33\n",
      "Learner:3\tReward total:0\tRunning mean: 7.042e-10\n",
      "Learner:4\tReward total:8\tRunning mean: 31.43\n",
      "Learner:5\tReward total:16\tRunning mean: 13.6\n",
      "Learner:6\tReward total:0\tRunning mean: 0.2465\n",
      "Learner:7\tReward total:0\tRunning mean: 0.5534\n",
      "Learner:8\tReward total:16\tRunning mean: 32.1\n",
      "Max Norms =  ['0.03', '60.79', '107.67', '8.29', '53.15', '44.68', '70.84', '32.41', '41.03']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 2800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.487e-13\n",
      "Learner:1\tReward total:0\tRunning mean: 0.003608\n",
      "Learner:2\tReward total:21\tRunning mean: 32.7\n",
      "Learner:3\tReward total:0\tRunning mean: 9.435e-11\n",
      "Learner:4\tReward total:38\tRunning mean: 30.32\n",
      "Learner:5\tReward total:16\tRunning mean: 13.26\n",
      "Learner:6\tReward total:0\tRunning mean: 0.04506\n",
      "Learner:7\tReward total:0\tRunning mean: 0.1932\n",
      "Learner:8\tReward total:41\tRunning mean: 27.8\n",
      "Max Norms =  ['0.31', '52.97', '79.88', '28.06', '35.26', '33.29', '69.52', '14.93', '69.67']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.992e-14\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0004835\n",
      "Learner:2\tReward total:34\tRunning mean: 40.05\n",
      "Learner:3\tReward total:0\tRunning mean: 1.264e-11\n",
      "Learner:4\tReward total:14\tRunning mean: 25.85\n",
      "Learner:5\tReward total:12\tRunning mean: 14.29\n",
      "Learner:6\tReward total:0\tRunning mean: 0.008437\n",
      "Learner:7\tReward total:0\tRunning mean: 0.3622\n",
      "Learner:8\tReward total:22\tRunning mean: 28.51\n",
      "Max Norms =  ['0.54', '69.86', '113.69', '1.16', '55.64', '2.59', '94.46', '57.40', '65.43']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 2.669e-15\n",
      "Learner:1\tReward total:0\tRunning mean: 6.477e-05\n",
      "Learner:2\tReward total:50\tRunning mean: 40.72\n",
      "Learner:3\tReward total:0\tRunning mean: 1.694e-12\n",
      "Learner:4\tReward total:9\tRunning mean: 15.92\n",
      "Learner:5\tReward total:5\tRunning mean: 7.346\n",
      "Learner:6\tReward total:0\tRunning mean: 0.00113\n",
      "Learner:7\tReward total:1\tRunning mean: 0.4835\n",
      "Learner:8\tReward total:42\tRunning mean: 27.81\n",
      "Max Norms =  ['0.13', '67.14', '109.60', '5.37', '20.81', '46.21', '78.93', '70.90', '108.44']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "........................................................................................................................................................................................................\n",
      "Episode 3400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 3.576e-16\n",
      "Learner:1\tReward total:0\tRunning mean: 8.678e-06\n",
      "Learner:2\tReward total:60\tRunning mean: 40.8\n",
      "Learner:3\tReward total:0\tRunning mean: 2.269e-13\n",
      "Learner:4\tReward total:16\tRunning mean: 17.75\n",
      "Learner:5\tReward total:13\tRunning mean: 7.81\n",
      "Learner:6\tReward total:0\tRunning mean: 0.01858\n",
      "Learner:7\tReward total:4\tRunning mean: 0.3393\n",
      "Learner:8\tReward total:26\tRunning mean: 21.87\n",
      "Max Norms =  ['0.25', '93.14', '73.37', '0.10', '19.04', '31.63', '46.38', '83.88', '113.68']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 4.791e-17\n",
      "Learner:1\tReward total:0\tRunning mean: 1.163e-06\n",
      "Learner:2\tReward total:77\tRunning mean: 48.07\n",
      "Learner:3\tReward total:0\tRunning mean: 3.04e-14\n",
      "Learner:4\tReward total:8\tRunning mean: 24.97\n",
      "Learner:5\tReward total:3\tRunning mean: 8.963\n",
      "Learner:6\tReward total:0\tRunning mean: 0.005668\n",
      "Learner:7\tReward total:0\tRunning mean: 0.1666\n",
      "Learner:8\tReward total:20\tRunning mean: 21.53\n",
      "Max Norms =  ['0.36', '90.96', '109.39', '0.45', '35.16', '16.57', '44.75', '76.29', '64.52']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 3800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 6.419e-18\n",
      "Learner:1\tReward total:0\tRunning mean: 1.558e-07\n",
      "Learner:2\tReward total:46\tRunning mean: 48.08\n",
      "Learner:3\tReward total:0\tRunning mean: 4.073e-15\n",
      "Learner:4\tReward total:33\tRunning mean: 23.98\n",
      "Learner:5\tReward total:17\tRunning mean: 8.2\n",
      "Learner:6\tReward total:0\tRunning mean: 0.04631\n",
      "Learner:7\tReward total:0\tRunning mean: 0.3101\n",
      "Learner:8\tReward total:7\tRunning mean: 22.66\n",
      "Max Norms =  ['0.00', '30.96', '72.91', '22.00', '58.62', '94.23', '37.99', '92.44', '73.45']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 4000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 8.6e-19\n",
      "Learner:1\tReward total:0\tRunning mean: 2.087e-08\n",
      "Learner:2\tReward total:50\tRunning mean: 41.76\n",
      "Learner:3\tReward total:0\tRunning mean: 5.457e-16\n",
      "Learner:4\tReward total:11\tRunning mean: 22.98\n",
      "Learner:5\tReward total:13\tRunning mean: 8.754\n",
      "Learner:6\tReward total:0\tRunning mean: 0.007716\n",
      "Learner:7\tReward total:5\tRunning mean: 0.5239\n",
      "Learner:8\tReward total:9\tRunning mean: 24.74\n",
      "Max Norms =  ['0.00', '25.53', '143.66', '0.25', '29.96', '0.22', '17.87', '57.49', '78.96']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 4200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.152e-19\n",
      "Learner:1\tReward total:0\tRunning mean: 2.796e-09\n",
      "Learner:2\tReward total:17\tRunning mean: 40.17\n",
      "Learner:3\tReward total:0\tRunning mean: 7.311e-17\n",
      "Learner:4\tReward total:61\tRunning mean: 25.16\n",
      "Learner:5\tReward total:3\tRunning mean: 8.058\n",
      "Learner:6\tReward total:0\tRunning mean: 0.001034\n",
      "Learner:7\tReward total:0\tRunning mean: 0.7753\n",
      "Learner:8\tReward total:15\tRunning mean: 21.55\n",
      "Max Norms =  ['0.01', '75.97', '71.22', '0.03', '100.22', '1.33', '67.03', '68.91', '59.53']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 4400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.544e-20\n",
      "Learner:1\tReward total:0\tRunning mean: 3.747e-10\n",
      "Learner:2\tReward total:21\tRunning mean: 42.57\n",
      "Learner:3\tReward total:0\tRunning mean: 9.796e-18\n",
      "Learner:4\tReward total:32\tRunning mean: 26.74\n",
      "Learner:5\tReward total:10\tRunning mean: 6.421\n",
      "Learner:6\tReward total:0\tRunning mean: 0.007316\n",
      "Learner:7\tReward total:5\tRunning mean: 4.481\n",
      "Learner:8\tReward total:22\tRunning mean: 18.96\n",
      "Max Norms =  ['0.00', '37.61', '87.79', '0.02', '42.65', '0.05', '63.08', '81.47', '65.26']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 4600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 2.068e-21\n",
      "Learner:1\tReward total:0\tRunning mean: 5.02e-11\n",
      "Learner:2\tReward total:67\tRunning mean: 45.41\n",
      "Learner:3\tReward total:0\tRunning mean: 1.312e-18\n",
      "Learner:4\tReward total:21\tRunning mean: 27.3\n",
      "Learner:5\tReward total:7\tRunning mean: 5.194\n",
      "Learner:6\tReward total:0\tRunning mean: 0.0009802\n",
      "Learner:7\tReward total:3\tRunning mean: 5.298\n",
      "Learner:8\tReward total:15\tRunning mean: 26.77\n",
      "Max Norms =  ['0.00', '30.19', '164.61', '0.01', '70.19', '2.33', '29.33', '66.36', '44.96']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 4800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 2.771e-22\n",
      "Learner:1\tReward total:0\tRunning mean: 6.725e-12\n",
      "Learner:2\tReward total:50\tRunning mean: 57.1\n",
      "Learner:3\tReward total:0\tRunning mean: 1.758e-19\n",
      "Learner:4\tReward total:10\tRunning mean: 19.38\n",
      "Learner:5\tReward total:10\tRunning mean: 5.676\n",
      "Learner:6\tReward total:0\tRunning mean: 0.0001313\n",
      "Learner:7\tReward total:5\tRunning mean: 6.152\n",
      "Learner:8\tReward total:20\tRunning mean: 20.6\n",
      "Max Norms =  ['0.00', '26.37', '133.35', '0.01', '52.63', '10.66', '43.09', '112.51', '62.95']\n",
      "........................................................................................................................................................................................................\n",
      "Episode 5000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 3.713e-23\n",
      "Learner:1\tReward total:0\tRunning mean: 9.01e-13\n",
      "Learner:2\tReward total:44\tRunning mean: 60.79\n",
      "Learner:3\tReward total:0\tRunning mean: 2.356e-20\n",
      "Learner:4\tReward total:38\tRunning mean: 22.56\n",
      "Learner:5\tReward total:1\tRunning mean: 2.614\n",
      "Learner:6\tReward total:0\tRunning mean: 1.759e-05\n",
      "Learner:7\tReward total:13\tRunning mean: 5.302\n",
      "Learner:8\tReward total:17\tRunning mean: 16.82\n",
      "Max Norms =  ['0.00', '27.87', '58.20', '0.00', '140.67', '0.35', '43.31', '89.63', '75.80']\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import sys\n",
    "import pickle\n",
    "from collections import deque\n",
    "\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import numpy as np\n",
    "\n",
    "# Initialize agents parameters\n",
    "#   9 agents - 9 learning agents, 0 trained agent, 0 random agent\n",
    "num_learners = 9\n",
    "num_trained = 0\n",
    "num_rdn = 0\n",
    "num_statics = num_trained + num_rdn\n",
    "num_agents = num_learners + num_statics  \n",
    "\n",
    "# Initialize environment\n",
    "game = \"Gather\"\n",
    "num_actions = 8                       # 8 actions in Gathering\n",
    "\n",
    "# Initialize training parameters\n",
    "warm_start = False\n",
    "num_frames = 4      # environ observation consists of a list of 4 stacked frames per agent\n",
    "max_episodes = 5000\n",
    "max_frames = 300\n",
    "max_frames_ep = 0   # track highest number of frames an episode can last\n",
    "\n",
    "# These trainer parameters works for Atari Breakout\n",
    "gamma = 0.99  \n",
    "lr = 1e-3\n",
    "temp_start = 1.8  # Temperature for explore/exploit\n",
    "temp_end = 1.0\n",
    "log_interval = 200\n",
    "save_interval = 500\n",
    "\n",
    "def unpack_env_obs(env_obs):\n",
    "    \"\"\"\n",
    "    Gathering is a partially-observable Markov Game. env_obs returned by GatheringEnv is a numpy \n",
    "    array of dimension (num_agent, 800), which represents the agents' observations of the game.\n",
    "\n",
    "    The 800 elements (view_box) encodes 4 layers of 10x20 pixels frames in the format:\n",
    "    (viewbox_width, viewbox_depth, 4).\n",
    "    \n",
    "    This code reshapes the above into stacked frames that can be accepted by the Policy class:\n",
    "    (batch_idx, in_channel, width, height)\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    num_agents = len(env_obs)  # environ observations is a list of agents' observations\n",
    "    \n",
    "    obs = []\n",
    "    for i in range(num_agents):\n",
    "        x = env_obs[i]   # take the indexed agent's observation\n",
    "        x = torch.Tensor(x)   # Convert to tensor\n",
    "        \n",
    "        # Policy is a 3-layer CNN\n",
    "        x = x.view(1, 10, 20, -1)  # reshape into environment defined stacked frames\n",
    "        x = x.permute(0, 3, 1, 2)  # permute to Policy accepted stacked frames\n",
    "        obs.append(x)\n",
    "        \n",
    "    return obs  # return a list of Tensors\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "For now, we do not implement LSTM            \n",
    "# LSTM Change: Need to cycle hx and cx thru function\n",
    "def select_action(model, state, lstm_hc, cuda):\n",
    "    hx , cx = lstm_hc \n",
    "    num_frames, height, width = state.shape\n",
    "    state = torch.FloatTensor(state.reshape(-1, num_frames, height, width))\n",
    "\n",
    "    if cuda:\n",
    "        state = state.cuda()\n",
    "\n",
    "    probs, value, (hx, cx) = model((Variable(state), (hx, cx)))\n",
    "\n",
    "    m = torch.distributions.Categorical(probs)\n",
    "    action = m.sample()\n",
    "    log_prob = m.log_prob(action)\n",
    "    # LSTM Change: Need to cycle hx and cx thru function\n",
    "    return action.data[0], log_prob, value, (hx, cx)\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "def select_learner_action(model, obs, cuda):\n",
    "    \"\"\"\n",
    "    This code expects obs to be an array of stacked frames of the following dim:\n",
    "    (batch_idx, in_channel, width, height)\n",
    "    \n",
    "    This is inputted into model - the agent's Policy, which outputs a probability \n",
    "    distribution over available actions.\n",
    "    \n",
    "    Policy gradient is implemented using torch.distributions.Categorical. \n",
    "    \"\"\"\n",
    "    \n",
    "    # Policy is a 3-layer CNN\n",
    "    # _, num_frames, width, height = obs.shape\n",
    "    # obs = torch.FloatTensor(obs.reshape(-1, num_frames, width, height))\n",
    "    \n",
    "    # Policy is a 2-layer NN for now\n",
    "    # obs = obs.view(1, -1)\n",
    "   \n",
    "    if cuda:\n",
    "        obs = obs.cuda()\n",
    "      \n",
    "    probs = model(obs)\n",
    "    m = torch.distributions.Categorical(probs)\n",
    "    action = m.sample()\n",
    "    log_prob = m.log_prob(action)\n",
    "\n",
    "    return action.item(), log_prob \n",
    "\n",
    "\n",
    "def load_info(agents, narrate=False):\n",
    "    for i in range(num_agents):    \n",
    "        agents[i].load_info(info[i])\n",
    "        if narrate:\n",
    "            if agents[i].tagged:\n",
    "                print('frame {}, agent{} is tagged'.format(frame,i))\n",
    "            if agents[i].laser_fired:\n",
    "                print('frame {}, agent{} fires its laser'.format(frame,i))\n",
    "                print('and hit {} US and {} THEM'.format(agents[i].US_hit, agents[i].THEM_hit))\n",
    "    return\n",
    "\n",
    "\n",
    "# The main code starts here!!!\n",
    "\n",
    "# Cultural parameter search\n",
    "cultures =[\n",
    "            {'name':'cooperative', 'coop_factor':100},\n",
    "            {'name':'cooperative', 'coop_factor':90},\n",
    "            {'name':'cooperative', 'coop_factor':80},\n",
    "            {'name':'cooperative', 'coop_factor':70},\n",
    "           #{'name':'cooperative', 'coop_factor':50},\n",
    "            {'name':'cooperative', 'coop_factor':40},\n",
    "            {'name':'cooperative', 'coop_factor':30},    \n",
    "           #{'name':'cooperative', 'coop_factor':25},\n",
    "           #{'name':'cooperative', 'coop_factor':20},\n",
    "           #{'name':'cooperative', 'coop_factor':15},\n",
    "           #{'name':'cooperative', 'coop_factor':12.5},\n",
    "           #{'name':'cooperative', 'coop_factor':10},\n",
    "           #{'name':'cooperative', 'coop_factor':7.5}, \n",
    "           #{'name':'cooperative', 'coop_factor':5.0},\n",
    "           #{'name':'cooperative', 'coop_factor':1.0},\n",
    "           #{'name':'cooperative', 'coop_factor':0.1},\n",
    "           #{'name':'cooperative', 'coop_factor':0.01}\n",
    "          ]\n",
    "\n",
    "for culture in cultures:   # Go down the list of cultures\n",
    "\n",
    "    # Data structure for agents\n",
    "    agents = []\n",
    "    actions = []\n",
    "    log_probs = []\n",
    "    tags = []\n",
    "    rewards = []\n",
    "    optimizers = []\n",
    "\n",
    "    # Cold start\n",
    "    if warm_start is False:\n",
    "   \n",
    "        # Initialize learner agents, then load static agents (trained followed by random)\n",
    "        for i in range(num_learners):\n",
    "            print(\"Learner agent {}\".format(i))\n",
    "            agents.append(Policy(num_frames, num_actions, i)) # No weights loaded for learning agent\n",
    "            optimizers.append(optim.Adam(agents[i].parameters(), lr=lr))\n",
    "        \n",
    "            # set up optimizer - this works for Atari Breakout\n",
    "            # optimizers.append(optim.RMSprop(agents[i].parameters(), lr=lr, weight_decay=0.1)) \n",
    "        \n",
    "        for i in range(num_learners, num_learners+num_trained):\n",
    "            print (\"No trained agent exist yet!\")\n",
    "            raise\n",
    "            \"\"\"\n",
    "            Disable for now! No trained model exist!!!\n",
    "            agents.append(Policy(num_frames, num_actions, i))\n",
    "            agents[i].load_weights()         # load weight for static agent        \n",
    "            \"\"\"\n",
    "        for i in range(num_learners+num_trained, num_agents):\n",
    "            print(\"Load random agent {}\".format(i))\n",
    "            agents.append(Rdn_Policy())\n",
    "\n",
    "    \n",
    "        # Initialize all agent data\n",
    "        actions = [0 for i in range(num_agents)]\n",
    "        log_probs = [0 for i in range(num_agents)]\n",
    "        tags = [0 for i in range(num_agents)]\n",
    "        rewards = [0 for i in range(num_agents)]\n",
    "\n",
    "        # Keep track of rewards learned by learners\n",
    "        episode_reward = [0 for i in range(num_learners)]   # reward for an episode\n",
    "        running_reward = [None for i in range(num_learners)]   # running average\n",
    "        running_rewards = [[] for i in range(num_learners)]   # history of running averages\n",
    "        best_reward = [0 for i in range(num_learners)]    # best running average (for storing best_model)\n",
    "\n",
    "        # This is to support warm start for training\n",
    "        prior_eps = 0\n",
    "\n",
    "    # Warm start\n",
    "    if warm_start:\n",
    "        print (\"Cannot warm start\")\n",
    "        raise\n",
    "    \n",
    "        \"\"\"\n",
    "        # Disable for now!  Need to ensure model can support training on GPU and game playing\n",
    "        # on both CPU and GPU.\n",
    "    \n",
    "        data_file = 'results/{}.p'.format(game)\n",
    "\n",
    "        try:\n",
    "            with open(data_file, 'rb') as f:\n",
    "                running_rewards = pickle.load(f)\n",
    "                running_reward = running_rewards[-1]\n",
    "\n",
    "            prior_eps = len(running_rewards)\n",
    "\n",
    "            model_file = 'saved_models/actor_critic_{}_ep_{}.p'.format(game, prior_eps)\n",
    "            with open(model_file, 'rb') as f:\n",
    "                # Model Save and Load Update: Include both model and optim parameters\n",
    "                saved_model = pickle.load(f)\n",
    "                model, optimizer = saved_model\n",
    "\n",
    "        except OSError:\n",
    "            print('Saved file not found. Creating new cold start model.')\n",
    "            model = Policy(input_channels=num_frames, num_actions=num_actions)\n",
    "            optimizer = optim.RMSprop(model.parameters(), lr=lr,\n",
    "                                      weight_decay=0.1)\n",
    "            running_rewards = []\n",
    "            prior_eps = 0\n",
    "        \"\"\"\n",
    "\n",
    "    # Establish tribal association\n",
    "\n",
    "    tribes = []\n",
    "    tribes.append(Tribe(name='Vikings',color='blue', culture=culture, \\\n",
    "                    agents=[agents[0], agents[1], agents[2]]))\n",
    "    tribes.append(Tribe(name='Saxons', color='red', culture=culture, \\\n",
    "                    agents=[agents[3], agents[4], agents[5]]))\n",
    "    tribes.append(Tribe(name='Franks', color='purple', culture=culture, \\\n",
    "                    agents=[agents[6], agents[7], agents[8]]))\n",
    "    # tribes.append(Tribe(name='Crazies', color='yellow', agents=[agents[9]]))   # random agents are crazy!!!\n",
    "\n",
    "\n",
    "    # 9 agents in 4 tribes, used map defined in default.txt\n",
    "    agent_colors = [agent.color for agent in agents]\n",
    "    agent_tribes = [agent.tribe for agent in agents]\n",
    "    env = GatheringEnv(n_agents=num_agents,agent_colors=agent_colors, agent_tribes=agent_tribes, \\\n",
    "                       map_name='default')    \n",
    "   \n",
    "    \n",
    "    cuda = torch.cuda.is_available()\n",
    "\n",
    "    if cuda:\n",
    "        for i in range(num_learners):    # Learning agents need to utilize GPU\n",
    "            agents[i].cuda()\n",
    "\n",
    "        \n",
    "    for ep in range(max_episodes):\n",
    "    \n",
    "        print('.', end='')  # To show progress\n",
    "    \n",
    "        # Anneal temperature from temp_start to temp_end\n",
    "        for i in range(num_learners):    # For learning agents\n",
    "            agents[i].temperature = max(temp_end, temp_start - (temp_start - temp_end) * (ep / max_episodes))\n",
    "\n",
    "        env_obs = env.reset()  # Env return observations\n",
    "\n",
    "        # For Debug only\n",
    "        # print (len(env_obs))\n",
    "        # print (env_obs[0].shape)\n",
    "    \n",
    "        # Unpack observations into data structure compatible with agent Policy\n",
    "        agents_obs = unpack_env_obs(env_obs)\n",
    "\n",
    "        for i in range(num_learners):    # Reset agent info - laser tag statistics\n",
    "            agents[i].reset_info()   \n",
    "\n",
    "        # For Debug only\n",
    "        # print (len(agents_obs))\n",
    "        # print (agents_obs[0].shape)\n",
    "    \n",
    "        \"\"\"\n",
    "        For now, we do not stack observations, and we do not implement LSTM\n",
    "    \n",
    "        state = np.stack([state]*num_frames)\n",
    "\n",
    "        # LSTM change - reset LSTM hidden units when episode begins\n",
    "        cx = Variable(torch.zeros(1, 256))\n",
    "        hx = Variable(torch.zeros(1, 256))\n",
    "        if cuda:\n",
    "            cx = cx.cuda()\n",
    "            hx = hx.cuda()\n",
    "        \"\"\"\n",
    "\n",
    "        episode_reward = [0 for i in range(num_learners)]   # reward for an episode\n",
    "    \n",
    "        for frame in range(max_frames):\n",
    "\n",
    "            \"\"\"\n",
    "            For now, we do not implement LSTM\n",
    "            # Select action\n",
    "            # LSTM Change: Need to cycle hx and cx thru select_action\n",
    "            action, log_prob, value, (hx,cx)  = select_action(model, state, (hx,cx), cuda)        \n",
    "            \"\"\"\n",
    "\n",
    "            for i in range(num_learners):    # For learning agents\n",
    "                actions[i], log_probs[i] = select_learner_action(agents[i], agents_obs[i], cuda)\n",
    "                if actions[i] is 6:\n",
    "                    tags[i] += 1   # record a tag for accessing aggressiveness\n",
    "                agents[i].saved_actions.append((log_probs[i]))\n",
    "            \n",
    "                # Do not implement LSTM for now\n",
    "                # actions[i].saved_actions.append((log_prob, value))\n",
    "            \n",
    "            for i in range(num_learners, num_learners+num_trained):\n",
    "                print (\"No trained agent exist yet!\")\n",
    "                raise\n",
    "            for i in range(num_learners+num_trained, num_agents):   # For random agents\n",
    "                actions[i] = agents[i].select_action(agents_obs[i])\n",
    "                if actions[i] is 6:\n",
    "                    tags[i] += 1   # record a tag for accessing aggressiveness\n",
    "\n",
    "            # For Debug only\n",
    "            # if frame % 20 == 0:\n",
    "            #    print (actions) \n",
    "            #    print (log_probs)\n",
    "            \n",
    "            # Perform step        \n",
    "            env_obs, reward, done, info = env.step(actions)\n",
    "        \n",
    "            \"\"\"\n",
    "            For Debug only\n",
    "            print (env_obs)\n",
    "            print (reward)\n",
    "            print (done) \n",
    "            \"\"\"\n",
    "       \n",
    "            # Unpack observations into data structure compatible with agent Policy\n",
    "            agents_obs = unpack_env_obs(env_obs)\n",
    "            load_info(agents, narrate=False)   # Load agent info for AI agents\n",
    "\n",
    "            # For learner agents only, generate reward statistics and reward stack for policy gradient\n",
    "            for i in range(num_learners):\n",
    "                agents[i].rewards.append(reward[i])  # Stack rewards (for policy gradient)\n",
    "                episode_reward[i] += reward[i]   # accumulate episode reward \n",
    "            \n",
    "            \"\"\"\n",
    "            For now, we do not stack observation, may come in handy later on\n",
    "        \n",
    "            # Evict oldest diff add new diff to state\n",
    "            next_state = np.stack([next_state]*num_frames)\n",
    "            next_state[1:, :, :] = state[:-1, :, :]\n",
    "            state = next_state\n",
    "            \"\"\"\n",
    "\n",
    "            if any(done):\n",
    "                print(\"Done after {} frames\".format(frame))\n",
    "                break\n",
    "            \n",
    "        if frame > max_frames_ep:\n",
    "            max_frames_ep = frame    # Keep track of highest frames/episode\n",
    "\n",
    "        # Update reward statistics for learners\n",
    "        for i in range(num_learners):\n",
    "            if running_reward[i] is None:\n",
    "                running_reward[i] = episode_reward[i]\n",
    "            running_reward[i] = running_reward[i] * 0.99 + episode_reward[i] * 0.01\n",
    "            running_rewards[i].append(running_reward[i])\n",
    "\n",
    "        # Track Episode #, temp and highest frames/episode\n",
    "        if (ep+prior_eps+1) % log_interval == 0: \n",
    "            verbose_str = '\\nEpisode {} complete'.format(ep+prior_eps+1)\n",
    "            # verbose_str += '\\tTemp = {:.4}'.format(model.temperature)\n",
    "            # verbose_str += '\\tMax frames = {}'.format(max_frames_ep+1)\n",
    "            print(verbose_str)\n",
    "    \n",
    "            # Display rewards and running rewards for learning agents\n",
    "            for i in range(num_learners):\n",
    "                verbose_str = 'Learner:{}'.format(i)\n",
    "                verbose_str += '\\tReward total:{}'.format(episode_reward[i])\n",
    "                verbose_str += '\\tRunning mean: {:.4}'.format(running_reward[i])\n",
    "                print(verbose_str)\n",
    "    \n",
    "        # Update model\n",
    "        total_norms = finish_episode(agents[0:num_learners], optimizers[0:num_learners], gamma, cuda)\n",
    "\n",
    "        if (ep+prior_eps+1) % log_interval == 0:\n",
    "            print('Max Norms = ',[\"%0.2f\" % i for i in total_norms])\n",
    "        \n",
    "        if (ep+prior_eps+1) % save_interval == 0: \n",
    "            for i in range(num_learners):\n",
    "                model_dir = 'MA_models/3T-9L/'\n",
    "                results_dir = 'results/3T-9L/'\n",
    "\n",
    "                model_file = model_dir+'{}/cf{}/MA{}_{}_ep{}.p'.format(culture['name'], culture['coop_factor'],\\\n",
    "                                 i, game, ep+prior_eps+1)\n",
    "                data_file = results_dir+'{}/cf{}/MA{}_{}.p'.format(culture['name'], culture['coop_factor'],\\\n",
    "                                                      i, game)\n",
    "                \n",
    "                os.makedirs(os.path.dirname(model_file), exist_ok=True)\n",
    "                os.makedirs(os.path.dirname(data_file), exist_ok=True)\n",
    "                \n",
    "                with open(model_file, 'wb') as f:\n",
    "                    # Model Save and Load Update: Include both model and optim parameters \n",
    "                    pickle.dump((agents[i].cpu(), optimizers[i]), f)\n",
    "\n",
    "                if cuda:\n",
    "                    agents[i] = agents[i].cuda()\n",
    "\n",
    "                with open(data_file, 'wb') as f:\n",
    "                    pickle.dump(running_rewards[i], f)    \n",
    "            \n",
    "    env.close()  # Close the environment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cooperative - Trained to 10000ep "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import sys\n",
    "import pickle\n",
    "from collections import deque\n",
    "\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import numpy as np\n",
    "\n",
    "# Initialize agents parameters\n",
    "#   10 agents - 9 learning agents, 0 trained agents, 1 random agents\n",
    "num_learners = 9\n",
    "num_trained = 0\n",
    "num_rdn = 1\n",
    "num_statics = num_trained + num_rdn\n",
    "num_agents = num_learners + num_statics  \n",
    "\n",
    "# Initialize environment\n",
    "game = \"Gather\"\n",
    "num_actions = 8                       # 8 actions in Gathering\n",
    "\n",
    "# Initialize training parameters\n",
    "warm_start = False\n",
    "num_frames = 4      # environ observation consists of a list of 4 stacked frames per agent\n",
    "max_episodes = 10000\n",
    "max_frames = 300\n",
    "max_frames_ep = 0   # track highest number of frames an episode can last\n",
    "\n",
    "# These trainer parameters works for Atari Breakout\n",
    "gamma = 0.99  \n",
    "lr = 1e-3\n",
    "temp_start = 1.8  # Temperature for explore/exploit\n",
    "temp_end = 1.0\n",
    "log_interval = 200\n",
    "save_interval = 500\n",
    "\n",
    "def unpack_env_obs(env_obs):\n",
    "    \"\"\"\n",
    "    Gathering is a partially-observable Markov Game. env_obs returned by GatheringEnv is a numpy \n",
    "    array of dimension (num_agent, 800), which represents the agents' observations of the game.\n",
    "\n",
    "    The 800 elements (view_box) encodes 4 layers of 10x20 pixels frames in the format:\n",
    "    (viewbox_width, viewbox_depth, 4).\n",
    "    \n",
    "    This code reshapes the above into stacked frames that can be accepted by the Policy class:\n",
    "    (batch_idx, in_channel, width, height)\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    num_agents = len(env_obs)  # environ observations is a list of agents' observations\n",
    "    \n",
    "    obs = []\n",
    "    for i in range(num_agents):\n",
    "        x = env_obs[i]   # take the indexed agent's observation\n",
    "        x = torch.Tensor(x)   # Convert to tensor\n",
    "        \n",
    "        # Policy is a 3-layer CNN\n",
    "        x = x.view(1, 10, 20, -1)  # reshape into environment defined stacked frames\n",
    "        x = x.permute(0, 3, 1, 2)  # permute to Policy accepted stacked frames\n",
    "        obs.append(x)\n",
    "        \n",
    "    return obs  # return a list of Tensors\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "For now, we do not implement LSTM            \n",
    "# LSTM Change: Need to cycle hx and cx thru function\n",
    "def select_action(model, state, lstm_hc, cuda):\n",
    "    hx , cx = lstm_hc \n",
    "    num_frames, height, width = state.shape\n",
    "    state = torch.FloatTensor(state.reshape(-1, num_frames, height, width))\n",
    "\n",
    "    if cuda:\n",
    "        state = state.cuda()\n",
    "\n",
    "    probs, value, (hx, cx) = model((Variable(state), (hx, cx)))\n",
    "\n",
    "    m = torch.distributions.Categorical(probs)\n",
    "    action = m.sample()\n",
    "    log_prob = m.log_prob(action)\n",
    "    # LSTM Change: Need to cycle hx and cx thru function\n",
    "    return action.data[0], log_prob, value, (hx, cx)\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "def select_learner_action(model, obs, cuda):\n",
    "    \"\"\"\n",
    "    This code expects obs to be an array of stacked frames of the following dim:\n",
    "    (batch_idx, in_channel, width, height)\n",
    "    \n",
    "    This is inputted into model - the agent's Policy, which outputs a probability \n",
    "    distribution over available actions.\n",
    "    \n",
    "    Policy gradient is implemented using torch.distributions.Categorical. \n",
    "    \"\"\"\n",
    "    \n",
    "    # Policy is a 3-layer CNN\n",
    "    # _, num_frames, width, height = obs.shape\n",
    "    # obs = torch.FloatTensor(obs.reshape(-1, num_frames, width, height))\n",
    "    \n",
    "    # Policy is a 2-layer NN for now\n",
    "    # obs = obs.view(1, -1)\n",
    "   \n",
    "    if cuda:\n",
    "        obs = obs.cuda()\n",
    "      \n",
    "    probs = model(obs)\n",
    "    m = torch.distributions.Categorical(probs)\n",
    "    action = m.sample()\n",
    "    log_prob = m.log_prob(action)\n",
    "\n",
    "    return action.item(), log_prob \n",
    "\n",
    "\n",
    "def load_info(agents, narrate=False):\n",
    "    for i in range(num_agents):    \n",
    "        agents[i].load_info(info[i])\n",
    "        if narrate:\n",
    "            if agents[i].tagged:\n",
    "                print('frame {}, agent{} is tagged'.format(frame,i))\n",
    "            if agents[i].laser_fired:\n",
    "                print('frame {}, agent{} fires its laser'.format(frame,i))\n",
    "                print('and hit {} US and {} THEM'.format(agents[i].US_hit, agents[i].THEM_hit))\n",
    "    return\n",
    "\n",
    "\n",
    "# The main code starts here!!!\n",
    "\n",
    "# Cultural parameter search\n",
    "cultures =[\n",
    "           #{'name':'cooperative', 'coop_factor':25},\n",
    "           #{'name':'cooperative', 'coop_factor':30},\n",
    "           {'name':'cooperative', 'coop_factor':15},\n",
    "           {'name':'cooperative', 'coop_factor':20}\n",
    "          ]\n",
    "\n",
    "for culture in cultures:   # Go down the list of cultures\n",
    "\n",
    "    # Data structure for agents\n",
    "    agents = []\n",
    "    actions = []\n",
    "    log_probs = []\n",
    "    tags = []\n",
    "    rewards = []\n",
    "    optimizers = []\n",
    "\n",
    "    # Cold start\n",
    "    if warm_start is False:\n",
    "   \n",
    "        # Initialize learner agents, then load static agents (trained followed by random)\n",
    "        for i in range(num_learners):\n",
    "            print(\"Learner agent {}\".format(i))\n",
    "            agents.append(Policy(num_frames, num_actions, i)) # No weights loaded for learning agent\n",
    "            optimizers.append(optim.Adam(agents[i].parameters(), lr=lr))\n",
    "        \n",
    "            # set up optimizer - this works for Atari Breakout\n",
    "            # optimizers.append(optim.RMSprop(agents[i].parameters(), lr=lr, weight_decay=0.1)) \n",
    "        \n",
    "        for i in range(num_learners, num_learners+num_trained):\n",
    "            print (\"No trained agent exist yet!\")\n",
    "            raise\n",
    "            \"\"\"\n",
    "            Disable for now! No trained model exist!!!\n",
    "            agents.append(Policy(num_frames, num_actions, i))\n",
    "            agents[i].load_weights()         # load weight for static agent        \n",
    "            \"\"\"\n",
    "        for i in range(num_learners+num_trained, num_agents):\n",
    "            print(\"Load random agent {}\".format(i))\n",
    "            agents.append(Rdn_Policy())\n",
    "\n",
    "    \n",
    "        # Initialize all agent data\n",
    "        actions = [0 for i in range(num_agents)]\n",
    "        log_probs = [0 for i in range(num_agents)]\n",
    "        tags = [0 for i in range(num_agents)]\n",
    "        rewards = [0 for i in range(num_agents)]\n",
    "\n",
    "        # Keep track of rewards learned by learners\n",
    "        episode_reward = [0 for i in range(num_learners)]   # reward for an episode\n",
    "        running_reward = [None for i in range(num_learners)]   # running average\n",
    "        running_rewards = [[] for i in range(num_learners)]   # history of running averages\n",
    "        best_reward = [0 for i in range(num_learners)]    # best running average (for storing best_model)\n",
    "\n",
    "        # This is to support warm start for training\n",
    "        prior_eps = 0\n",
    "\n",
    "    # Warm start\n",
    "    if warm_start:\n",
    "        print (\"Cannot warm start\")\n",
    "        raise\n",
    "    \n",
    "        \"\"\"\n",
    "        # Disable for now!  Need to ensure model can support training on GPU and game playing\n",
    "        # on both CPU and GPU.\n",
    "    \n",
    "        data_file = 'results/{}.p'.format(game)\n",
    "\n",
    "        try:\n",
    "            with open(data_file, 'rb') as f:\n",
    "                running_rewards = pickle.load(f)\n",
    "                running_reward = running_rewards[-1]\n",
    "\n",
    "            prior_eps = len(running_rewards)\n",
    "\n",
    "            model_file = 'saved_models/actor_critic_{}_ep_{}.p'.format(game, prior_eps)\n",
    "            with open(model_file, 'rb') as f:\n",
    "                # Model Save and Load Update: Include both model and optim parameters\n",
    "                saved_model = pickle.load(f)\n",
    "                model, optimizer = saved_model\n",
    "\n",
    "        except OSError:\n",
    "            print('Saved file not found. Creating new cold start model.')\n",
    "            model = Policy(input_channels=num_frames, num_actions=num_actions)\n",
    "            optimizer = optim.RMSprop(model.parameters(), lr=lr,\n",
    "                                      weight_decay=0.1)\n",
    "            running_rewards = []\n",
    "            prior_eps = 0\n",
    "        \"\"\"\n",
    "\n",
    "    # Establish tribal association\n",
    "\n",
    "    tribes = []\n",
    "    tribes.append(Tribe(name='Vikings',color='blue', culture=culture, \\\n",
    "                    agents=[agents[0], agents[1], agents[2]]))\n",
    "    tribes.append(Tribe(name='Saxons', color='red', culture=culture, \\\n",
    "                    agents=[agents[3], agents[4], agents[5]]))\n",
    "    tribes.append(Tribe(name='Franks', color='purple', culture=culture, \\\n",
    "                    agents=[agents[6], agents[7], agents[8]]))\n",
    "    tribes.append(Tribe(name='Crazies', color='yellow', agents=[agents[9]]))   # random agents are crazy!!!\n",
    "\n",
    "\n",
    "    # 9 agents in 4 tribes, used map defined in default.txt\n",
    "    agent_colors = [agent.color for agent in agents]\n",
    "    agent_tribes = [agent.tribe for agent in agents]\n",
    "    env = GatheringEnv(n_agents=num_agents,agent_colors=agent_colors, agent_tribes=agent_tribes, \\\n",
    "                       map_name='default')    \n",
    "   \n",
    "    \n",
    "    cuda = torch.cuda.is_available()\n",
    "\n",
    "    if cuda:\n",
    "        for i in range(num_learners):    # Learning agents need to utilize GPU\n",
    "            agents[i].cuda()\n",
    "\n",
    "        \n",
    "    for ep in range(max_episodes):\n",
    "    \n",
    "        print('.', end='')  # To show progress\n",
    "    \n",
    "        # Anneal temperature from temp_start to temp_end\n",
    "        for i in range(num_learners):    # For learning agents\n",
    "            agents[i].temperature = max(temp_end, temp_start - (temp_start - temp_end) * (ep / max_episodes))\n",
    "\n",
    "        env_obs = env.reset()  # Env return observations\n",
    "\n",
    "        # For Debug only\n",
    "        # print (len(env_obs))\n",
    "        # print (env_obs[0].shape)\n",
    "    \n",
    "        # Unpack observations into data structure compatible with agent Policy\n",
    "        agents_obs = unpack_env_obs(env_obs)\n",
    "\n",
    "        for i in range(num_learners):    # Reset agent info - laser tag statistics\n",
    "            agents[i].reset_info()   \n",
    "\n",
    "        # For Debug only\n",
    "        # print (len(agents_obs))\n",
    "        # print (agents_obs[0].shape)\n",
    "    \n",
    "        \"\"\"\n",
    "        For now, we do not stack observations, and we do not implement LSTM\n",
    "    \n",
    "        state = np.stack([state]*num_frames)\n",
    "\n",
    "        # LSTM change - reset LSTM hidden units when episode begins\n",
    "        cx = Variable(torch.zeros(1, 256))\n",
    "        hx = Variable(torch.zeros(1, 256))\n",
    "        if cuda:\n",
    "            cx = cx.cuda()\n",
    "            hx = hx.cuda()\n",
    "        \"\"\"\n",
    "\n",
    "        episode_reward = [0 for i in range(num_learners)]   # reward for an episode\n",
    "    \n",
    "        for frame in range(max_frames):\n",
    "\n",
    "            \"\"\"\n",
    "            For now, we do not implement LSTM\n",
    "            # Select action\n",
    "            # LSTM Change: Need to cycle hx and cx thru select_action\n",
    "            action, log_prob, value, (hx,cx)  = select_action(model, state, (hx,cx), cuda)        \n",
    "            \"\"\"\n",
    "\n",
    "            for i in range(num_learners):    # For learning agents\n",
    "                actions[i], log_probs[i] = select_learner_action(agents[i], agents_obs[i], cuda)\n",
    "                if actions[i] is 6:\n",
    "                    tags[i] += 1   # record a tag for accessing aggressiveness\n",
    "                agents[i].saved_actions.append((log_probs[i]))\n",
    "            \n",
    "                # Do not implement LSTM for now\n",
    "                # actions[i].saved_actions.append((log_prob, value))\n",
    "            \n",
    "            for i in range(num_learners, num_learners+num_trained):\n",
    "                print (\"No trained agent exist yet!\")\n",
    "                raise\n",
    "            for i in range(num_learners+num_trained, num_agents):   # For random agents\n",
    "                actions[i] = agents[i].select_action(agents_obs[i])\n",
    "                if actions[i] is 6:\n",
    "                    tags[i] += 1   # record a tag for accessing aggressiveness\n",
    "\n",
    "            # For Debug only\n",
    "            # if frame % 20 == 0:\n",
    "            #    print (actions) \n",
    "            #    print (log_probs)\n",
    "            \n",
    "            # Perform step        \n",
    "            env_obs, reward, done, info = env.step(actions)\n",
    "        \n",
    "            \"\"\"\n",
    "            For Debug only\n",
    "            print (env_obs)\n",
    "            print (reward)\n",
    "            print (done) \n",
    "            \"\"\"\n",
    "       \n",
    "            # Unpack observations into data structure compatible with agent Policy\n",
    "            agents_obs = unpack_env_obs(env_obs)\n",
    "            load_info(agents, narrate=False)   # Load agent info for AI agents\n",
    "\n",
    "            # For learner agents only, generate reward statistics and reward stack for policy gradient\n",
    "            for i in range(num_learners):\n",
    "                agents[i].rewards.append(reward[i])  # Stack rewards (for policy gradient)\n",
    "                episode_reward[i] += reward[i]   # accumulate episode reward \n",
    "            \n",
    "            \"\"\"\n",
    "            For now, we do not stack observation, may come in handy later on\n",
    "        \n",
    "            # Evict oldest diff add new diff to state\n",
    "            next_state = np.stack([next_state]*num_frames)\n",
    "            next_state[1:, :, :] = state[:-1, :, :]\n",
    "            state = next_state\n",
    "            \"\"\"\n",
    "\n",
    "            if any(done):\n",
    "                print(\"Done after {} frames\".format(frame))\n",
    "                break\n",
    "            \n",
    "        if frame > max_frames_ep:\n",
    "            max_frames_ep = frame    # Keep track of highest frames/episode\n",
    "\n",
    "        # Update reward statistics for learners\n",
    "        for i in range(num_learners):\n",
    "            if running_reward[i] is None:\n",
    "                running_reward[i] = episode_reward[i]\n",
    "            running_reward[i] = running_reward[i] * 0.99 + episode_reward[i] * 0.01\n",
    "            running_rewards[i].append(running_reward[i])\n",
    "\n",
    "        # Track Episode #, temp and highest frames/episode\n",
    "        if (ep+prior_eps+1) % log_interval == 0: \n",
    "            verbose_str = '\\nEpisode {} complete'.format(ep+prior_eps+1)\n",
    "            # verbose_str += '\\tTemp = {:.4}'.format(model.temperature)\n",
    "            # verbose_str += '\\tMax frames = {}'.format(max_frames_ep+1)\n",
    "            print(verbose_str)\n",
    "    \n",
    "            # Display rewards and running rewards for learning agents\n",
    "            for i in range(num_learners):\n",
    "                verbose_str = 'Learner:{}'.format(i)\n",
    "                verbose_str += '\\tReward total:{}'.format(episode_reward[i])\n",
    "                verbose_str += '\\tRunning mean: {:.4}'.format(running_reward[i])\n",
    "                print(verbose_str)\n",
    "    \n",
    "        # Update model\n",
    "        total_norms = finish_episode(agents[0:num_learners], optimizers[0:num_learners], gamma, cuda)\n",
    "\n",
    "        if (ep+prior_eps+1) % log_interval == 0:\n",
    "            print('Max Norms = ',[\"%0.2f\" % i for i in total_norms])\n",
    "        \n",
    "        if (ep+prior_eps+1) % save_interval == 0: \n",
    "            for i in range(num_learners):\n",
    "                model_dir = 'MA_models/3T-9L1R/'\n",
    "                results_dir = 'results/3T-9L1R/'\n",
    "\n",
    "                model_file = model_dir+'{}-10000ep/cf{}/MA{}_{}__ep{}.p'.format(culture['name'], culture['coop_factor'],\\\n",
    "                                 i, game, ep+prior_eps+1)\n",
    "                data_file = results_dir+'{}-10000ep/cf{}/MA{}_{}.p'.format(culture['name'], culture['coop_factor'],\\\n",
    "                                                      i, game)\n",
    "                \n",
    "                os.makedirs(os.path.dirname(model_file), exist_ok=True)\n",
    "                os.makedirs(os.path.dirname(data_file), exist_ok=True)\n",
    "                \n",
    "                with open(model_file, 'wb') as f:\n",
    "                    # Model Save and Load Update: Include both model and optim parameters \n",
    "                    pickle.dump((agents[i].cpu(), optimizers[i]), f)\n",
    "\n",
    "                if cuda:\n",
    "                    agents[i] = agents[i].cuda()\n",
    "\n",
    "                with open(data_file, 'wb') as f:\n",
    "                    pickle.dump(running_rewards[i], f)    \n",
    "            \n",
    "    env.close()  # Close the environment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Culture = No Fragging\n",
    "\n",
    "Run finish_episode() specific to this culture."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "def finish_episode(learners, optimizers, gamma, cuda):\n",
    "    \"\"\" \n",
    "    Note that in RL, policy gradient is calculated at the end of an episode and only then used to \n",
    "    update the weights of an agent's policy. This is very different compared to image recog.\n",
    "    \n",
    "    The code will perform policy update on each learning agent independently. Reward for each time \n",
    "    step is stored in the list policy.rewards[] --> r(t)\n",
    "    \"\"\"  \n",
    "    \n",
    "    num_learners = len(learners)\n",
    "    total_norms = [0 for i in range(num_learners)]\n",
    "    policy_losses = [[] for i in range(num_learners)]\n",
    "    losses = [[] for i in range(num_learners)]\n",
    "    T_reward = []\n",
    "\n",
    "   \n",
    "    for i in range(num_learners):\n",
    "\n",
    "        R = 0\n",
    "        saved_actions = learners[i].saved_actions\n",
    "        \n",
    "        for t in tribes:\n",
    "            if t.name is learners[i].tribe:\n",
    "                T_reward = t.tribal_awards(US_hits = learners[i].US_hits)\n",
    "       \n",
    "        # Do not implement actor-critic for now\n",
    "        # value_losses = []\n",
    "        \n",
    "        rewards = deque()\n",
    "\n",
    "        for r,T in zip(learners[i].rewards[::-1],T_reward[::-1]):\n",
    "            # The agent is incentivized to cooperate by an award of 30% of what the tribe takes\n",
    "            # in by all its members\n",
    "            R = r + T + gamma * R\n",
    "            rewards.appendleft(R)\n",
    "            \n",
    "        rewards = list(rewards)\n",
    "        rewards = torch.Tensor(rewards)\n",
    "        if cuda:\n",
    "            rewards = rewards.cuda()\n",
    "\n",
    "        # z-score rewards\n",
    "        rewards = (rewards - rewards.mean()) / (1.1e-7+rewards.std())\n",
    "        \n",
    "        #Debug     \n",
    "        #print (rewards)       \n",
    "        \n",
    "        \"\"\"\n",
    "        Do not implement actor-critic for now!!!\n",
    "        for (log_prob, state_value), r in zip(saved_actions, rewards):\n",
    "            reward = r - state_value.data[0]\n",
    "            policy_losses.append(-log_prob * Variable(reward))\n",
    "            r = torch.Tensor([r])\n",
    "            if cuda:\n",
    "                r = r.cuda()\n",
    "            value_losses.append(torch.nn.functional.smooth_l1_loss(state_value,\n",
    "                                                               Variable(r)))\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss = torch.stack(policy_losses).sum() + torch.stack(value_losses).sum()\n",
    "        loss.backward()        \n",
    "        \n",
    "        \n",
    "        \"\"\"\n",
    "        for log_prob, r in zip(saved_actions, rewards):\n",
    "            r = torch.Tensor([r])\n",
    "            if cuda:\n",
    "                r = r.cuda()\n",
    "            policy_losses[i].append(-log_prob * Variable(r))\n",
    "\n",
    "        optimizers[i].zero_grad()\n",
    "        losses[i] = torch.stack(policy_losses[i]).sum()\n",
    "        losses[i].backward()\n",
    "        \n",
    "        # Gradient Clipping Update: prevent exploding gradient\n",
    "        total_norms[i] = torch.nn.utils.clip_grad_norm_(learners[i].parameters(), 8000)\n",
    "        \n",
    "        optimizers[i].step()\n",
    "        learners[i].clear_history()   # clear an agent's history at the end of episode\n",
    "\n",
    "\n",
    "    return total_norms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Load random agent 8\n",
      "....................................................................................................\n",
      "Episode 100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.01379\n",
      "Learner:4\tReward total:0\tRunning mean: 0.01083\n",
      "Learner:5\tReward total:0\tRunning mean: 0.05354\n",
      "Learner:6\tReward total:0\tRunning mean: 0.1491\n",
      "Learner:7\tReward total:0\tRunning mean: 0.2374\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '0.00', '230.45', '78.84', '0.00']\n",
      "....................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.2032\n",
      "Learner:4\tReward total:10\tRunning mean: 1.478\n",
      "Learner:5\tReward total:0\tRunning mean: 0.3615\n",
      "Learner:6\tReward total:0\tRunning mean: 0.669\n",
      "Learner:7\tReward total:20\tRunning mean: 3.246\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '0.00', '94.69', '0.00', '0.00', '99.08']\n",
      "....................................................................................................\n",
      "Episode 300 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:3\tRunning mean: 0.8071\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.5546\n",
      "Learner:4\tReward total:10\tRunning mean: 6.57\n",
      "Learner:5\tReward total:0\tRunning mean: 2.079\n",
      "Learner:6\tReward total:9\tRunning mean: 2.212\n",
      "Learner:7\tReward total:30\tRunning mean: 13.4\n",
      "Max Norms =  ['0.00', '93.56', '0.00', '0.00', '120.15', '0.00', '86.35', '136.02']\n",
      "....................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:4\tRunning mean: 2.196\n",
      "Learner:2\tReward total:0\tRunning mean: 0.06302\n",
      "Learner:3\tReward total:7\tRunning mean: 2.503\n",
      "Learner:4\tReward total:23\tRunning mean: 15.98\n",
      "Learner:5\tReward total:23\tRunning mean: 6.922\n",
      "Learner:6\tReward total:0\tRunning mean: 6.819\n",
      "Learner:7\tReward total:35\tRunning mean: 23.78\n",
      "Max Norms =  ['0.00', '103.67', '0.00', '127.33', '114.90', '138.60', '0.00', '123.45']\n",
      "....................................................................................................\n",
      "Episode 500 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:12\tRunning mean: 7.499\n",
      "Learner:2\tReward total:0\tRunning mean: 0.2254\n",
      "Learner:3\tReward total:1\tRunning mean: 5.855\n",
      "Learner:4\tReward total:10\tRunning mean: 21.79\n",
      "Learner:5\tReward total:15\tRunning mean: 13.95\n",
      "Learner:6\tReward total:19\tRunning mean: 10.54\n",
      "Learner:7\tReward total:41\tRunning mean: 29.74\n",
      "Max Norms =  ['0.00', '149.49', '0.00', '96.94', '86.25', '66.42', '115.84', '173.32']\n",
      "....................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:20\tRunning mean: 9.518\n",
      "Learner:2\tReward total:0\tRunning mean: 0.9015\n",
      "Learner:3\tReward total:24\tRunning mean: 12.57\n",
      "Learner:4\tReward total:16\tRunning mean: 26.57\n",
      "Learner:5\tReward total:17\tRunning mean: 18.85\n",
      "Learner:6\tReward total:25\tRunning mean: 14.53\n",
      "Learner:7\tReward total:37\tRunning mean: 29.33\n",
      "Max Norms =  ['0.00', '30.66', '0.00', '103.21', '105.62', '65.39', '130.71', '100.85']\n",
      "....................................................................................................\n",
      "Episode 700 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:11\tRunning mean: 12.22\n",
      "Learner:2\tReward total:7\tRunning mean: 1.496\n",
      "Learner:3\tReward total:22\tRunning mean: 16.14\n",
      "Learner:4\tReward total:33\tRunning mean: 27.27\n",
      "Learner:5\tReward total:25\tRunning mean: 20.03\n",
      "Learner:6\tReward total:11\tRunning mean: 14.14\n",
      "Learner:7\tReward total:29\tRunning mean: 32.26\n",
      "Max Norms =  ['0.00', '57.79', '90.75', '133.52', '91.18', '60.88', '107.97', '151.07']\n",
      "....................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:8\tRunning mean: 12.01\n",
      "Learner:2\tReward total:11\tRunning mean: 3.286\n",
      "Learner:3\tReward total:36\tRunning mean: 18.2\n",
      "Learner:4\tReward total:24\tRunning mean: 37.64\n",
      "Learner:5\tReward total:13\tRunning mean: 16.87\n",
      "Learner:6\tReward total:6\tRunning mean: 11.77\n",
      "Learner:7\tReward total:19\tRunning mean: 27.43\n",
      "Max Norms =  ['0.00', '57.45', '89.82', '79.21', '57.11', '52.44', '107.45', '128.15']\n",
      "....................................................................................................\n",
      "Episode 900 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:15\tRunning mean: 12.56\n",
      "Learner:2\tReward total:3\tRunning mean: 6.277\n",
      "Learner:3\tReward total:25\tRunning mean: 20.38\n",
      "Learner:4\tReward total:31\tRunning mean: 40.37\n",
      "Learner:5\tReward total:8\tRunning mean: 14.31\n",
      "Learner:6\tReward total:11\tRunning mean: 11.76\n",
      "Learner:7\tReward total:33\tRunning mean: 21.73\n",
      "Max Norms =  ['0.00', '41.84', '98.54', '78.05', '55.18', '42.70', '135.50', '116.82']\n",
      "....................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:2\tRunning mean: 10.16\n",
      "Learner:2\tReward total:1\tRunning mean: 8.082\n",
      "Learner:3\tReward total:16\tRunning mean: 19.24\n",
      "Learner:4\tReward total:40\tRunning mean: 42.5\n",
      "Learner:5\tReward total:21\tRunning mean: 14.23\n",
      "Learner:6\tReward total:2\tRunning mean: 12.23\n",
      "Learner:7\tReward total:18\tRunning mean: 18.71\n",
      "Max Norms =  ['0.00', '32.74', '85.06', '83.20', '107.08', '22.82', '62.31', '90.91']\n",
      "....................................................................................................\n",
      "Episode 1100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:4\tRunning mean: 7.933\n",
      "Learner:2\tReward total:6\tRunning mean: 8.432\n",
      "Learner:3\tReward total:29\tRunning mean: 19.18\n",
      "Learner:4\tReward total:69\tRunning mean: 43.19\n",
      "Learner:5\tReward total:5\tRunning mean: 11.2\n",
      "Learner:6\tReward total:9\tRunning mean: 10.78\n",
      "Learner:7\tReward total:5\tRunning mean: 16.1\n",
      "Max Norms =  ['0.00', '50.87', '115.17', '80.79', '119.50', '29.18', '111.56', '84.64']\n",
      "....................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:2\tRunning mean: 7.631\n",
      "Learner:2\tReward total:2\tRunning mean: 8.746\n",
      "Learner:3\tReward total:4\tRunning mean: 24.4\n",
      "Learner:4\tReward total:61\tRunning mean: 40.04\n",
      "Learner:5\tReward total:12\tRunning mean: 10.72\n",
      "Learner:6\tReward total:20\tRunning mean: 10.49\n",
      "Learner:7\tReward total:20\tRunning mean: 16.94\n",
      "Max Norms =  ['0.00', '56.34', '62.75', '74.99', '88.85', '27.16', '71.73', '65.79']\n",
      "....................................................................................................\n",
      "Episode 1300 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:6\tRunning mean: 7.016\n",
      "Learner:2\tReward total:5\tRunning mean: 8.419\n",
      "Learner:3\tReward total:14\tRunning mean: 24.04\n",
      "Learner:4\tReward total:45\tRunning mean: 43.11\n",
      "Learner:5\tReward total:5\tRunning mean: 9.441\n",
      "Learner:6\tReward total:13\tRunning mean: 9.154\n",
      "Learner:7\tReward total:11\tRunning mean: 14.77\n",
      "Max Norms =  ['0.00', '137.85', '86.43', '144.10', '82.96', '74.39', '167.53', '72.44']\n",
      "....................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:0\tRunning mean: 6.414\n",
      "Learner:2\tReward total:9\tRunning mean: 8.723\n",
      "Learner:3\tReward total:78\tRunning mean: 27.46\n",
      "Learner:4\tReward total:6\tRunning mean: 35.03\n",
      "Learner:5\tReward total:10\tRunning mean: 9.106\n",
      "Learner:6\tReward total:12\tRunning mean: 9.187\n",
      "Learner:7\tReward total:9\tRunning mean: 14.81\n",
      "Max Norms =  ['0.00', '0.00', '101.52', '161.07', '141.23', '55.22', '91.47', '87.68']\n",
      "....................................................................................................\n",
      "Episode 1500 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:9\tRunning mean: 7.365\n",
      "Learner:2\tReward total:11\tRunning mean: 10.62\n",
      "Learner:3\tReward total:16\tRunning mean: 31.04\n",
      "Learner:4\tReward total:47\tRunning mean: 33.33\n",
      "Learner:5\tReward total:12\tRunning mean: 9.688\n",
      "Learner:6\tReward total:20\tRunning mean: 11.19\n",
      "Learner:7\tReward total:7\tRunning mean: 16.82\n",
      "Max Norms =  ['0.00', '31.10', '72.53', '122.17', '80.37', '55.31', '113.74', '95.64']\n",
      "....................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:12\tRunning mean: 7.147\n",
      "Learner:2\tReward total:1\tRunning mean: 10.45\n",
      "Learner:3\tReward total:12\tRunning mean: 28.33\n",
      "Learner:4\tReward total:31\tRunning mean: 32.78\n",
      "Learner:5\tReward total:6\tRunning mean: 8.811\n",
      "Learner:6\tReward total:13\tRunning mean: 11.15\n",
      "Learner:7\tReward total:12\tRunning mean: 18.0\n",
      "Max Norms =  ['0.00', '41.40', '57.86', '103.71', '92.71', '16.52', '106.41', '125.49']\n",
      "....................................................................................................\n",
      "Episode 1700 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:6\tRunning mean: 6.346\n",
      "Learner:2\tReward total:10\tRunning mean: 9.606\n",
      "Learner:3\tReward total:31\tRunning mean: 31.94\n",
      "Learner:4\tReward total:21\tRunning mean: 33.22\n",
      "Learner:5\tReward total:3\tRunning mean: 8.188\n",
      "Learner:6\tReward total:12\tRunning mean: 11.06\n",
      "Learner:7\tReward total:31\tRunning mean: 17.31\n",
      "Max Norms =  ['0.00', '31.78', '97.50', '140.47', '78.08', '24.46', '115.75', '126.14']\n",
      "....................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:4\tRunning mean: 7.291\n",
      "Learner:2\tReward total:11\tRunning mean: 10.59\n",
      "Learner:3\tReward total:20\tRunning mean: 30.75\n",
      "Learner:4\tReward total:34\tRunning mean: 30.74\n",
      "Learner:5\tReward total:8\tRunning mean: 8.17\n",
      "Learner:6\tReward total:6\tRunning mean: 12.1\n",
      "Learner:7\tReward total:40\tRunning mean: 20.21\n",
      "Max Norms =  ['0.00', '10.91', '70.21', '103.92', '64.59', '8.64', '66.86', '104.56']\n",
      "....................................................................................................\n",
      "Episode 1900 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:7\tRunning mean: 7.063\n",
      "Learner:2\tReward total:12\tRunning mean: 9.137\n",
      "Learner:3\tReward total:34\tRunning mean: 29.66\n",
      "Learner:4\tReward total:4\tRunning mean: 25.48\n",
      "Learner:5\tReward total:13\tRunning mean: 7.007\n",
      "Learner:6\tReward total:4\tRunning mean: 11.06\n",
      "Learner:7\tReward total:37\tRunning mean: 24.25\n",
      "Max Norms =  ['0.00', '1.68', '82.56', '153.45', '37.19', '28.59', '126.46', '153.57']\n",
      "....................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:15\tRunning mean: 7.329\n",
      "Learner:2\tReward total:10\tRunning mean: 10.2\n",
      "Learner:3\tReward total:25\tRunning mean: 29.36\n",
      "Learner:4\tReward total:30\tRunning mean: 25.22\n",
      "Learner:5\tReward total:9\tRunning mean: 7.336\n",
      "Learner:6\tReward total:10\tRunning mean: 10.74\n",
      "Learner:7\tReward total:22\tRunning mean: 27.02\n",
      "Max Norms =  ['0.00', '7.61', '165.34', '151.69', '90.88', '12.04', '103.62', '63.17']\n",
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Load random agent 8\n",
      "....................................................................................................\n",
      "Episode 100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.008601\n",
      "Learner:4\tReward total:0\tRunning mean: 0.2236\n",
      "Learner:5\tReward total:0\tRunning mean: 0.3434\n",
      "Learner:6\tReward total:10\tRunning mean: 0.2887\n",
      "Learner:7\tReward total:0\tRunning mean: 0.5879\n",
      "Max Norms =  ['122.16', '99.10', '118.62', '0.00', '144.63', '87.78', '84.33', '0.00']\n",
      "....................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.06336\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.007323\n",
      "Learner:3\tReward total:0\tRunning mean: 0.2115\n",
      "Learner:4\tReward total:0\tRunning mean: 1.126\n",
      "Learner:5\tReward total:1\tRunning mean: 0.3785\n",
      "Learner:6\tReward total:0\tRunning mean: 0.75\n",
      "Learner:7\tReward total:9\tRunning mean: 2.457\n",
      "Max Norms =  ['0.00', '0.00', '145.91', '0.00', '0.00', '76.79', '121.12', '123.63']\n",
      "....................................................................................................\n",
      "Episode 300 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.4512\n",
      "Learner:1\tReward total:17\tRunning mean: 1.281\n",
      "Learner:2\tReward total:0\tRunning mean: 0.00268\n",
      "Learner:3\tReward total:4\tRunning mean: 1.675\n",
      "Learner:4\tReward total:40\tRunning mean: 8.366\n",
      "Learner:5\tReward total:0\tRunning mean: 0.8524\n",
      "Learner:6\tReward total:6\tRunning mean: 4.545\n",
      "Learner:7\tReward total:3\tRunning mean: 4.849\n",
      "Max Norms =  ['0.00', '131.00', '0.00', '89.78', '112.67', '0.00', '103.29', '114.03']\n",
      "....................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:2\tRunning mean: 1.212\n",
      "Learner:1\tReward total:7\tRunning mean: 8.636\n",
      "Learner:2\tReward total:0\tRunning mean: 0.04489\n",
      "Learner:3\tReward total:7\tRunning mean: 3.578\n",
      "Learner:4\tReward total:13\tRunning mean: 18.12\n",
      "Learner:5\tReward total:41\tRunning mean: 4.106\n",
      "Learner:6\tReward total:18\tRunning mean: 15.37\n",
      "Learner:7\tReward total:30\tRunning mean: 10.44\n",
      "Max Norms =  ['122.18', '85.18', '0.00', '103.15', '109.41', '120.47', '109.43', '127.41']\n",
      "....................................................................................................\n",
      "Episode 500 complete\n",
      "Learner:0\tReward total:14\tRunning mean: 4.317\n",
      "Learner:1\tReward total:31\tRunning mean: 16.27\n",
      "Learner:2\tReward total:0\tRunning mean: 0.06129\n",
      "Learner:3\tReward total:0\tRunning mean: 4.301\n",
      "Learner:4\tReward total:18\tRunning mean: 22.48\n",
      "Learner:5\tReward total:0\tRunning mean: 7.303\n",
      "Learner:6\tReward total:41\tRunning mean: 24.8\n",
      "Learner:7\tReward total:35\tRunning mean: 19.87\n",
      "Max Norms =  ['63.45', '128.14', '0.00', '0.00', '115.54', '0.00', '116.50', '161.24']\n",
      "....................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:7\tRunning mean: 6.99\n",
      "Learner:1\tReward total:35\tRunning mean: 21.28\n",
      "Learner:2\tReward total:0\tRunning mean: 0.02243\n",
      "Learner:3\tReward total:2\tRunning mean: 4.64\n",
      "Learner:4\tReward total:40\tRunning mean: 26.37\n",
      "Learner:5\tReward total:30\tRunning mean: 15.08\n",
      "Learner:6\tReward total:35\tRunning mean: 27.85\n",
      "Learner:7\tReward total:17\tRunning mean: 23.55\n",
      "Max Norms =  ['79.68', '104.90', '0.00', '70.00', '105.84', '96.31', '177.64', '125.92']\n",
      "....................................................................................................\n",
      "Episode 700 complete\n",
      "Learner:0\tReward total:4\tRunning mean: 7.918\n",
      "Learner:1\tReward total:29\tRunning mean: 21.37\n",
      "Learner:2\tReward total:0\tRunning mean: 0.01791\n",
      "Learner:3\tReward total:9\tRunning mean: 7.46\n",
      "Learner:4\tReward total:35\tRunning mean: 26.03\n",
      "Learner:5\tReward total:22\tRunning mean: 20.02\n",
      "Learner:6\tReward total:29\tRunning mean: 30.79\n",
      "Learner:7\tReward total:46\tRunning mean: 26.84\n",
      "Max Norms =  ['91.71', '72.89', '0.00', '115.97', '118.39', '129.33', '149.59', '126.87']\n",
      "....................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:7\tRunning mean: 8.033\n",
      "Learner:1\tReward total:17\tRunning mean: 20.65\n",
      "Learner:2\tReward total:0\tRunning mean: 0.2394\n",
      "Learner:3\tReward total:4\tRunning mean: 9.218\n",
      "Learner:4\tReward total:37\tRunning mean: 25.87\n",
      "Learner:5\tReward total:12\tRunning mean: 21.38\n",
      "Learner:6\tReward total:19\tRunning mean: 30.69\n",
      "Learner:7\tReward total:21\tRunning mean: 28.18\n",
      "Max Norms =  ['49.50', '52.89', '0.00', '71.78', '100.64', '86.19', '123.18', '87.46']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "....................................................................................................\n",
      "Episode 900 complete\n",
      "Learner:0\tReward total:14\tRunning mean: 8.763\n",
      "Learner:1\tReward total:16\tRunning mean: 16.65\n",
      "Learner:2\tReward total:4\tRunning mean: 0.8184\n",
      "Learner:3\tReward total:0\tRunning mean: 9.195\n",
      "Learner:4\tReward total:5\tRunning mean: 19.85\n",
      "Learner:5\tReward total:36\tRunning mean: 30.76\n",
      "Learner:6\tReward total:21\tRunning mean: 24.19\n",
      "Learner:7\tReward total:17\tRunning mean: 24.7\n",
      "Max Norms =  ['37.47', '59.82', '91.08', '0.00', '43.96', '100.75', '166.32', '130.34']\n",
      "....................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:3\tRunning mean: 7.295\n",
      "Learner:1\tReward total:7\tRunning mean: 14.03\n",
      "Learner:2\tReward total:5\tRunning mean: 3.623\n",
      "Learner:3\tReward total:0\tRunning mean: 6.953\n",
      "Learner:4\tReward total:12\tRunning mean: 16.15\n",
      "Learner:5\tReward total:3\tRunning mean: 25.08\n",
      "Learner:6\tReward total:5\tRunning mean: 19.67\n",
      "Learner:7\tReward total:68\tRunning mean: 32.74\n",
      "Max Norms =  ['68.78', '100.66', '111.19', '0.00', '47.18', '117.60', '86.04', '141.58']\n",
      "....................................................................................................\n",
      "Episode 1100 complete\n",
      "Learner:0\tReward total:13\tRunning mean: 9.069\n",
      "Learner:1\tReward total:2\tRunning mean: 14.26\n",
      "Learner:2\tReward total:6\tRunning mean: 6.102\n",
      "Learner:3\tReward total:1\tRunning mean: 7.458\n",
      "Learner:4\tReward total:5\tRunning mean: 12.55\n",
      "Learner:5\tReward total:32\tRunning mean: 19.85\n",
      "Learner:6\tReward total:14\tRunning mean: 17.11\n",
      "Learner:7\tReward total:57\tRunning mean: 35.75\n",
      "Max Norms =  ['32.88', '37.01', '62.07', '71.11', '80.46', '74.54', '84.34', '132.21']\n",
      "....................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:9\tRunning mean: 7.174\n",
      "Learner:1\tReward total:11\tRunning mean: 11.58\n",
      "Learner:2\tReward total:11\tRunning mean: 7.946\n",
      "Learner:3\tReward total:4\tRunning mean: 7.161\n",
      "Learner:4\tReward total:15\tRunning mean: 14.89\n",
      "Learner:5\tReward total:12\tRunning mean: 19.38\n",
      "Learner:6\tReward total:19\tRunning mean: 15.87\n",
      "Learner:7\tReward total:68\tRunning mean: 43.29\n",
      "Max Norms =  ['59.49', '101.47', '91.18', '127.17', '122.84', '77.73', '80.84', '210.08']\n",
      "....................................................................................................\n",
      "Episode 1300 complete\n",
      "Learner:0\tReward total:5\tRunning mean: 6.646\n",
      "Learner:1\tReward total:25\tRunning mean: 12.52\n",
      "Learner:2\tReward total:18\tRunning mean: 10.53\n",
      "Learner:3\tReward total:14\tRunning mean: 8.406\n",
      "Learner:4\tReward total:10\tRunning mean: 17.71\n",
      "Learner:5\tReward total:20\tRunning mean: 19.04\n",
      "Learner:6\tReward total:33\tRunning mean: 16.04\n",
      "Learner:7\tReward total:33\tRunning mean: 46.82\n",
      "Max Norms =  ['36.33', '120.77', '84.71', '84.31', '82.19', '79.89', '102.24', '243.38']\n",
      "....................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:6\tRunning mean: 5.86\n",
      "Learner:1\tReward total:4\tRunning mean: 10.97\n",
      "Learner:2\tReward total:7\tRunning mean: 11.61\n",
      "Learner:3\tReward total:0\tRunning mean: 8.153\n",
      "Learner:4\tReward total:6\tRunning mean: 15.4\n",
      "Learner:5\tReward total:25\tRunning mean: 22.8\n",
      "Learner:6\tReward total:28\tRunning mean: 15.93\n",
      "Learner:7\tReward total:43\tRunning mean: 46.14\n",
      "Max Norms =  ['74.51', '52.09', '137.20', '0.00', '109.35', '92.20', '135.49', '88.21']\n",
      "....................................................................................................\n",
      "Episode 1500 complete\n",
      "Learner:0\tReward total:3\tRunning mean: 4.549\n",
      "Learner:1\tReward total:5\tRunning mean: 10.32\n",
      "Learner:2\tReward total:11\tRunning mean: 11.19\n",
      "Learner:3\tReward total:16\tRunning mean: 7.701\n",
      "Learner:4\tReward total:7\tRunning mean: 16.03\n",
      "Learner:5\tReward total:48\tRunning mean: 29.17\n",
      "Learner:6\tReward total:11\tRunning mean: 14.93\n",
      "Learner:7\tReward total:37\tRunning mean: 41.63\n",
      "Max Norms =  ['56.80', '33.00', '98.79', '164.45', '27.57', '166.83', '140.68', '112.63']\n",
      "....................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:6\tRunning mean: 4.671\n",
      "Learner:1\tReward total:19\tRunning mean: 9.796\n",
      "Learner:2\tReward total:7\tRunning mean: 10.23\n",
      "Learner:3\tReward total:7\tRunning mean: 6.891\n",
      "Learner:4\tReward total:10\tRunning mean: 14.27\n",
      "Learner:5\tReward total:27\tRunning mean: 26.82\n",
      "Learner:6\tReward total:6\tRunning mean: 12.79\n",
      "Learner:7\tReward total:56\tRunning mean: 40.51\n",
      "Max Norms =  ['80.57', '22.14', '107.66', '46.13', '39.21', '132.71', '57.17', '140.06']\n",
      "....................................................................................................\n",
      "Episode 1700 complete\n",
      "Learner:0\tReward total:4\tRunning mean: 4.633\n",
      "Learner:1\tReward total:8\tRunning mean: 9.434\n",
      "Learner:2\tReward total:12\tRunning mean: 9.522\n",
      "Learner:3\tReward total:9\tRunning mean: 5.918\n",
      "Learner:4\tReward total:15\tRunning mean: 12.61\n",
      "Learner:5\tReward total:30\tRunning mean: 28.96\n",
      "Learner:6\tReward total:20\tRunning mean: 10.38\n",
      "Learner:7\tReward total:32\tRunning mean: 36.78\n",
      "Max Norms =  ['35.27', '28.21', '80.49', '126.58', '28.64', '99.05', '146.33', '136.81']\n",
      "....................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:8\tRunning mean: 5.088\n",
      "Learner:1\tReward total:8\tRunning mean: 8.85\n",
      "Learner:2\tReward total:12\tRunning mean: 9.631\n",
      "Learner:3\tReward total:4\tRunning mean: 5.888\n",
      "Learner:4\tReward total:6\tRunning mean: 13.15\n",
      "Learner:5\tReward total:49\tRunning mean: 29.85\n",
      "Learner:6\tReward total:6\tRunning mean: 10.52\n",
      "Learner:7\tReward total:28\tRunning mean: 36.65\n",
      "Max Norms =  ['10.04', '6.54', '97.54', '67.13', '41.68', '155.84', '137.63', '79.31']\n",
      "....................................................................................................\n",
      "Episode 1900 complete\n",
      "Learner:0\tReward total:2\tRunning mean: 5.613\n",
      "Learner:1\tReward total:5\tRunning mean: 10.63\n",
      "Learner:2\tReward total:16\tRunning mean: 10.19\n",
      "Learner:3\tReward total:13\tRunning mean: 8.994\n",
      "Learner:4\tReward total:11\tRunning mean: 13.0\n",
      "Learner:5\tReward total:28\tRunning mean: 31.02\n",
      "Learner:6\tReward total:10\tRunning mean: 12.43\n",
      "Learner:7\tReward total:14\tRunning mean: 34.8\n",
      "Max Norms =  ['2.37', '47.87', '112.14', '35.53', '59.84', '171.07', '186.00', '64.71']\n",
      "....................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:9\tRunning mean: 5.934\n",
      "Learner:1\tReward total:8\tRunning mean: 10.78\n",
      "Learner:2\tReward total:21\tRunning mean: 9.745\n",
      "Learner:3\tReward total:11\tRunning mean: 7.858\n",
      "Learner:4\tReward total:2\tRunning mean: 11.58\n",
      "Learner:5\tReward total:27\tRunning mean: 35.96\n",
      "Learner:6\tReward total:14\tRunning mean: 11.77\n",
      "Learner:7\tReward total:31\tRunning mean: 35.55\n",
      "Max Norms =  ['11.92', '47.55', '118.61', '44.69', '76.95', '116.61', '62.95', '132.88']\n",
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Load random agent 8\n",
      "....................................................................................................\n",
      "Episode 100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.05053\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0099\n",
      "Learner:2\tReward total:0\tRunning mean: 0.05936\n",
      "Learner:3\tReward total:0\tRunning mean: 0.04977\n",
      "Learner:4\tReward total:0\tRunning mean: 0.0\n",
      "Learner:5\tReward total:0\tRunning mean: 0.1556\n",
      "Learner:6\tReward total:0\tRunning mean: 0.1687\n",
      "Learner:7\tReward total:0\tRunning mean: 0.2007\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '129.83', '84.49', '98.25', '0.00', '99.20']\n",
      "....................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.4101\n",
      "Learner:1\tReward total:8\tRunning mean: 0.8257\n",
      "Learner:2\tReward total:0\tRunning mean: 0.3665\n",
      "Learner:3\tReward total:0\tRunning mean: 0.1494\n",
      "Learner:4\tReward total:0\tRunning mean: 0.06991\n",
      "Learner:5\tReward total:0\tRunning mean: 0.2174\n",
      "Learner:6\tReward total:0\tRunning mean: 0.7421\n",
      "Learner:7\tReward total:9\tRunning mean: 2.2\n",
      "Max Norms =  ['0.00', '148.86', '0.00', '0.00', '0.00', '0.00', '0.00', '90.25']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "....................................................................................................\n",
      "Episode 300 complete\n",
      "Learner:0\tReward total:2\tRunning mean: 0.5549\n",
      "Learner:1\tReward total:5\tRunning mean: 3.394\n",
      "Learner:2\tReward total:1\tRunning mean: 1.045\n",
      "Learner:3\tReward total:0\tRunning mean: 1.546\n",
      "Learner:4\tReward total:4\tRunning mean: 2.057\n",
      "Learner:5\tReward total:0\tRunning mean: 0.7854\n",
      "Learner:6\tReward total:1\tRunning mean: 2.034\n",
      "Learner:7\tReward total:48\tRunning mean: 18.7\n",
      "Max Norms =  ['119.38', '125.12', '128.07', '0.00', '91.93', '0.00', '172.45', '146.38']\n",
      "....................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:1\tRunning mean: 2.172\n",
      "Learner:1\tReward total:8\tRunning mean: 5.893\n",
      "Learner:2\tReward total:0\tRunning mean: 2.015\n",
      "Learner:3\tReward total:9\tRunning mean: 6.294\n",
      "Learner:4\tReward total:18\tRunning mean: 7.327\n",
      "Learner:5\tReward total:10\tRunning mean: 2.261\n",
      "Learner:6\tReward total:23\tRunning mean: 9.119\n",
      "Learner:7\tReward total:31\tRunning mean: 34.69\n",
      "Max Norms =  ['170.21', '96.76', '0.00', '83.61', '108.26', '138.28', '160.91', '118.23']\n",
      "....................................................................................................\n",
      "Episode 500 complete\n",
      "Learner:0\tReward total:12\tRunning mean: 5.962\n",
      "Learner:1\tReward total:16\tRunning mean: 9.42\n",
      "Learner:2\tReward total:38\tRunning mean: 6.04\n",
      "Learner:3\tReward total:18\tRunning mean: 13.81\n",
      "Learner:4\tReward total:6\tRunning mean: 13.29\n",
      "Learner:5\tReward total:27\tRunning mean: 8.612\n",
      "Learner:6\tReward total:28\tRunning mean: 17.74\n",
      "Learner:7\tReward total:23\tRunning mean: 34.91\n",
      "Max Norms =  ['82.24', '83.34', '135.91', '125.06', '96.51', '104.72', '139.44', '147.27']\n",
      "....................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:14\tRunning mean: 10.65\n",
      "Learner:1\tReward total:23\tRunning mean: 13.47\n",
      "Learner:2\tReward total:7\tRunning mean: 14.2\n",
      "Learner:3\tReward total:33\tRunning mean: 18.66\n",
      "Learner:4\tReward total:15\tRunning mean: 14.93\n",
      "Learner:5\tReward total:12\tRunning mean: 13.23\n",
      "Learner:6\tReward total:27\tRunning mean: 18.36\n",
      "Learner:7\tReward total:37\tRunning mean: 30.65\n",
      "Max Norms =  ['105.39', '117.37', '150.25', '166.45', '169.61', '92.86', '97.59', '199.42']\n",
      "....................................................................................................\n",
      "Episode 700 complete\n",
      "Learner:0\tReward total:23\tRunning mean: 13.13\n",
      "Learner:1\tReward total:12\tRunning mean: 16.32\n",
      "Learner:2\tReward total:7\tRunning mean: 13.67\n",
      "Learner:3\tReward total:20\tRunning mean: 22.29\n",
      "Learner:4\tReward total:11\tRunning mean: 15.17\n",
      "Learner:5\tReward total:25\tRunning mean: 15.27\n",
      "Learner:6\tReward total:17\tRunning mean: 19.71\n",
      "Learner:7\tReward total:27\tRunning mean: 27.87\n",
      "Max Norms =  ['101.51', '75.79', '113.87', '80.88', '153.66', '66.53', '47.79', '131.02']\n",
      "....................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:14\tRunning mean: 14.75\n",
      "Learner:1\tReward total:29\tRunning mean: 16.13\n",
      "Learner:2\tReward total:22\tRunning mean: 13.99\n",
      "Learner:3\tReward total:26\tRunning mean: 23.99\n",
      "Learner:4\tReward total:13\tRunning mean: 16.42\n",
      "Learner:5\tReward total:11\tRunning mean: 15.98\n",
      "Learner:6\tReward total:9\tRunning mean: 19.91\n",
      "Learner:7\tReward total:22\tRunning mean: 28.5\n",
      "Max Norms =  ['92.81', '120.49', '134.65', '81.90', '76.95', '73.05', '29.51', '144.58']\n",
      "....................................................................................................\n",
      "Episode 900 complete\n",
      "Learner:0\tReward total:14\tRunning mean: 14.71\n",
      "Learner:1\tReward total:16\tRunning mean: 15.23\n",
      "Learner:2\tReward total:7\tRunning mean: 13.71\n",
      "Learner:3\tReward total:33\tRunning mean: 29.04\n",
      "Learner:4\tReward total:4\tRunning mean: 14.72\n",
      "Learner:5\tReward total:12\tRunning mean: 15.73\n",
      "Learner:6\tReward total:20\tRunning mean: 19.0\n",
      "Learner:7\tReward total:28\tRunning mean: 27.4\n",
      "Max Norms =  ['68.11', '125.33', '243.67', '117.00', '156.80', '101.66', '62.03', '139.20']\n",
      "....................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:5\tRunning mean: 13.15\n",
      "Learner:1\tReward total:18\tRunning mean: 15.08\n",
      "Learner:2\tReward total:23\tRunning mean: 11.94\n",
      "Learner:3\tReward total:43\tRunning mean: 36.54\n",
      "Learner:4\tReward total:8\tRunning mean: 11.43\n",
      "Learner:5\tReward total:8\tRunning mean: 13.6\n",
      "Learner:6\tReward total:15\tRunning mean: 15.95\n",
      "Learner:7\tReward total:13\tRunning mean: 19.64\n",
      "Max Norms =  ['60.38', '87.91', '110.36', '144.15', '83.98', '92.73', '51.13', '118.87']\n",
      "....................................................................................................\n",
      "Episode 1100 complete\n",
      "Learner:0\tReward total:21\tRunning mean: 12.84\n",
      "Learner:1\tReward total:11\tRunning mean: 15.18\n",
      "Learner:2\tReward total:26\tRunning mean: 13.46\n",
      "Learner:3\tReward total:66\tRunning mean: 45.8\n",
      "Learner:4\tReward total:9\tRunning mean: 10.03\n",
      "Learner:5\tReward total:9\tRunning mean: 13.79\n",
      "Learner:6\tReward total:2\tRunning mean: 14.62\n",
      "Learner:7\tReward total:17\tRunning mean: 18.22\n",
      "Max Norms =  ['78.78', '110.03', '105.33', '135.59', '121.96', '102.27', '76.47', '156.27']\n",
      "....................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:7\tRunning mean: 11.9\n",
      "Learner:1\tReward total:7\tRunning mean: 15.26\n",
      "Learner:2\tReward total:9\tRunning mean: 11.57\n",
      "Learner:3\tReward total:88\tRunning mean: 58.08\n",
      "Learner:4\tReward total:16\tRunning mean: 8.91\n",
      "Learner:5\tReward total:11\tRunning mean: 11.44\n",
      "Learner:6\tReward total:6\tRunning mean: 12.17\n",
      "Learner:7\tReward total:11\tRunning mean: 15.9\n",
      "Max Norms =  ['85.88', '41.17', '107.32', '165.43', '144.82', '129.65', '57.34', '132.83']\n",
      "....................................................................................................\n",
      "Episode 1300 complete\n",
      "Learner:0\tReward total:6\tRunning mean: 11.45\n",
      "Learner:1\tReward total:28\tRunning mean: 13.47\n",
      "Learner:2\tReward total:17\tRunning mean: 11.01\n",
      "Learner:3\tReward total:44\tRunning mean: 62.39\n",
      "Learner:4\tReward total:2\tRunning mean: 9.205\n",
      "Learner:5\tReward total:11\tRunning mean: 12.11\n",
      "Learner:6\tReward total:23\tRunning mean: 10.92\n",
      "Learner:7\tReward total:13\tRunning mean: 16.04\n",
      "Max Norms =  ['74.87', '95.09', '83.65', '106.08', '166.87', '93.12', '58.48', '108.17']\n",
      "....................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:3\tRunning mean: 11.28\n",
      "Learner:1\tReward total:8\tRunning mean: 13.02\n",
      "Learner:2\tReward total:9\tRunning mean: 10.26\n",
      "Learner:3\tReward total:61\tRunning mean: 67.0\n",
      "Learner:4\tReward total:9\tRunning mean: 8.067\n",
      "Learner:5\tReward total:2\tRunning mean: 12.44\n",
      "Learner:6\tReward total:11\tRunning mean: 11.51\n",
      "Learner:7\tReward total:12\tRunning mean: 14.99\n",
      "Max Norms =  ['79.18', '63.01', '92.74', '99.83', '99.79', '76.45', '43.30', '100.07']\n",
      "....................................................................................................\n",
      "Episode 1500 complete\n",
      "Learner:0\tReward total:3\tRunning mean: 10.27\n",
      "Learner:1\tReward total:3\tRunning mean: 11.78\n",
      "Learner:2\tReward total:7\tRunning mean: 10.97\n",
      "Learner:3\tReward total:93\tRunning mean: 73.82\n",
      "Learner:4\tReward total:0\tRunning mean: 6.773\n",
      "Learner:5\tReward total:5\tRunning mean: 11.87\n",
      "Learner:6\tReward total:14\tRunning mean: 10.09\n",
      "Learner:7\tReward total:13\tRunning mean: 11.96\n",
      "Max Norms =  ['18.32', '47.09', '81.67', '118.54', '0.00', '54.91', '95.68', '100.96']\n",
      "....................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:5\tRunning mean: 9.454\n",
      "Learner:1\tReward total:8\tRunning mean: 12.09\n",
      "Learner:2\tReward total:4\tRunning mean: 9.406\n",
      "Learner:3\tReward total:79\tRunning mean: 65.2\n",
      "Learner:4\tReward total:11\tRunning mean: 8.712\n",
      "Learner:5\tReward total:15\tRunning mean: 11.88\n",
      "Learner:6\tReward total:7\tRunning mean: 10.5\n",
      "Learner:7\tReward total:7\tRunning mean: 13.56\n",
      "Max Norms =  ['181.45', '89.27', '104.22', '71.84', '135.74', '86.65', '122.48', '59.52']\n",
      "....................................................................................................\n",
      "Episode 1700 complete\n",
      "Learner:0\tReward total:18\tRunning mean: 9.996\n",
      "Learner:1\tReward total:8\tRunning mean: 13.18\n",
      "Learner:2\tReward total:9\tRunning mean: 9.177\n",
      "Learner:3\tReward total:39\tRunning mean: 55.82\n",
      "Learner:4\tReward total:28\tRunning mean: 15.87\n",
      "Learner:5\tReward total:18\tRunning mean: 13.13\n",
      "Learner:6\tReward total:10\tRunning mean: 11.39\n",
      "Learner:7\tReward total:11\tRunning mean: 15.21\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max Norms =  ['89.44', '72.66', '131.57', '104.81', '131.32', '103.82', '165.68', '74.59']\n",
      "....................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:7\tRunning mean: 10.67\n",
      "Learner:1\tReward total:20\tRunning mean: 13.87\n",
      "Learner:2\tReward total:12\tRunning mean: 10.16\n",
      "Learner:3\tReward total:66\tRunning mean: 51.91\n",
      "Learner:4\tReward total:9\tRunning mean: 20.58\n",
      "Learner:5\tReward total:14\tRunning mean: 14.41\n",
      "Learner:6\tReward total:6\tRunning mean: 9.853\n",
      "Learner:7\tReward total:23\tRunning mean: 15.73\n",
      "Max Norms =  ['98.87', '101.20', '108.78', '105.92', '119.64', '24.94', '111.71', '107.25']\n",
      "....................................................................................................\n",
      "Episode 1900 complete\n",
      "Learner:0\tReward total:5\tRunning mean: 11.45\n",
      "Learner:1\tReward total:7\tRunning mean: 13.45\n",
      "Learner:2\tReward total:11\tRunning mean: 8.888\n",
      "Learner:3\tReward total:39\tRunning mean: 52.68\n",
      "Learner:4\tReward total:41\tRunning mean: 21.15\n",
      "Learner:5\tReward total:11\tRunning mean: 14.41\n",
      "Learner:6\tReward total:13\tRunning mean: 9.794\n",
      "Learner:7\tReward total:18\tRunning mean: 15.74\n",
      "Max Norms =  ['58.94', '13.07', '131.98', '99.07', '153.73', '62.26', '115.27', '80.20']\n",
      "....................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:14\tRunning mean: 11.44\n",
      "Learner:1\tReward total:6\tRunning mean: 12.63\n",
      "Learner:2\tReward total:15\tRunning mean: 8.658\n",
      "Learner:3\tReward total:68\tRunning mean: 50.04\n",
      "Learner:4\tReward total:14\tRunning mean: 21.62\n",
      "Learner:5\tReward total:2\tRunning mean: 12.85\n",
      "Learner:6\tReward total:17\tRunning mean: 8.436\n",
      "Learner:7\tReward total:10\tRunning mean: 15.35\n",
      "Max Norms =  ['53.57', '89.24', '57.28', '178.68', '178.94', '52.72', '22.24', '136.91']\n",
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Load random agent 8\n",
      "....................................................................................................\n",
      "Episode 100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.005812\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.02519\n",
      "Learner:4\tReward total:7\tRunning mean: 0.08381\n",
      "Learner:5\tReward total:0\tRunning mean: 0.3488\n",
      "Learner:6\tReward total:0\tRunning mean: 0.03554\n",
      "Learner:7\tReward total:0\tRunning mean: 0.2553\n",
      "Max Norms =  ['93.45', '75.67', '107.77', '0.00', '223.50', '123.75', '0.00', '179.96']\n",
      "....................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.006175\n",
      "Learner:1\tReward total:0\tRunning mean: 0.08175\n",
      "Learner:2\tReward total:0\tRunning mean: 0.04368\n",
      "Learner:3\tReward total:0\tRunning mean: 0.08758\n",
      "Learner:4\tReward total:0\tRunning mean: 0.1515\n",
      "Learner:5\tReward total:0\tRunning mean: 0.6213\n",
      "Learner:6\tReward total:0\tRunning mean: 0.4297\n",
      "Learner:7\tReward total:8\tRunning mean: 4.176\n",
      "Max Norms =  ['64.75', '0.00', '0.00', '0.00', '0.00', '0.00', '0.00', '122.27']\n",
      "....................................................................................................\n",
      "Episode 300 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.009884\n",
      "Learner:1\tReward total:0\tRunning mean: 0.6527\n",
      "Learner:2\tReward total:0\tRunning mean: 0.03574\n",
      "Learner:3\tReward total:7\tRunning mean: 0.9865\n",
      "Learner:4\tReward total:0\tRunning mean: 1.977\n",
      "Learner:5\tReward total:0\tRunning mean: 1.221\n",
      "Learner:6\tReward total:3\tRunning mean: 0.7501\n",
      "Learner:7\tReward total:35\tRunning mean: 17.51\n",
      "Max Norms =  ['86.51', '0.00', '0.00', '104.79', '0.00', '0.00', '115.14', '131.00']\n",
      "....................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.003618\n",
      "Learner:1\tReward total:1\tRunning mean: 1.52\n",
      "Learner:2\tReward total:0\tRunning mean: 0.05702\n",
      "Learner:3\tReward total:4\tRunning mean: 2.811\n",
      "Learner:4\tReward total:4\tRunning mean: 6.485\n",
      "Learner:5\tReward total:12\tRunning mean: 4.022\n",
      "Learner:6\tReward total:1\tRunning mean: 2.086\n",
      "Learner:7\tReward total:57\tRunning mean: 30.7\n",
      "Max Norms =  ['0.00', '75.01', '0.00', '117.39', '62.65', '104.00', '84.66', '93.39']\n",
      "....................................................................................................\n",
      "Episode 500 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.117\n",
      "Learner:1\tReward total:12\tRunning mean: 5.869\n",
      "Learner:2\tReward total:0\tRunning mean: 0.02087\n",
      "Learner:3\tReward total:2\tRunning mean: 10.16\n",
      "Learner:4\tReward total:15\tRunning mean: 9.079\n",
      "Learner:5\tReward total:28\tRunning mean: 12.15\n",
      "Learner:6\tReward total:7\tRunning mean: 7.69\n",
      "Learner:7\tReward total:38\tRunning mean: 32.82\n",
      "Max Norms =  ['0.00', '151.89', '0.00', '157.91', '84.97', '102.30', '122.68', '136.20']\n",
      "....................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.07877\n",
      "Learner:1\tReward total:27\tRunning mean: 16.13\n",
      "Learner:2\tReward total:0\tRunning mean: 0.03401\n",
      "Learner:3\tReward total:17\tRunning mean: 15.92\n",
      "Learner:4\tReward total:17\tRunning mean: 11.01\n",
      "Learner:5\tReward total:30\tRunning mean: 20.82\n",
      "Learner:6\tReward total:30\tRunning mean: 16.08\n",
      "Learner:7\tReward total:20\tRunning mean: 29.58\n",
      "Max Norms =  ['0.00', '104.19', '0.00', '99.12', '75.41', '113.81', '148.12', '53.51']\n",
      "....................................................................................................\n",
      "Episode 700 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.02883\n",
      "Learner:1\tReward total:41\tRunning mean: 20.42\n",
      "Learner:2\tReward total:0\tRunning mean: 0.02937\n",
      "Learner:3\tReward total:24\tRunning mean: 19.28\n",
      "Learner:4\tReward total:8\tRunning mean: 12.12\n",
      "Learner:5\tReward total:24\tRunning mean: 24.17\n",
      "Learner:6\tReward total:23\tRunning mean: 19.49\n",
      "Learner:7\tReward total:11\tRunning mean: 26.42\n",
      "Max Norms =  ['0.00', '118.56', '0.00', '133.40', '80.71', '114.73', '108.70', '33.72']\n",
      "....................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.04294\n",
      "Learner:1\tReward total:31\tRunning mean: 23.08\n",
      "Learner:2\tReward total:0\tRunning mean: 0.01075\n",
      "Learner:3\tReward total:18\tRunning mean: 20.01\n",
      "Learner:4\tReward total:11\tRunning mean: 11.74\n",
      "Learner:5\tReward total:38\tRunning mean: 25.76\n",
      "Learner:6\tReward total:55\tRunning mean: 25.8\n",
      "Learner:7\tReward total:18\tRunning mean: 24.42\n",
      "Max Norms =  ['0.00', '97.57', '0.00', '134.57', '73.02', '122.79', '186.70', '47.63']\n",
      "....................................................................................................\n",
      "Episode 900 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.1202\n",
      "Learner:1\tReward total:23\tRunning mean: 25.34\n",
      "Learner:2\tReward total:0\tRunning mean: 0.003934\n",
      "Learner:3\tReward total:19\tRunning mean: 21.65\n",
      "Learner:4\tReward total:12\tRunning mean: 14.1\n",
      "Learner:5\tReward total:36\tRunning mean: 25.54\n",
      "Learner:6\tReward total:19\tRunning mean: 25.62\n",
      "Learner:7\tReward total:7\tRunning mean: 22.75\n",
      "Max Norms =  ['0.00', '52.80', '0.00', '60.21', '63.66', '136.21', '104.90', '52.19']\n",
      "....................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.3077\n",
      "Learner:1\tReward total:14\tRunning mean: 27.04\n",
      "Learner:2\tReward total:0\tRunning mean: 0.03956\n",
      "Learner:3\tReward total:35\tRunning mean: 23.79\n",
      "Learner:4\tReward total:9\tRunning mean: 14.94\n",
      "Learner:5\tReward total:13\tRunning mean: 23.11\n",
      "Learner:6\tReward total:16\tRunning mean: 24.44\n",
      "Learner:7\tReward total:39\tRunning mean: 25.7\n",
      "Max Norms =  ['0.00', '101.18', '0.00', '98.63', '65.38', '108.27', '133.42', '46.93']\n",
      "....................................................................................................\n",
      "Episode 1100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 2.645\n",
      "Learner:1\tReward total:7\tRunning mean: 24.77\n",
      "Learner:2\tReward total:0\tRunning mean: 0.01448\n",
      "Learner:3\tReward total:21\tRunning mean: 23.94\n",
      "Learner:4\tReward total:15\tRunning mean: 14.1\n",
      "Learner:5\tReward total:14\tRunning mean: 24.44\n",
      "Learner:6\tReward total:20\tRunning mean: 25.34\n",
      "Learner:7\tReward total:24\tRunning mean: 23.75\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max Norms =  ['0.00', '69.30', '0.00', '104.90', '45.40', '112.61', '188.58', '54.62']\n",
      "....................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:2\tRunning mean: 8.493\n",
      "Learner:1\tReward total:7\tRunning mean: 24.12\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0053\n",
      "Learner:3\tReward total:20\tRunning mean: 24.4\n",
      "Learner:4\tReward total:22\tRunning mean: 13.97\n",
      "Learner:5\tReward total:29\tRunning mean: 24.94\n",
      "Learner:6\tReward total:33\tRunning mean: 25.52\n",
      "Learner:7\tReward total:22\tRunning mean: 23.94\n",
      "Max Norms =  ['77.99', '93.89', '0.00', '78.64', '67.35', '103.30', '120.42', '26.15']\n",
      "....................................................................................................\n",
      "Episode 1300 complete\n",
      "Learner:0\tReward total:9\tRunning mean: 12.69\n",
      "Learner:1\tReward total:21\tRunning mean: 24.59\n",
      "Learner:2\tReward total:0\tRunning mean: 0.02698\n",
      "Learner:3\tReward total:33\tRunning mean: 23.45\n",
      "Learner:4\tReward total:11\tRunning mean: 14.41\n",
      "Learner:5\tReward total:24\tRunning mean: 24.88\n",
      "Learner:6\tReward total:43\tRunning mean: 23.84\n",
      "Learner:7\tReward total:39\tRunning mean: 23.72\n",
      "Max Norms =  ['61.92', '105.79', '0.00', '62.80', '42.77', '117.14', '186.85', '3.27']\n",
      "....................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:23\tRunning mean: 14.77\n",
      "Learner:1\tReward total:19\tRunning mean: 24.95\n",
      "Learner:2\tReward total:0\tRunning mean: 0.009874\n",
      "Learner:3\tReward total:10\tRunning mean: 23.21\n",
      "Learner:4\tReward total:16\tRunning mean: 14.74\n",
      "Learner:5\tReward total:15\tRunning mean: 23.67\n",
      "Learner:6\tReward total:12\tRunning mean: 24.6\n",
      "Learner:7\tReward total:20\tRunning mean: 22.2\n",
      "Max Norms =  ['65.28', '73.49', '0.00', '126.12', '51.85', '71.68', '94.85', '3.73']\n",
      "....................................................................................................\n",
      "Episode 1500 complete\n",
      "Learner:0\tReward total:20\tRunning mean: 16.25\n",
      "Learner:1\tReward total:3\tRunning mean: 25.99\n",
      "Learner:2\tReward total:0\tRunning mean: 0.04593\n",
      "Learner:3\tReward total:39\tRunning mean: 22.23\n",
      "Learner:4\tReward total:13\tRunning mean: 15.68\n",
      "Learner:5\tReward total:14\tRunning mean: 23.83\n",
      "Learner:6\tReward total:30\tRunning mean: 23.63\n",
      "Learner:7\tReward total:15\tRunning mean: 19.97\n",
      "Max Norms =  ['106.38', '126.65', '0.00', '111.89', '51.31', '105.58', '83.53', '13.23']\n",
      "....................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:40\tRunning mean: 16.28\n",
      "Learner:1\tReward total:12\tRunning mean: 24.84\n",
      "Learner:2\tReward total:0\tRunning mean: 0.01681\n",
      "Learner:3\tReward total:21\tRunning mean: 21.48\n",
      "Learner:4\tReward total:11\tRunning mean: 16.07\n",
      "Learner:5\tReward total:23\tRunning mean: 24.92\n",
      "Learner:6\tReward total:12\tRunning mean: 21.65\n",
      "Learner:7\tReward total:18\tRunning mean: 19.61\n",
      "Max Norms =  ['41.00', '90.36', '0.00', '81.44', '59.48', '87.08', '190.28', '1.32']\n",
      "....................................................................................................\n",
      "Episode 1700 complete\n",
      "Learner:0\tReward total:10\tRunning mean: 17.65\n",
      "Learner:1\tReward total:9\tRunning mean: 23.3\n",
      "Learner:2\tReward total:0\tRunning mean: 0.006154\n",
      "Learner:3\tReward total:27\tRunning mean: 21.78\n",
      "Learner:4\tReward total:16\tRunning mean: 14.36\n",
      "Learner:5\tReward total:12\tRunning mean: 23.87\n",
      "Learner:6\tReward total:36\tRunning mean: 24.08\n",
      "Learner:7\tReward total:11\tRunning mean: 19.26\n",
      "Max Norms =  ['73.88', '134.33', '0.00', '86.36', '47.71', '65.50', '104.91', '0.64']\n",
      "....................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:10\tRunning mean: 16.34\n",
      "Learner:1\tReward total:19\tRunning mean: 23.96\n",
      "Learner:2\tReward total:0\tRunning mean: 0.002253\n",
      "Learner:3\tReward total:9\tRunning mean: 22.65\n",
      "Learner:4\tReward total:22\tRunning mean: 14.7\n",
      "Learner:5\tReward total:25\tRunning mean: 24.25\n",
      "Learner:6\tReward total:7\tRunning mean: 25.24\n",
      "Learner:7\tReward total:25\tRunning mean: 19.26\n",
      "Max Norms =  ['15.09', '111.46', '0.00', '83.55', '5.80', '91.94', '114.75', '0.60']\n",
      "....................................................................................................\n",
      "Episode 1900 complete\n",
      "Learner:0\tReward total:29\tRunning mean: 17.23\n",
      "Learner:1\tReward total:9\tRunning mean: 23.79\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0008245\n",
      "Learner:3\tReward total:27\tRunning mean: 22.11\n",
      "Learner:4\tReward total:14\tRunning mean: 15.24\n",
      "Learner:5\tReward total:18\tRunning mean: 23.73\n",
      "Learner:6\tReward total:10\tRunning mean: 27.44\n",
      "Learner:7\tReward total:11\tRunning mean: 18.44\n",
      "Max Norms =  ['42.45', '96.72', '0.00', '55.91', '21.69', '77.08', '82.21', '0.81']\n",
      "....................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:8\tRunning mean: 17.36\n",
      "Learner:1\tReward total:18\tRunning mean: 26.02\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0003018\n",
      "Learner:3\tReward total:16\tRunning mean: 21.5\n",
      "Learner:4\tReward total:10\tRunning mean: 14.96\n",
      "Learner:5\tReward total:26\tRunning mean: 23.03\n",
      "Learner:6\tReward total:27\tRunning mean: 25.67\n",
      "Learner:7\tReward total:16\tRunning mean: 18.63\n",
      "Max Norms =  ['86.91', '187.22', '0.00', '87.11', '47.33', '70.10', '137.39', '43.82']\n",
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Load random agent 8\n",
      "....................................................................................................\n",
      "Episode 100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0211\n",
      "Learner:2\tReward total:0\tRunning mean: 0.09801\n",
      "Learner:3\tReward total:0\tRunning mean: 0.07418\n",
      "Learner:4\tReward total:0\tRunning mean: 0.03176\n",
      "Learner:5\tReward total:0\tRunning mean: 0.1599\n",
      "Learner:6\tReward total:0\tRunning mean: 0.09812\n",
      "Learner:7\tReward total:3\tRunning mean: 0.5034\n",
      "Max Norms =  ['0.00', '128.20', '0.00', '111.73', '0.00', '127.50', '0.00', '110.30']\n",
      "....................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.4706\n",
      "Learner:1\tReward total:0\tRunning mean: 0.211\n",
      "Learner:2\tReward total:0\tRunning mean: 0.5754\n",
      "Learner:3\tReward total:0\tRunning mean: 0.6853\n",
      "Learner:4\tReward total:3\tRunning mean: 0.2878\n",
      "Learner:5\tReward total:4\tRunning mean: 1.041\n",
      "Learner:6\tReward total:0\tRunning mean: 0.5399\n",
      "Learner:7\tReward total:5\tRunning mean: 2.852\n",
      "Max Norms =  ['94.05', '0.00', '0.00', '0.00', '123.73', '102.49', '0.00', '91.76']\n",
      "....................................................................................................\n",
      "Episode 300 complete\n",
      "Learner:0\tReward total:7\tRunning mean: 2.924\n",
      "Learner:1\tReward total:5\tRunning mean: 0.9208\n",
      "Learner:2\tReward total:3\tRunning mean: 0.8678\n",
      "Learner:3\tReward total:0\tRunning mean: 1.35\n",
      "Learner:4\tReward total:0\tRunning mean: 1.259\n",
      "Learner:5\tReward total:0\tRunning mean: 2.064\n",
      "Learner:6\tReward total:6\tRunning mean: 5.396\n",
      "Learner:7\tReward total:3\tRunning mean: 5.386\n",
      "Max Norms =  ['132.89', '110.67', '111.63', '0.00', '0.00', '0.00', '109.09', '94.21']\n",
      "....................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:4\tRunning mean: 7.673\n",
      "Learner:1\tReward total:23\tRunning mean: 4.299\n",
      "Learner:2\tReward total:0\tRunning mean: 1.43\n",
      "Learner:3\tReward total:7\tRunning mean: 3.24\n",
      "Learner:4\tReward total:9\tRunning mean: 2.829\n",
      "Learner:5\tReward total:30\tRunning mean: 8.592\n",
      "Learner:6\tReward total:18\tRunning mean: 15.55\n",
      "Learner:7\tReward total:28\tRunning mean: 13.43\n",
      "Max Norms =  ['108.19', '81.91', '0.00', '81.13', '141.05', '100.80', '118.14', '124.80']\n",
      "....................................................................................................\n",
      "Episode 500 complete\n",
      "Learner:0\tReward total:17\tRunning mean: 14.05\n",
      "Learner:1\tReward total:20\tRunning mean: 8.109\n",
      "Learner:2\tReward total:17\tRunning mean: 4.756\n",
      "Learner:3\tReward total:7\tRunning mean: 6.962\n",
      "Learner:4\tReward total:6\tRunning mean: 4.177\n",
      "Learner:5\tReward total:13\tRunning mean: 20.53\n",
      "Learner:6\tReward total:41\tRunning mean: 17.67\n",
      "Learner:7\tReward total:20\tRunning mean: 18.83\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max Norms =  ['108.78', '85.71', '116.13', '125.00', '113.89', '143.01', '112.85', '115.93']\n",
      "....................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:34\tRunning mean: 16.47\n",
      "Learner:1\tReward total:14\tRunning mean: 14.78\n",
      "Learner:2\tReward total:21\tRunning mean: 11.22\n",
      "Learner:3\tReward total:4\tRunning mean: 10.59\n",
      "Learner:4\tReward total:7\tRunning mean: 8.561\n",
      "Learner:5\tReward total:15\tRunning mean: 19.46\n",
      "Learner:6\tReward total:13\tRunning mean: 18.53\n",
      "Learner:7\tReward total:27\tRunning mean: 23.17\n",
      "Max Norms =  ['143.93', '98.90', '114.25', '84.79', '57.74', '79.15', '70.65', '108.62']\n",
      "....................................................................................................\n",
      "Episode 700 complete\n",
      "Learner:0\tReward total:27\tRunning mean: 18.73\n",
      "Learner:1\tReward total:18\tRunning mean: 17.62\n",
      "Learner:2\tReward total:13\tRunning mean: 15.19\n",
      "Learner:3\tReward total:10\tRunning mean: 13.03\n",
      "Learner:4\tReward total:24\tRunning mean: 11.7\n",
      "Learner:5\tReward total:12\tRunning mean: 19.01\n",
      "Learner:6\tReward total:21\tRunning mean: 17.36\n",
      "Learner:7\tReward total:25\tRunning mean: 25.48\n",
      "Max Norms =  ['114.34', '81.12', '106.99', '166.68', '83.97', '67.58', '93.67', '106.43']\n",
      "....................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:16\tRunning mean: 17.97\n",
      "Learner:1\tReward total:24\tRunning mean: 18.97\n",
      "Learner:2\tReward total:27\tRunning mean: 16.84\n",
      "Learner:3\tReward total:10\tRunning mean: 15.11\n",
      "Learner:4\tReward total:9\tRunning mean: 13.85\n",
      "Learner:5\tReward total:17\tRunning mean: 20.12\n",
      "Learner:6\tReward total:27\tRunning mean: 17.52\n",
      "Learner:7\tReward total:30\tRunning mean: 25.99\n",
      "Max Norms =  ['110.62', '87.32', '88.31', '61.77', '44.98', '135.68', '107.39', '104.05']\n",
      "....................................................................................................\n",
      "Episode 900 complete\n",
      "Learner:0\tReward total:29\tRunning mean: 18.73\n",
      "Learner:1\tReward total:4\tRunning mean: 19.01\n",
      "Learner:2\tReward total:27\tRunning mean: 17.92\n",
      "Learner:3\tReward total:1\tRunning mean: 15.33\n",
      "Learner:4\tReward total:15\tRunning mean: 14.54\n",
      "Learner:5\tReward total:12\tRunning mean: 19.47\n",
      "Learner:6\tReward total:30\tRunning mean: 17.35\n",
      "Learner:7\tReward total:23\tRunning mean: 28.7\n",
      "Max Norms =  ['151.15', '65.67', '48.47', '69.21', '92.96', '71.10', '95.46', '124.16']\n",
      "....................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:11\tRunning mean: 20.28\n",
      "Learner:1\tReward total:17\tRunning mean: 20.14\n",
      "Learner:2\tReward total:20\tRunning mean: 17.7\n",
      "Learner:3\tReward total:20\tRunning mean: 14.4\n",
      "Learner:4\tReward total:16\tRunning mean: 15.23\n",
      "Learner:5\tReward total:15\tRunning mean: 19.57\n",
      "Learner:6\tReward total:21\tRunning mean: 17.09\n",
      "Learner:7\tReward total:25\tRunning mean: 27.41\n",
      "Max Norms =  ['119.90', '121.11', '78.12', '114.04', '60.28', '40.20', '50.68', '92.79']\n",
      "....................................................................................................\n",
      "Episode 1100 complete\n",
      "Learner:0\tReward total:24\tRunning mean: 17.46\n",
      "Learner:1\tReward total:14\tRunning mean: 19.84\n",
      "Learner:2\tReward total:13\tRunning mean: 18.58\n",
      "Learner:3\tReward total:13\tRunning mean: 14.75\n",
      "Learner:4\tReward total:17\tRunning mean: 15.25\n",
      "Learner:5\tReward total:25\tRunning mean: 18.4\n",
      "Learner:6\tReward total:39\tRunning mean: 19.79\n",
      "Learner:7\tReward total:22\tRunning mean: 27.69\n",
      "Max Norms =  ['136.10', '69.56', '71.22', '165.02', '94.12', '88.58', '123.58', '84.95']\n",
      "....................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:16\tRunning mean: 19.05\n",
      "Learner:1\tReward total:17\tRunning mean: 20.26\n",
      "Learner:2\tReward total:30\tRunning mean: 19.41\n",
      "Learner:3\tReward total:10\tRunning mean: 16.94\n",
      "Learner:4\tReward total:14\tRunning mean: 15.76\n",
      "Learner:5\tReward total:21\tRunning mean: 16.52\n",
      "Learner:6\tReward total:46\tRunning mean: 18.8\n",
      "Learner:7\tReward total:17\tRunning mean: 27.54\n",
      "Max Norms =  ['167.36', '36.44', '106.65', '97.08', '53.47', '65.89', '70.60', '75.00']\n",
      "....................................................................................................\n",
      "Episode 1300 complete\n",
      "Learner:0\tReward total:11\tRunning mean: 18.71\n",
      "Learner:1\tReward total:39\tRunning mean: 20.19\n",
      "Learner:2\tReward total:11\tRunning mean: 18.19\n",
      "Learner:3\tReward total:27\tRunning mean: 18.36\n",
      "Learner:4\tReward total:20\tRunning mean: 16.16\n",
      "Learner:5\tReward total:15\tRunning mean: 16.7\n",
      "Learner:6\tReward total:11\tRunning mean: 17.9\n",
      "Learner:7\tReward total:27\tRunning mean: 27.5\n",
      "Max Norms =  ['114.20', '73.32', '41.60', '112.91', '68.17', '56.55', '45.33', '90.81']\n",
      "....................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:35\tRunning mean: 20.27\n",
      "Learner:1\tReward total:34\tRunning mean: 20.59\n",
      "Learner:2\tReward total:17\tRunning mean: 18.55\n",
      "Learner:3\tReward total:3\tRunning mean: 17.57\n",
      "Learner:4\tReward total:4\tRunning mean: 16.18\n",
      "Learner:5\tReward total:15\tRunning mean: 17.46\n",
      "Learner:6\tReward total:26\tRunning mean: 19.48\n",
      "Learner:7\tReward total:32\tRunning mean: 27.36\n",
      "Max Norms =  ['121.34', '105.73', '102.96', '125.91', '3.30', '21.81', '96.36', '84.02']\n",
      "....................................................................................................\n",
      "Episode 1500 complete\n",
      "Learner:0\tReward total:18\tRunning mean: 19.33\n",
      "Learner:1\tReward total:19\tRunning mean: 20.0\n",
      "Learner:2\tReward total:12\tRunning mean: 17.18\n",
      "Learner:3\tReward total:19\tRunning mean: 18.73\n",
      "Learner:4\tReward total:13\tRunning mean: 15.28\n",
      "Learner:5\tReward total:11\tRunning mean: 17.09\n",
      "Learner:6\tReward total:31\tRunning mean: 20.62\n",
      "Learner:7\tReward total:36\tRunning mean: 29.14\n",
      "Max Norms =  ['213.60', '84.55', '55.23', '129.15', '20.58', '3.30', '73.53', '91.17']\n",
      "....................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:41\tRunning mean: 20.77\n",
      "Learner:1\tReward total:30\tRunning mean: 18.51\n",
      "Learner:2\tReward total:8\tRunning mean: 16.85\n",
      "Learner:3\tReward total:21\tRunning mean: 20.67\n",
      "Learner:4\tReward total:17\tRunning mean: 15.68\n",
      "Learner:5\tReward total:16\tRunning mean: 15.96\n",
      "Learner:6\tReward total:22\tRunning mean: 20.43\n",
      "Learner:7\tReward total:17\tRunning mean: 28.02\n",
      "Max Norms =  ['101.35', '128.58', '144.97', '139.66', '4.22', '0.76', '171.57', '67.33']\n",
      "....................................................................................................\n",
      "Episode 1700 complete\n",
      "Learner:0\tReward total:42\tRunning mean: 21.76\n",
      "Learner:1\tReward total:29\tRunning mean: 19.87\n",
      "Learner:2\tReward total:20\tRunning mean: 18.78\n",
      "Learner:3\tReward total:21\tRunning mean: 20.36\n",
      "Learner:4\tReward total:21\tRunning mean: 15.75\n",
      "Learner:5\tReward total:15\tRunning mean: 16.12\n",
      "Learner:6\tReward total:21\tRunning mean: 20.81\n",
      "Learner:7\tReward total:14\tRunning mean: 24.84\n",
      "Max Norms =  ['227.66', '187.84', '82.60', '125.84', '2.70', '1.47', '9.81', '87.14']\n",
      "....................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:8\tRunning mean: 21.68\n",
      "Learner:1\tReward total:22\tRunning mean: 21.08\n",
      "Learner:2\tReward total:15\tRunning mean: 18.82\n",
      "Learner:3\tReward total:20\tRunning mean: 19.44\n",
      "Learner:4\tReward total:4\tRunning mean: 16.0\n",
      "Learner:5\tReward total:5\tRunning mean: 16.34\n",
      "Learner:6\tReward total:30\tRunning mean: 22.11\n",
      "Learner:7\tReward total:39\tRunning mean: 26.33\n",
      "Max Norms =  ['182.65', '102.86', '66.71', '107.72', '45.38', '0.76', '56.54', '217.56']\n",
      "....................................................................................................\n",
      "Episode 1900 complete\n",
      "Learner:0\tReward total:29\tRunning mean: 22.57\n",
      "Learner:1\tReward total:19\tRunning mean: 21.22\n",
      "Learner:2\tReward total:21\tRunning mean: 18.07\n",
      "Learner:3\tReward total:19\tRunning mean: 19.9\n",
      "Learner:4\tReward total:11\tRunning mean: 15.82\n",
      "Learner:5\tReward total:16\tRunning mean: 16.23\n",
      "Learner:6\tReward total:28\tRunning mean: 22.03\n",
      "Learner:7\tReward total:38\tRunning mean: 25.58\n",
      "Max Norms =  ['289.59', '129.66', '55.37', '66.22', '18.33', '0.28', '93.57', '133.53']\n",
      "....................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:8\tRunning mean: 22.14\n",
      "Learner:1\tReward total:25\tRunning mean: 20.81\n",
      "Learner:2\tReward total:21\tRunning mean: 17.61\n",
      "Learner:3\tReward total:10\tRunning mean: 17.44\n",
      "Learner:4\tReward total:11\tRunning mean: 14.96\n",
      "Learner:5\tReward total:14\tRunning mean: 15.1\n",
      "Learner:6\tReward total:21\tRunning mean: 21.78\n",
      "Learner:7\tReward total:34\tRunning mean: 26.74\n",
      "Max Norms =  ['134.69', '125.52', '79.07', '72.74', '0.79', '0.72', '20.80', '109.74']\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import sys\n",
    "import pickle\n",
    "from collections import deque\n",
    "\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import numpy as np\n",
    "\n",
    "# Initialize agents parameters\n",
    "#   9 agents - 9 learning agents, 0 trained agent, 0 random agent\n",
    "num_learners = 9\n",
    "num_trained = 0\n",
    "num_rdn = 0\n",
    "num_statics = num_trained + num_rdn\n",
    "num_agents = num_learners + num_statics  \n",
    "\n",
    "# Initialize environment\n",
    "game = \"Gather\"\n",
    "num_actions = 8                       # 8 actions in Gathering\n",
    "\n",
    "# Initialize training parameters\n",
    "warm_start = False\n",
    "num_frames = 4      # environ observation consists of a list of 4 stacked frames per agent\n",
    "max_episodes = 5000\n",
    "max_frames = 300\n",
    "max_frames_ep = 0   # track highest number of frames an episode can last\n",
    "\n",
    "# These trainer parameters works for Atari Breakout\n",
    "gamma = 0.99  \n",
    "lr = 1e-3\n",
    "temp_start = 1.8  # Temperature for explore/exploit\n",
    "temp_end = 1.0\n",
    "log_interval = 200\n",
    "save_interval = 500\n",
    "\n",
    "def unpack_env_obs(env_obs):\n",
    "    \"\"\"\n",
    "    Gathering is a partially-observable Markov Game. env_obs returned by GatheringEnv is a numpy \n",
    "    array of dimension (num_agent, 800), which represents the agents' observations of the game.\n",
    "\n",
    "    The 800 elements (view_box) encodes 4 layers of 10x20 pixels frames in the format:\n",
    "    (viewbox_width, viewbox_depth, 4).\n",
    "    \n",
    "    This code reshapes the above into stacked frames that can be accepted by the Policy class:\n",
    "    (batch_idx, in_channel, width, height)\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    num_agents = len(env_obs)  # environ observations is a list of agents' observations\n",
    "    \n",
    "    obs = []\n",
    "    for i in range(num_agents):\n",
    "        x = env_obs[i]   # take the indexed agent's observation\n",
    "        x = torch.Tensor(x)   # Convert to tensor\n",
    "        \n",
    "        # Policy is a 3-layer CNN\n",
    "        x = x.view(1, 10, 20, -1)  # reshape into environment defined stacked frames\n",
    "        x = x.permute(0, 3, 1, 2)  # permute to Policy accepted stacked frames\n",
    "        obs.append(x)\n",
    "        \n",
    "    return obs  # return a list of Tensors\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "For now, we do not implement LSTM            \n",
    "# LSTM Change: Need to cycle hx and cx thru function\n",
    "def select_action(model, state, lstm_hc, cuda):\n",
    "    hx , cx = lstm_hc \n",
    "    num_frames, height, width = state.shape\n",
    "    state = torch.FloatTensor(state.reshape(-1, num_frames, height, width))\n",
    "\n",
    "    if cuda:\n",
    "        state = state.cuda()\n",
    "\n",
    "    probs, value, (hx, cx) = model((Variable(state), (hx, cx)))\n",
    "\n",
    "    m = torch.distributions.Categorical(probs)\n",
    "    action = m.sample()\n",
    "    log_prob = m.log_prob(action)\n",
    "    # LSTM Change: Need to cycle hx and cx thru function\n",
    "    return action.data[0], log_prob, value, (hx, cx)\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "def select_learner_action(model, obs, cuda):\n",
    "    \"\"\"\n",
    "    This code expects obs to be an array of stacked frames of the following dim:\n",
    "    (batch_idx, in_channel, width, height)\n",
    "    \n",
    "    This is inputted into model - the agent's Policy, which outputs a probability \n",
    "    distribution over available actions.\n",
    "    \n",
    "    Policy gradient is implemented using torch.distributions.Categorical. \n",
    "    \"\"\"\n",
    "    \n",
    "    # Policy is a 3-layer CNN\n",
    "    # _, num_frames, width, height = obs.shape\n",
    "    # obs = torch.FloatTensor(obs.reshape(-1, num_frames, width, height))\n",
    "    \n",
    "    # Policy is a 2-layer NN for now\n",
    "    # obs = obs.view(1, -1)\n",
    "   \n",
    "    if cuda:\n",
    "        obs = obs.cuda()\n",
    "      \n",
    "    probs = model(obs)\n",
    "    m = torch.distributions.Categorical(probs)\n",
    "    action = m.sample()\n",
    "    log_prob = m.log_prob(action)\n",
    "\n",
    "    return action.item(), log_prob \n",
    "\n",
    "\n",
    "def load_info(agents, narrate=False):\n",
    "    for i in range(num_agents):    \n",
    "        agents[i].load_info(info[i])\n",
    "        if narrate:\n",
    "            if agents[i].tagged:\n",
    "                print('frame {}, agent{} is tagged'.format(frame,i))\n",
    "            if agents[i].laser_fired:\n",
    "                print('frame {}, agent{} fires its laser'.format(frame,i))\n",
    "                print('and hit {} US and {} THEM'.format(agents[i].US_hit, agents[i].THEM_hit))\n",
    "    return\n",
    "\n",
    "\n",
    "# The main code starts here!!!\n",
    "\n",
    "# Cultural parameter search\n",
    "cultures =[{'name':'no_fragging', 'penalty':-0.01},\n",
    "           {'name':'no_fragging', 'penalty':-0.1},\n",
    "           {'name':'no_fragging', 'penalty':-1.0},\n",
    "           {'name':'no_fragging', 'penalty':-10.0},\n",
    "           {'name':'no_fragging', 'penalty':-100.0}\n",
    "          ]\n",
    "\n",
    "for culture in cultures:   # Go down the list of cultures\n",
    "\n",
    "    # Data structure for agents\n",
    "    agents = []\n",
    "    actions = []\n",
    "    log_probs = []\n",
    "    tags = []\n",
    "    rewards = []\n",
    "    optimizers = []\n",
    "\n",
    "    # Cold start\n",
    "    if warm_start is False:\n",
    "   \n",
    "        # Initialize learner agents, then load static agents (trained followed by random)\n",
    "        for i in range(num_learners):\n",
    "            print(\"Learner agent {}\".format(i))\n",
    "            agents.append(Policy(num_frames, num_actions, i)) # No weights loaded for learning agent\n",
    "            optimizers.append(optim.Adam(agents[i].parameters(), lr=lr))\n",
    "        \n",
    "            # set up optimizer - this works for Atari Breakout\n",
    "            # optimizers.append(optim.RMSprop(agents[i].parameters(), lr=lr, weight_decay=0.1)) \n",
    "        \n",
    "        for i in range(num_learners, num_learners+num_trained):\n",
    "            print (\"No trained agent exist yet!\")\n",
    "            raise\n",
    "            \"\"\"\n",
    "            Disable for now! No trained model exist!!!\n",
    "            agents.append(Policy(num_frames, num_actions, i))\n",
    "            agents[i].load_weights()         # load weight for static agent        \n",
    "            \"\"\"\n",
    "        for i in range(num_learners+num_trained, num_agents):\n",
    "            print(\"Load random agent {}\".format(i))\n",
    "            agents.append(Rdn_Policy())\n",
    "\n",
    "    \n",
    "        # Initialize all agent data\n",
    "        actions = [0 for i in range(num_agents)]\n",
    "        log_probs = [0 for i in range(num_agents)]\n",
    "        tags = [0 for i in range(num_agents)]\n",
    "        rewards = [0 for i in range(num_agents)]\n",
    "\n",
    "        # Keep track of rewards learned by learners\n",
    "        episode_reward = [0 for i in range(num_learners)]   # reward for an episode\n",
    "        running_reward = [None for i in range(num_learners)]   # running average\n",
    "        running_rewards = [[] for i in range(num_learners)]   # history of running averages\n",
    "        best_reward = [0 for i in range(num_learners)]    # best running average (for storing best_model)\n",
    "\n",
    "        # This is to support warm start for training\n",
    "        prior_eps = 0\n",
    "\n",
    "    # Warm start\n",
    "    if warm_start:\n",
    "        print (\"Cannot warm start\")\n",
    "        raise\n",
    "    \n",
    "        \"\"\"\n",
    "        # Disable for now!  Need to ensure model can support training on GPU and game playing\n",
    "        # on both CPU and GPU.\n",
    "    \n",
    "        data_file = 'results/{}.p'.format(game)\n",
    "\n",
    "        try:\n",
    "            with open(data_file, 'rb') as f:\n",
    "                running_rewards = pickle.load(f)\n",
    "                running_reward = running_rewards[-1]\n",
    "\n",
    "            prior_eps = len(running_rewards)\n",
    "\n",
    "            model_file = 'saved_models/actor_critic_{}_ep_{}.p'.format(game, prior_eps)\n",
    "            with open(model_file, 'rb') as f:\n",
    "                # Model Save and Load Update: Include both model and optim parameters\n",
    "                saved_model = pickle.load(f)\n",
    "                model, optimizer = saved_model\n",
    "\n",
    "        except OSError:\n",
    "            print('Saved file not found. Creating new cold start model.')\n",
    "            model = Policy(input_channels=num_frames, num_actions=num_actions)\n",
    "            optimizer = optim.RMSprop(model.parameters(), lr=lr,\n",
    "                                      weight_decay=0.1)\n",
    "            running_rewards = []\n",
    "            prior_eps = 0\n",
    "        \"\"\"\n",
    "\n",
    "    # Establish tribal association\n",
    "\n",
    "    tribes = []\n",
    "    tribes.append(Tribe(name='Vikings',color='blue', culture=culture, \\\n",
    "                    agents=[agents[0], agents[1], agents[2]]))\n",
    "    tribes.append(Tribe(name='Saxons', color='red', culture=culture, \\\n",
    "                    agents=[agents[3], agents[4], agents[5]]))\n",
    "    tribes.append(Tribe(name='Franks', color='purple', culture=culture, \\\n",
    "                    agents=[agents[6], agents[7], agents[8]]))\n",
    "    # tribes.append(Tribe(name='Crazies', color='yellow', agents=[agents[9]]))   # random agents are crazy!!!\n",
    "\n",
    "\n",
    "\n",
    "    # 9 agents in 4 tribes, used map defined in default.txt\n",
    "    agent_colors = [agent.color for agent in agents]\n",
    "    agent_tribes = [agent.tribe for agent in agents]\n",
    "    env = GatheringEnv(n_agents=num_agents,agent_colors=agent_colors, agent_tribes=agent_tribes, \\\n",
    "                       map_name='default')    \n",
    "   \n",
    "    \n",
    "    cuda = torch.cuda.is_available()\n",
    "\n",
    "    if cuda:\n",
    "        for i in range(num_learners):    # Learning agents need to utilize GPU\n",
    "            agents[i].cuda()\n",
    "\n",
    "        \n",
    "    for ep in range(max_episodes):\n",
    "    \n",
    "        print('.', end='')  # To show progress\n",
    "    \n",
    "        # Anneal temperature from temp_start to temp_end\n",
    "        for i in range(num_learners):    # For learning agents\n",
    "            agents[i].temperature = max(temp_end, temp_start - (temp_start - temp_end) * (ep / max_episodes))\n",
    "\n",
    "        env_obs = env.reset()  # Env return observations\n",
    "\n",
    "        # For Debug only\n",
    "        # print (len(env_obs))\n",
    "        # print (env_obs[0].shape)\n",
    "    \n",
    "        # Unpack observations into data structure compatible with agent Policy\n",
    "        agents_obs = unpack_env_obs(env_obs)\n",
    "\n",
    "        for i in range(num_learners):    # Reset agent info - laser tag statistics\n",
    "            agents[i].reset_info()   \n",
    "\n",
    "        # For Debug only\n",
    "        # print (len(agents_obs))\n",
    "        # print (agents_obs[0].shape)\n",
    "    \n",
    "        \"\"\"\n",
    "        For now, we do not stack observations, and we do not implement LSTM\n",
    "    \n",
    "        state = np.stack([state]*num_frames)\n",
    "\n",
    "        # LSTM change - reset LSTM hidden units when episode begins\n",
    "        cx = Variable(torch.zeros(1, 256))\n",
    "        hx = Variable(torch.zeros(1, 256))\n",
    "        if cuda:\n",
    "            cx = cx.cuda()\n",
    "            hx = hx.cuda()\n",
    "        \"\"\"\n",
    "\n",
    "        episode_reward = [0 for i in range(num_learners)]   # reward for an episode\n",
    "    \n",
    "        for frame in range(max_frames):\n",
    "\n",
    "            \"\"\"\n",
    "            For now, we do not implement LSTM\n",
    "            # Select action\n",
    "            # LSTM Change: Need to cycle hx and cx thru select_action\n",
    "            action, log_prob, value, (hx,cx)  = select_action(model, state, (hx,cx), cuda)        \n",
    "            \"\"\"\n",
    "\n",
    "            for i in range(num_learners):    # For learning agents\n",
    "                actions[i], log_probs[i] = select_learner_action(agents[i], agents_obs[i], cuda)\n",
    "                if actions[i] is 6:\n",
    "                    tags[i] += 1   # record a tag for accessing aggressiveness\n",
    "                agents[i].saved_actions.append((log_probs[i]))\n",
    "            \n",
    "                # Do not implement LSTM for now\n",
    "                # actions[i].saved_actions.append((log_prob, value))\n",
    "            \n",
    "            for i in range(num_learners, num_learners+num_trained):\n",
    "                print (\"No trained agent exist yet!\")\n",
    "                raise\n",
    "            for i in range(num_learners+num_trained, num_agents):   # For random agents\n",
    "                actions[i] = agents[i].select_action(agents_obs[i])\n",
    "                if actions[i] is 6:\n",
    "                    tags[i] += 1   # record a tag for accessing aggressiveness\n",
    "\n",
    "            # For Debug only\n",
    "            # if frame % 20 == 0:\n",
    "            #    print (actions) \n",
    "            #    print (log_probs)\n",
    "            \n",
    "            # Perform step        \n",
    "            env_obs, reward, done, info = env.step(actions)\n",
    "        \n",
    "            \"\"\"\n",
    "            For Debug only\n",
    "            print (env_obs)\n",
    "            print (reward)\n",
    "            print (done) \n",
    "            \"\"\"\n",
    "       \n",
    "            # Unpack observations into data structure compatible with agent Policy\n",
    "            agents_obs = unpack_env_obs(env_obs)\n",
    "            load_info(agents, narrate=False)   # Load agent info for AI agents\n",
    "\n",
    "            # For learner agents only, generate reward statistics and reward stack for policy gradient\n",
    "            for i in range(num_learners):\n",
    "                agents[i].rewards.append(reward[i])  # Stack rewards (for policy gradient)\n",
    "                episode_reward[i] += reward[i]   # accumulate episode reward \n",
    "            \n",
    "            \"\"\"\n",
    "            For now, we do not stack observation, may come in handy later on\n",
    "        \n",
    "            # Evict oldest diff add new diff to state\n",
    "            next_state = np.stack([next_state]*num_frames)\n",
    "            next_state[1:, :, :] = state[:-1, :, :]\n",
    "            state = next_state\n",
    "            \"\"\"\n",
    "\n",
    "            if any(done):\n",
    "                print(\"Done after {} frames\".format(frame))\n",
    "                break\n",
    "            \n",
    "        if frame > max_frames_ep:\n",
    "            max_frames_ep = frame    # Keep track of highest frames/episode\n",
    "\n",
    "        # Update reward statistics for learners\n",
    "        for i in range(num_learners):\n",
    "            if running_reward[i] is None:\n",
    "                running_reward[i] = episode_reward[i]\n",
    "            running_reward[i] = running_reward[i] * 0.99 + episode_reward[i] * 0.01\n",
    "            running_rewards[i].append(running_reward[i])\n",
    "\n",
    "        # Track Episode #, temp and highest frames/episode\n",
    "        if (ep+prior_eps+1) % log_interval == 0: \n",
    "            verbose_str = '\\nEpisode {} complete'.format(ep+prior_eps+1)\n",
    "            # verbose_str += '\\tTemp = {:.4}'.format(model.temperature)\n",
    "            # verbose_str += '\\tMax frames = {}'.format(max_frames_ep+1)\n",
    "            print(verbose_str)\n",
    "    \n",
    "            # Display rewards and running rewards for learning agents\n",
    "            for i in range(num_learners):\n",
    "                verbose_str = 'Learner:{}'.format(i)\n",
    "                verbose_str += '\\tReward total:{}'.format(episode_reward[i])\n",
    "                verbose_str += '\\tRunning mean: {:.4}'.format(running_reward[i])\n",
    "                print(verbose_str)\n",
    "    \n",
    "        # Update model\n",
    "        total_norms = finish_episode(agents[0:num_learners], optimizers[0:num_learners], gamma, cuda)\n",
    "\n",
    "        if (ep+prior_eps+1) % log_interval == 0:\n",
    "            print('Max Norms = ',[\"%0.2f\" % i for i in total_norms])\n",
    "        \n",
    "        if (ep+prior_eps+1) % save_interval == 0: \n",
    "            for i in range(num_learners):\n",
    "                model_dir = 'MA_models/3T-9L/'\n",
    "                results_dir = 'results/3T-9L/'\n",
    "\n",
    "                model_file = model_dir+'{}/p{}/MA{}_{}_ep{}.p'.format(culture['name'], culture['penalty'],\\\n",
    "                                 i, game, ep+prior_eps+1)\n",
    "                data_file = results_dir+'{}/p{}/MA{}_{}.p'.format(culture['name'], culture['penalty'],\\\n",
    "                                                      i, game)\n",
    "                \n",
    "                os.makedirs(os.path.dirname(model_file), exist_ok=True)\n",
    "                os.makedirs(os.path.dirname(data_file), exist_ok=True)\n",
    "                \n",
    "                with open(model_file, 'wb') as f:\n",
    "                    # Model Save and Load Update: Include both model and optim parameters \n",
    "                    pickle.dump((agents[i].cpu(), optimizers[i]), f)\n",
    "\n",
    "                if cuda:\n",
    "                    agents[i] = agents[i].cuda()\n",
    "\n",
    "                with open(data_file, 'wb') as f:\n",
    "                    pickle.dump(running_rewards[i], f)    \n",
    "            \n",
    "    env.close()  # Close the environment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Culture = Pacifist\n",
    "\n",
    "Run finish_episode() specific to this culture."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "def finish_episode(learners, optimizers, gamma, cuda):\n",
    "    \"\"\" \n",
    "    Note that in RL, policy gradient is calculated at the end of an episode and only then used to \n",
    "    update the weights of an agent's policy. This is very different compared to image recog.\n",
    "    \n",
    "    The code will perform policy update on each learning agent independently. Reward for each time \n",
    "    step is stored in the list policy.rewards[] --> r(t)\n",
    "    \"\"\"  \n",
    "    \n",
    "    num_learners = len(learners)\n",
    "    total_norms = [0 for i in range(num_learners)]\n",
    "    policy_losses = [[] for i in range(num_learners)]\n",
    "    losses = [[] for i in range(num_learners)]\n",
    "    T_reward = []\n",
    "\n",
    "   \n",
    "    for i in range(num_learners):\n",
    "\n",
    "        R = 0\n",
    "        saved_actions = learners[i].saved_actions\n",
    "        \n",
    "        for t in tribes:\n",
    "            if t.name is learners[i].tribe:\n",
    "                T_reward = t.tribal_awards(tag_hist = learners[i].tag_hist)\n",
    " \n",
    "                # For debug only\n",
    "                # print('Agent{} receives tribal award from Tribe{}'.format(i,t.name))\n",
    "                # print (T_reward)\n",
    "                # print (learners[i].rewards)\n",
    "                \n",
    "        # Do not implement actor-critic for now\n",
    "        # value_losses = []\n",
    "        \n",
    "        rewards = deque()\n",
    "\n",
    "        for r,T in zip(learners[i].rewards[::-1],T_reward[::-1]):\n",
    "            # The agent is incentivized to cooperate by an award of 30% of what the tribe takes\n",
    "            # in by all its members\n",
    "            R = r + T + gamma * R\n",
    "            rewards.appendleft(R)\n",
    "            \n",
    "        rewards = list(rewards)\n",
    "        rewards = torch.Tensor(rewards)\n",
    "        if cuda:\n",
    "            rewards = rewards.cuda()\n",
    "\n",
    "        # z-score rewards\n",
    "        rewards = (rewards - rewards.mean()) / (1.1e-7+rewards.std())\n",
    "        \n",
    "        #Debug     \n",
    "        #print (rewards)       \n",
    "        \n",
    "        \"\"\"\n",
    "        Do not implement actor-critic for now!!!\n",
    "        for (log_prob, state_value), r in zip(saved_actions, rewards):\n",
    "            reward = r - state_value.data[0]\n",
    "            policy_losses.append(-log_prob * Variable(reward))\n",
    "            r = torch.Tensor([r])\n",
    "            if cuda:\n",
    "                r = r.cuda()\n",
    "            value_losses.append(torch.nn.functional.smooth_l1_loss(state_value,\n",
    "                                                               Variable(r)))\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss = torch.stack(policy_losses).sum() + torch.stack(value_losses).sum()\n",
    "        loss.backward()        \n",
    "        \n",
    "        \n",
    "        \"\"\"\n",
    "        for log_prob, r in zip(saved_actions, rewards):\n",
    "            r = torch.Tensor([r])\n",
    "            if cuda:\n",
    "                r = r.cuda()\n",
    "            policy_losses[i].append(-log_prob * Variable(r))\n",
    "\n",
    "        optimizers[i].zero_grad()\n",
    "        losses[i] = torch.stack(policy_losses[i]).sum()\n",
    "        losses[i].backward()\n",
    "        \n",
    "        # Gradient Clipping Update: prevent exploding gradient\n",
    "        total_norms[i] = torch.nn.utils.clip_grad_norm_(learners[i].parameters(), 8000)\n",
    "        \n",
    "        optimizers[i].step()\n",
    "        learners[i].clear_history()   # clear an agent's history at the end of episode\n",
    "\n",
    "\n",
    "    return total_norms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Load random agent 8\n",
      "....................................................................................................\n",
      "Episode 100 complete\n",
      "Learner:0\tReward total:11\tRunning mean: 0.2503\n",
      "Learner:1\tReward total:0\tRunning mean: 0.05007\n",
      "Learner:2\tReward total:0\tRunning mean: 0.06556\n",
      "Learner:3\tReward total:0\tRunning mean: 0.0\n",
      "Learner:4\tReward total:0\tRunning mean: 0.008772\n",
      "Learner:5\tReward total:0\tRunning mean: 0.0\n",
      "Learner:6\tReward total:0\tRunning mean: 0.229\n",
      "Learner:7\tReward total:0\tRunning mean: 0.2573\n",
      "Max Norms =  ['142.54', '0.00', '124.16', '0.00', '114.05', '111.61', '75.80', '0.00']\n",
      "....................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.6429\n",
      "Learner:1\tReward total:0\tRunning mean: 0.1067\n",
      "Learner:2\tReward total:0\tRunning mean: 0.2008\n",
      "Learner:3\tReward total:0\tRunning mean: 0.1829\n",
      "Learner:4\tReward total:0\tRunning mean: 0.2425\n",
      "Learner:5\tReward total:0\tRunning mean: 0.1895\n",
      "Learner:6\tReward total:0\tRunning mean: 0.5917\n",
      "Learner:7\tReward total:4\tRunning mean: 1.119\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '139.19', '0.00', '0.00', '0.00', '135.88']\n",
      "....................................................................................................\n",
      "Episode 300 complete\n",
      "Learner:0\tReward total:8\tRunning mean: 2.145\n",
      "Learner:1\tReward total:0\tRunning mean: 0.3823\n",
      "Learner:2\tReward total:14\tRunning mean: 3.176\n",
      "Learner:3\tReward total:0\tRunning mean: 0.1086\n",
      "Learner:4\tReward total:1\tRunning mean: 0.4714\n",
      "Learner:5\tReward total:3\tRunning mean: 1.422\n",
      "Learner:6\tReward total:8\tRunning mean: 3.495\n",
      "Learner:7\tReward total:15\tRunning mean: 8.327\n",
      "Max Norms =  ['165.30', '0.00', '145.61', '0.00', '167.41', '101.70', '102.06', '126.43']\n",
      "....................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:8\tRunning mean: 5.833\n",
      "Learner:1\tReward total:29\tRunning mean: 5.297\n",
      "Learner:2\tReward total:15\tRunning mean: 7.086\n",
      "Learner:3\tReward total:0\tRunning mean: 0.07072\n",
      "Learner:4\tReward total:0\tRunning mean: 2.026\n",
      "Learner:5\tReward total:10\tRunning mean: 2.908\n",
      "Learner:6\tReward total:2\tRunning mean: 6.672\n",
      "Learner:7\tReward total:41\tRunning mean: 23.65\n",
      "Max Norms =  ['109.49', '104.83', '118.69', '0.00', '0.00', '72.61', '97.86', '325.17']\n",
      "....................................................................................................\n",
      "Episode 500 complete\n",
      "Learner:0\tReward total:37\tRunning mean: 12.33\n",
      "Learner:1\tReward total:8\tRunning mean: 9.087\n",
      "Learner:2\tReward total:13\tRunning mean: 8.89\n",
      "Learner:3\tReward total:0\tRunning mean: 0.2448\n",
      "Learner:4\tReward total:1\tRunning mean: 4.912\n",
      "Learner:5\tReward total:18\tRunning mean: 5.549\n",
      "Learner:6\tReward total:27\tRunning mean: 12.88\n",
      "Learner:7\tReward total:15\tRunning mean: 34.95\n",
      "Max Norms =  ['143.59', '99.25', '105.53', '0.00', '181.30', '75.00', '153.66', '123.05']\n",
      "....................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:25\tRunning mean: 18.18\n",
      "Learner:1\tReward total:27\tRunning mean: 14.14\n",
      "Learner:2\tReward total:14\tRunning mean: 10.15\n",
      "Learner:3\tReward total:1\tRunning mean: 0.2018\n",
      "Learner:4\tReward total:12\tRunning mean: 8.025\n",
      "Learner:5\tReward total:12\tRunning mean: 9.665\n",
      "Learner:6\tReward total:13\tRunning mean: 16.91\n",
      "Learner:7\tReward total:57\tRunning mean: 36.55\n",
      "Max Norms =  ['124.58', '80.34', '120.71', '141.47', '125.68', '183.76', '73.93', '155.04']\n",
      "....................................................................................................\n",
      "Episode 700 complete\n",
      "Learner:0\tReward total:8\tRunning mean: 20.67\n",
      "Learner:1\tReward total:15\tRunning mean: 14.26\n",
      "Learner:2\tReward total:5\tRunning mean: 11.84\n",
      "Learner:3\tReward total:0\tRunning mean: 0.4744\n",
      "Learner:4\tReward total:12\tRunning mean: 11.84\n",
      "Learner:5\tReward total:30\tRunning mean: 14.89\n",
      "Learner:6\tReward total:16\tRunning mean: 20.28\n",
      "Learner:7\tReward total:40\tRunning mean: 36.41\n",
      "Max Norms =  ['124.90', '82.96', '156.55', '0.00', '141.95', '131.91', '65.88', '213.34']\n",
      "....................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:14\tRunning mean: 20.12\n",
      "Learner:1\tReward total:5\tRunning mean: 14.62\n",
      "Learner:2\tReward total:14\tRunning mean: 14.36\n",
      "Learner:3\tReward total:0\tRunning mean: 0.5253\n",
      "Learner:4\tReward total:3\tRunning mean: 12.32\n",
      "Learner:5\tReward total:22\tRunning mean: 18.77\n",
      "Learner:6\tReward total:6\tRunning mean: 22.41\n",
      "Learner:7\tReward total:39\tRunning mean: 35.17\n",
      "Max Norms =  ['172.54', '142.21', '150.32', '0.00', '156.79', '92.86', '61.24', '134.47']\n",
      "....................................................................................................\n",
      "Episode 900 complete\n",
      "Learner:0\tReward total:17\tRunning mean: 20.21\n",
      "Learner:1\tReward total:14\tRunning mean: 15.47\n",
      "Learner:2\tReward total:24\tRunning mean: 16.28\n",
      "Learner:3\tReward total:0\tRunning mean: 1.542\n",
      "Learner:4\tReward total:22\tRunning mean: 12.43\n",
      "Learner:5\tReward total:14\tRunning mean: 18.48\n",
      "Learner:6\tReward total:37\tRunning mean: 23.16\n",
      "Learner:7\tReward total:41\tRunning mean: 35.34\n",
      "Max Norms =  ['96.91', '78.43', '133.68', '0.00', '106.83', '76.40', '71.61', '172.12']\n",
      "....................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:2\tRunning mean: 18.76\n",
      "Learner:1\tReward total:15\tRunning mean: 15.05\n",
      "Learner:2\tReward total:1\tRunning mean: 15.39\n",
      "Learner:3\tReward total:25\tRunning mean: 8.815\n",
      "Learner:4\tReward total:2\tRunning mean: 13.55\n",
      "Learner:5\tReward total:22\tRunning mean: 19.28\n",
      "Learner:6\tReward total:22\tRunning mean: 22.83\n",
      "Learner:7\tReward total:41\tRunning mean: 35.01\n",
      "Max Norms =  ['109.88', '137.05', '120.12', '76.02', '169.66', '76.44', '59.89', '153.56']\n",
      "....................................................................................................\n",
      "Episode 1100 complete\n",
      "Learner:0\tReward total:13\tRunning mean: 18.01\n",
      "Learner:1\tReward total:5\tRunning mean: 15.04\n",
      "Learner:2\tReward total:18\tRunning mean: 13.81\n",
      "Learner:3\tReward total:21\tRunning mean: 12.69\n",
      "Learner:4\tReward total:12\tRunning mean: 17.03\n",
      "Learner:5\tReward total:15\tRunning mean: 14.36\n",
      "Learner:6\tReward total:19\tRunning mean: 23.95\n",
      "Learner:7\tReward total:40\tRunning mean: 34.81\n",
      "Max Norms =  ['91.23', '102.99', '147.36', '95.99', '107.33', '92.73', '93.80', '188.08']\n",
      "....................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:15\tRunning mean: 18.04\n",
      "Learner:1\tReward total:14\tRunning mean: 14.54\n",
      "Learner:2\tReward total:23\tRunning mean: 15.09\n",
      "Learner:3\tReward total:12\tRunning mean: 13.82\n",
      "Learner:4\tReward total:28\tRunning mean: 15.48\n",
      "Learner:5\tReward total:10\tRunning mean: 16.25\n",
      "Learner:6\tReward total:23\tRunning mean: 24.88\n",
      "Learner:7\tReward total:31\tRunning mean: 30.49\n",
      "Max Norms =  ['93.54', '64.30', '130.47', '76.13', '139.57', '128.14', '34.28', '169.60']\n",
      "....................................................................................................\n",
      "Episode 1300 complete\n",
      "Learner:0\tReward total:22\tRunning mean: 17.85\n",
      "Learner:1\tReward total:27\tRunning mean: 15.05\n",
      "Learner:2\tReward total:22\tRunning mean: 15.39\n",
      "Learner:3\tReward total:16\tRunning mean: 15.43\n",
      "Learner:4\tReward total:11\tRunning mean: 16.44\n",
      "Learner:5\tReward total:22\tRunning mean: 15.73\n",
      "Learner:6\tReward total:31\tRunning mean: 26.35\n",
      "Learner:7\tReward total:23\tRunning mean: 30.13\n",
      "Max Norms =  ['90.89', '106.70', '112.52', '74.03', '109.08', '159.41', '75.31', '323.44']\n",
      "....................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:23\tRunning mean: 18.78\n",
      "Learner:1\tReward total:17\tRunning mean: 13.81\n",
      "Learner:2\tReward total:21\tRunning mean: 15.14\n",
      "Learner:3\tReward total:16\tRunning mean: 15.0\n",
      "Learner:4\tReward total:23\tRunning mean: 16.01\n",
      "Learner:5\tReward total:19\tRunning mean: 18.61\n",
      "Learner:6\tReward total:7\tRunning mean: 26.94\n",
      "Learner:7\tReward total:39\tRunning mean: 30.77\n",
      "Max Norms =  ['179.15', '66.13', '149.51', '92.39', '184.46', '469.12', '18.11', '133.86']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "....................................................................................................\n",
      "Episode 1500 complete\n",
      "Learner:0\tReward total:33\tRunning mean: 17.2\n",
      "Learner:1\tReward total:10\tRunning mean: 14.83\n",
      "Learner:2\tReward total:11\tRunning mean: 16.8\n",
      "Learner:3\tReward total:10\tRunning mean: 14.83\n",
      "Learner:4\tReward total:6\tRunning mean: 15.9\n",
      "Learner:5\tReward total:13\tRunning mean: 18.25\n",
      "Learner:6\tReward total:24\tRunning mean: 26.1\n",
      "Learner:7\tReward total:52\tRunning mean: 31.96\n",
      "Max Norms =  ['160.27', '42.69', '80.03', '75.40', '162.99', '161.64', '102.68', '148.88']\n",
      "....................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:10\tRunning mean: 16.69\n",
      "Learner:1\tReward total:15\tRunning mean: 14.46\n",
      "Learner:2\tReward total:29\tRunning mean: 17.74\n",
      "Learner:3\tReward total:14\tRunning mean: 15.36\n",
      "Learner:4\tReward total:10\tRunning mean: 14.74\n",
      "Learner:5\tReward total:13\tRunning mean: 17.08\n",
      "Learner:6\tReward total:27\tRunning mean: 24.79\n",
      "Learner:7\tReward total:35\tRunning mean: 34.57\n",
      "Max Norms =  ['112.92', '107.36', '63.50', '7.30', '180.39', '134.78', '38.55', '161.84']\n",
      "....................................................................................................\n",
      "Episode 1700 complete\n",
      "Learner:0\tReward total:23\tRunning mean: 17.09\n",
      "Learner:1\tReward total:18\tRunning mean: 14.24\n",
      "Learner:2\tReward total:15\tRunning mean: 17.84\n",
      "Learner:3\tReward total:19\tRunning mean: 15.57\n",
      "Learner:4\tReward total:18\tRunning mean: 16.47\n",
      "Learner:5\tReward total:9\tRunning mean: 17.57\n",
      "Learner:6\tReward total:26\tRunning mean: 25.63\n",
      "Learner:7\tReward total:50\tRunning mean: 35.1\n",
      "Max Norms =  ['218.47', '70.31', '125.68', '4.78', '164.58', '294.88', '41.31', '230.06']\n",
      "....................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:20\tRunning mean: 18.13\n",
      "Learner:1\tReward total:11\tRunning mean: 12.76\n",
      "Learner:2\tReward total:8\tRunning mean: 19.49\n",
      "Learner:3\tReward total:7\tRunning mean: 15.31\n",
      "Learner:4\tReward total:25\tRunning mean: 16.22\n",
      "Learner:5\tReward total:12\tRunning mean: 16.63\n",
      "Learner:6\tReward total:22\tRunning mean: 26.06\n",
      "Learner:7\tReward total:7\tRunning mean: 28.28\n",
      "Max Norms =  ['131.24', '62.02', '98.71', '2.19', '147.82', '209.02', '49.18', '149.80']\n",
      "....................................................................................................\n",
      "Episode 1900 complete\n",
      "Learner:0\tReward total:26\tRunning mean: 18.3\n",
      "Learner:1\tReward total:21\tRunning mean: 13.65\n",
      "Learner:2\tReward total:22\tRunning mean: 20.14\n",
      "Learner:3\tReward total:17\tRunning mean: 15.64\n",
      "Learner:4\tReward total:12\tRunning mean: 16.68\n",
      "Learner:5\tReward total:16\tRunning mean: 16.56\n",
      "Learner:6\tReward total:15\tRunning mean: 25.86\n",
      "Learner:7\tReward total:54\tRunning mean: 27.86\n",
      "Max Norms =  ['185.44', '45.60', '161.52', '35.48', '169.70', '47.59', '62.38', '124.57']\n",
      "....................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:24\tRunning mean: 17.94\n",
      "Learner:1\tReward total:20\tRunning mean: 13.3\n",
      "Learner:2\tReward total:13\tRunning mean: 19.81\n",
      "Learner:3\tReward total:18\tRunning mean: 16.9\n",
      "Learner:4\tReward total:23\tRunning mean: 19.28\n",
      "Learner:5\tReward total:18\tRunning mean: 15.3\n",
      "Learner:6\tReward total:27\tRunning mean: 26.33\n",
      "Learner:7\tReward total:36\tRunning mean: 26.47\n",
      "Max Norms =  ['188.83', '48.32', '101.12', '20.41', '129.21', '251.09', '42.22', '183.55']\n",
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Load random agent 8\n",
      "....................................................................................................\n",
      "Episode 100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:0\tRunning mean: 0.03537\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.04173\n",
      "Learner:4\tReward total:0\tRunning mean: 0.07786\n",
      "Learner:5\tReward total:2\tRunning mean: 0.2154\n",
      "Learner:6\tReward total:0\tRunning mean: 0.4947\n",
      "Learner:7\tReward total:5\tRunning mean: 0.5075\n",
      "Max Norms =  ['0.00', '0.00', '108.98', '125.02', '140.78', '80.65', '0.00', '96.89']\n",
      "....................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.2512\n",
      "Learner:1\tReward total:0\tRunning mean: 0.2431\n",
      "Learner:2\tReward total:0\tRunning mean: 0.1168\n",
      "Learner:3\tReward total:4\tRunning mean: 1.135\n",
      "Learner:4\tReward total:0\tRunning mean: 0.5474\n",
      "Learner:5\tReward total:0\tRunning mean: 1.026\n",
      "Learner:6\tReward total:9\tRunning mean: 2.247\n",
      "Learner:7\tReward total:9\tRunning mean: 4.509\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '145.21', '0.00', '0.00', '169.71', '97.09']\n",
      "....................................................................................................\n",
      "Episode 300 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.1516\n",
      "Learner:1\tReward total:0\tRunning mean: 0.8585\n",
      "Learner:2\tReward total:0\tRunning mean: 0.06303\n",
      "Learner:3\tReward total:9\tRunning mean: 5.193\n",
      "Learner:4\tReward total:0\tRunning mean: 1.672\n",
      "Learner:5\tReward total:18\tRunning mean: 2.549\n",
      "Learner:6\tReward total:20\tRunning mean: 9.246\n",
      "Learner:7\tReward total:46\tRunning mean: 15.63\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '175.89', '0.00', '103.29', '112.19', '86.63']\n",
      "....................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.09743\n",
      "Learner:1\tReward total:2\tRunning mean: 3.502\n",
      "Learner:2\tReward total:0\tRunning mean: 0.03202\n",
      "Learner:3\tReward total:10\tRunning mean: 8.383\n",
      "Learner:4\tReward total:11\tRunning mean: 2.979\n",
      "Learner:5\tReward total:25\tRunning mean: 7.352\n",
      "Learner:6\tReward total:34\tRunning mean: 16.3\n",
      "Learner:7\tReward total:37\tRunning mean: 25.95\n",
      "Max Norms =  ['0.00', '83.88', '0.00', '120.05', '116.41', '117.29', '136.35', '125.95']\n",
      "....................................................................................................\n",
      "Episode 500 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.8252\n",
      "Learner:1\tReward total:1\tRunning mean: 8.501\n",
      "Learner:2\tReward total:10\tRunning mean: 0.3321\n",
      "Learner:3\tReward total:11\tRunning mean: 10.55\n",
      "Learner:4\tReward total:3\tRunning mean: 6.543\n",
      "Learner:5\tReward total:10\tRunning mean: 12.44\n",
      "Learner:6\tReward total:29\tRunning mean: 19.71\n",
      "Learner:7\tReward total:39\tRunning mean: 33.79\n",
      "Max Norms =  ['0.00', '125.44', '138.34', '122.11', '96.72', '72.74', '108.95', '81.01']\n",
      "....................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:6\tRunning mean: 4.182\n",
      "Learner:1\tReward total:16\tRunning mean: 11.45\n",
      "Learner:2\tReward total:0\tRunning mean: 0.8316\n",
      "Learner:3\tReward total:16\tRunning mean: 11.7\n",
      "Learner:4\tReward total:10\tRunning mean: 11.89\n",
      "Learner:5\tReward total:0\tRunning mean: 15.2\n",
      "Learner:6\tReward total:39\tRunning mean: 23.88\n",
      "Learner:7\tReward total:44\tRunning mean: 33.44\n",
      "Max Norms =  ['134.24', '122.75', '0.00', '90.50', '144.85', '0.00', '240.61', '93.10']\n",
      "....................................................................................................\n",
      "Episode 700 complete\n",
      "Learner:0\tReward total:19\tRunning mean: 8.784\n",
      "Learner:1\tReward total:16\tRunning mean: 12.24\n",
      "Learner:2\tReward total:16\tRunning mean: 7.189\n",
      "Learner:3\tReward total:19\tRunning mean: 13.42\n",
      "Learner:4\tReward total:28\tRunning mean: 14.19\n",
      "Learner:5\tReward total:20\tRunning mean: 15.32\n",
      "Learner:6\tReward total:14\tRunning mean: 25.49\n",
      "Learner:7\tReward total:25\tRunning mean: 30.32\n",
      "Max Norms =  ['86.33', '162.51', '75.97', '109.88', '121.39', '107.36', '150.86', '76.71']\n",
      "....................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:8\tRunning mean: 10.62\n",
      "Learner:1\tReward total:19\tRunning mean: 13.97\n",
      "Learner:2\tReward total:20\tRunning mean: 13.47\n",
      "Learner:3\tReward total:19\tRunning mean: 14.0\n",
      "Learner:4\tReward total:17\tRunning mean: 15.85\n",
      "Learner:5\tReward total:14\tRunning mean: 17.26\n",
      "Learner:6\tReward total:22\tRunning mean: 22.97\n",
      "Learner:7\tReward total:32\tRunning mean: 29.19\n",
      "Max Norms =  ['117.17', '166.22', '125.20', '66.63', '105.08', '91.17', '141.03', '101.20']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "....................................................................................................\n",
      "Episode 900 complete\n",
      "Learner:0\tReward total:14\tRunning mean: 12.9\n",
      "Learner:1\tReward total:17\tRunning mean: 15.09\n",
      "Learner:2\tReward total:37\tRunning mean: 16.21\n",
      "Learner:3\tReward total:18\tRunning mean: 13.91\n",
      "Learner:4\tReward total:12\tRunning mean: 16.36\n",
      "Learner:5\tReward total:16\tRunning mean: 18.23\n",
      "Learner:6\tReward total:24\tRunning mean: 25.15\n",
      "Learner:7\tReward total:21\tRunning mean: 25.68\n",
      "Max Norms =  ['138.87', '112.21', '109.53', '48.82', '106.72', '58.84', '158.18', '49.68']\n",
      "....................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:17\tRunning mean: 13.52\n",
      "Learner:1\tReward total:2\tRunning mean: 16.06\n",
      "Learner:2\tReward total:20\tRunning mean: 19.45\n",
      "Learner:3\tReward total:2\tRunning mean: 13.31\n",
      "Learner:4\tReward total:18\tRunning mean: 15.73\n",
      "Learner:5\tReward total:45\tRunning mean: 19.75\n",
      "Learner:6\tReward total:17\tRunning mean: 22.61\n",
      "Learner:7\tReward total:23\tRunning mean: 25.83\n",
      "Max Norms =  ['93.78', '150.41', '127.58', '109.65', '107.18', '78.41', '219.70', '85.23']\n",
      "....................................................................................................\n",
      "Episode 1100 complete\n",
      "Learner:0\tReward total:11\tRunning mean: 13.64\n",
      "Learner:1\tReward total:24\tRunning mean: 15.84\n",
      "Learner:2\tReward total:11\tRunning mean: 19.52\n",
      "Learner:3\tReward total:18\tRunning mean: 13.21\n",
      "Learner:4\tReward total:15\tRunning mean: 15.18\n",
      "Learner:5\tReward total:23\tRunning mean: 20.66\n",
      "Learner:6\tReward total:33\tRunning mean: 24.63\n",
      "Learner:7\tReward total:15\tRunning mean: 25.12\n",
      "Max Norms =  ['105.42', '119.84', '101.88', '25.12', '162.83', '87.79', '153.65', '74.86']\n",
      "....................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:18\tRunning mean: 14.0\n",
      "Learner:1\tReward total:17\tRunning mean: 16.09\n",
      "Learner:2\tReward total:20\tRunning mean: 22.39\n",
      "Learner:3\tReward total:14\tRunning mean: 12.96\n",
      "Learner:4\tReward total:24\tRunning mean: 15.74\n",
      "Learner:5\tReward total:21\tRunning mean: 19.88\n",
      "Learner:6\tReward total:35\tRunning mean: 25.32\n",
      "Learner:7\tReward total:19\tRunning mean: 22.7\n",
      "Max Norms =  ['152.58', '107.20', '178.81', '48.29', '108.72', '67.59', '130.25', '71.44']\n",
      "....................................................................................................\n",
      "Episode 1300 complete\n",
      "Learner:0\tReward total:14\tRunning mean: 15.85\n",
      "Learner:1\tReward total:9\tRunning mean: 17.68\n",
      "Learner:2\tReward total:12\tRunning mean: 19.85\n",
      "Learner:3\tReward total:12\tRunning mean: 13.51\n",
      "Learner:4\tReward total:21\tRunning mean: 15.71\n",
      "Learner:5\tReward total:21\tRunning mean: 21.79\n",
      "Learner:6\tReward total:26\tRunning mean: 23.73\n",
      "Learner:7\tReward total:37\tRunning mean: 24.21\n",
      "Max Norms =  ['166.24', '110.06', '81.26', '38.39', '142.10', '37.98', '192.53', '229.80']\n",
      "....................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:22\tRunning mean: 16.5\n",
      "Learner:1\tReward total:19\tRunning mean: 17.1\n",
      "Learner:2\tReward total:36\tRunning mean: 22.3\n",
      "Learner:3\tReward total:8\tRunning mean: 13.19\n",
      "Learner:4\tReward total:26\tRunning mean: 16.7\n",
      "Learner:5\tReward total:30\tRunning mean: 23.64\n",
      "Learner:6\tReward total:22\tRunning mean: 22.4\n",
      "Learner:7\tReward total:19\tRunning mean: 20.86\n",
      "Max Norms =  ['185.81', '211.17', '238.62', '56.06', '158.61', '133.17', '121.61', '55.98']\n",
      "....................................................................................................\n",
      "Episode 1500 complete\n",
      "Learner:0\tReward total:41\tRunning mean: 19.44\n",
      "Learner:1\tReward total:13\tRunning mean: 18.14\n",
      "Learner:2\tReward total:16\tRunning mean: 25.3\n",
      "Learner:3\tReward total:12\tRunning mean: 10.62\n",
      "Learner:4\tReward total:24\tRunning mean: 18.36\n",
      "Learner:5\tReward total:17\tRunning mean: 17.68\n",
      "Learner:6\tReward total:29\tRunning mean: 22.4\n",
      "Learner:7\tReward total:32\tRunning mean: 24.16\n",
      "Max Norms =  ['124.07', '129.71', '223.19', '86.57', '70.59', '87.95', '206.45', '144.26']\n",
      "....................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:8\tRunning mean: 18.29\n",
      "Learner:1\tReward total:4\tRunning mean: 17.29\n",
      "Learner:2\tReward total:12\tRunning mean: 25.45\n",
      "Learner:3\tReward total:8\tRunning mean: 10.44\n",
      "Learner:4\tReward total:26\tRunning mean: 16.85\n",
      "Learner:5\tReward total:11\tRunning mean: 15.68\n",
      "Learner:6\tReward total:28\tRunning mean: 24.8\n",
      "Learner:7\tReward total:27\tRunning mean: 24.27\n",
      "Max Norms =  ['61.33', '126.66', '62.40', '45.72', '63.39', '119.07', '209.24', '69.51']\n",
      "....................................................................................................\n",
      "Episode 1700 complete\n",
      "Learner:0\tReward total:20\tRunning mean: 17.95\n",
      "Learner:1\tReward total:15\tRunning mean: 19.12\n",
      "Learner:2\tReward total:9\tRunning mean: 25.62\n",
      "Learner:3\tReward total:13\tRunning mean: 10.79\n",
      "Learner:4\tReward total:4\tRunning mean: 16.85\n",
      "Learner:5\tReward total:9\tRunning mean: 17.77\n",
      "Learner:6\tReward total:40\tRunning mean: 25.97\n",
      "Learner:7\tReward total:18\tRunning mean: 23.56\n",
      "Max Norms =  ['229.95', '103.68', '179.77', '29.00', '59.70', '100.08', '178.43', '128.93']\n",
      "....................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:10\tRunning mean: 19.11\n",
      "Learner:1\tReward total:20\tRunning mean: 19.65\n",
      "Learner:2\tReward total:12\tRunning mean: 23.36\n",
      "Learner:3\tReward total:7\tRunning mean: 11.47\n",
      "Learner:4\tReward total:13\tRunning mean: 16.47\n",
      "Learner:5\tReward total:12\tRunning mean: 19.36\n",
      "Learner:6\tReward total:37\tRunning mean: 25.69\n",
      "Learner:7\tReward total:23\tRunning mean: 23.39\n",
      "Max Norms =  ['78.46', '123.53', '132.58', '36.71', '115.08', '74.70', '303.13', '104.84']\n",
      "....................................................................................................\n",
      "Episode 1900 complete\n",
      "Learner:0\tReward total:16\tRunning mean: 18.91\n",
      "Learner:1\tReward total:20\tRunning mean: 20.22\n",
      "Learner:2\tReward total:20\tRunning mean: 20.58\n",
      "Learner:3\tReward total:13\tRunning mean: 11.66\n",
      "Learner:4\tReward total:11\tRunning mean: 14.93\n",
      "Learner:5\tReward total:25\tRunning mean: 21.19\n",
      "Learner:6\tReward total:32\tRunning mean: 28.01\n",
      "Learner:7\tReward total:27\tRunning mean: 21.91\n",
      "Max Norms =  ['270.70', '136.44', '133.43', '37.09', '110.40', '94.26', '151.83', '110.31']\n",
      "....................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:38\tRunning mean: 18.68\n",
      "Learner:1\tReward total:26\tRunning mean: 21.4\n",
      "Learner:2\tReward total:16\tRunning mean: 21.32\n",
      "Learner:3\tReward total:16\tRunning mean: 11.98\n",
      "Learner:4\tReward total:5\tRunning mean: 14.88\n",
      "Learner:5\tReward total:25\tRunning mean: 22.77\n",
      "Learner:6\tReward total:32\tRunning mean: 27.5\n",
      "Learner:7\tReward total:30\tRunning mean: 23.44\n",
      "Max Norms =  ['175.32', '154.00', '150.92', '81.46', '111.09', '254.87', '252.85', '167.29']\n",
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Load random agent 8\n",
      "....................................................................................................\n",
      "Episode 100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.0\n",
      "Learner:1\tReward total:1\tRunning mean: 0.09007\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.1486\n",
      "Learner:4\tReward total:0\tRunning mean: 2.718\n",
      "Learner:5\tReward total:0\tRunning mean: 0.04082\n",
      "Learner:6\tReward total:0\tRunning mean: 0.1192\n",
      "Learner:7\tReward total:0\tRunning mean: 0.4696\n",
      "Max Norms =  ['98.33', '117.07', '0.00', '160.17', '0.00', '0.00', '0.00', '0.00']\n",
      "....................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.02619\n",
      "Learner:1\tReward total:0\tRunning mean: 0.08306\n",
      "Learner:2\tReward total:0\tRunning mean: 0.07335\n",
      "Learner:3\tReward total:0\tRunning mean: 0.3714\n",
      "Learner:4\tReward total:0\tRunning mean: 2.152\n",
      "Learner:5\tReward total:0\tRunning mean: 0.01494\n",
      "Learner:6\tReward total:0\tRunning mean: 0.1887\n",
      "Learner:7\tReward total:0\tRunning mean: 0.5348\n",
      "Max Norms =  ['0.00', '88.32', '0.00', '0.00', '86.84', '0.00', '0.00', '0.00']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "....................................................................................................\n",
      "Episode 300 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.07774\n",
      "Learner:1\tReward total:0\tRunning mean: 0.8713\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0892\n",
      "Learner:3\tReward total:14\tRunning mean: 2.648\n",
      "Learner:4\tReward total:0\tRunning mean: 4.152\n",
      "Learner:5\tReward total:0\tRunning mean: 0.005468\n",
      "Learner:6\tReward total:8\tRunning mean: 1.16\n",
      "Learner:7\tReward total:41\tRunning mean: 3.79\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '134.12', '0.00', '0.00', '87.29', '106.07']\n",
      "....................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.02846\n",
      "Learner:1\tReward total:9\tRunning mean: 2.389\n",
      "Learner:2\tReward total:0\tRunning mean: 0.03265\n",
      "Learner:3\tReward total:31\tRunning mean: 5.863\n",
      "Learner:4\tReward total:13\tRunning mean: 7.353\n",
      "Learner:5\tReward total:0\tRunning mean: 0.002002\n",
      "Learner:6\tReward total:20\tRunning mean: 6.023\n",
      "Learner:7\tReward total:23\tRunning mean: 25.05\n",
      "Max Norms =  ['0.00', '126.30', '0.00', '111.34', '104.68', '0.00', '104.03', '130.75']\n",
      "....................................................................................................\n",
      "Episode 500 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.01902\n",
      "Learner:1\tReward total:19\tRunning mean: 6.596\n",
      "Learner:2\tReward total:0\tRunning mean: 0.02788\n",
      "Learner:3\tReward total:18\tRunning mean: 16.53\n",
      "Learner:4\tReward total:30\tRunning mean: 12.11\n",
      "Learner:5\tReward total:0\tRunning mean: 0.0007327\n",
      "Learner:6\tReward total:11\tRunning mean: 14.01\n",
      "Learner:7\tReward total:43\tRunning mean: 37.52\n",
      "Max Norms =  ['0.00', '82.69', '0.00', '178.64', '124.30', '0.00', '148.99', '163.49']\n",
      "....................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.3287\n",
      "Learner:1\tReward total:7\tRunning mean: 10.41\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0102\n",
      "Learner:3\tReward total:13\tRunning mean: 21.27\n",
      "Learner:4\tReward total:21\tRunning mean: 21.68\n",
      "Learner:5\tReward total:0\tRunning mean: 0.0002682\n",
      "Learner:6\tReward total:46\tRunning mean: 20.42\n",
      "Learner:7\tReward total:40\tRunning mean: 40.82\n",
      "Max Norms =  ['0.00', '32.07', '0.00', '121.90', '122.96', '0.00', '131.76', '151.62']\n",
      "....................................................................................................\n",
      "Episode 700 complete\n",
      "Learner:0\tReward total:2\tRunning mean: 1.191\n",
      "Learner:1\tReward total:8\tRunning mean: 11.78\n",
      "Learner:2\tReward total:0\tRunning mean: 0.003735\n",
      "Learner:3\tReward total:28\tRunning mean: 22.48\n",
      "Learner:4\tReward total:32\tRunning mean: 25.23\n",
      "Learner:5\tReward total:0\tRunning mean: 9.816e-05\n",
      "Learner:6\tReward total:11\tRunning mean: 22.35\n",
      "Learner:7\tReward total:58\tRunning mean: 45.04\n",
      "Max Norms =  ['116.91', '74.82', '0.00', '160.91', '116.91', '0.00', '116.26', '151.29']\n",
      "....................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:12\tRunning mean: 5.604\n",
      "Learner:1\tReward total:3\tRunning mean: 12.08\n",
      "Learner:2\tReward total:0\tRunning mean: 0.001367\n",
      "Learner:3\tReward total:3\tRunning mean: 23.64\n",
      "Learner:4\tReward total:20\tRunning mean: 27.73\n",
      "Learner:5\tReward total:0\tRunning mean: 3.593e-05\n",
      "Learner:6\tReward total:12\tRunning mean: 24.13\n",
      "Learner:7\tReward total:43\tRunning mean: 44.15\n",
      "Max Norms =  ['116.52', '47.96', '0.00', '152.29', '156.01', '0.00', '103.55', '103.70']\n",
      "....................................................................................................\n",
      "Episode 900 complete\n",
      "Learner:0\tReward total:9\tRunning mean: 8.775\n",
      "Learner:1\tReward total:10\tRunning mean: 12.11\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0005004\n",
      "Learner:3\tReward total:32\tRunning mean: 25.5\n",
      "Learner:4\tReward total:43\tRunning mean: 30.75\n",
      "Learner:5\tReward total:0\tRunning mean: 1.315e-05\n",
      "Learner:6\tReward total:22\tRunning mean: 22.49\n",
      "Learner:7\tReward total:30\tRunning mean: 44.68\n",
      "Max Norms =  ['73.16', '36.50', '0.00', '96.70', '132.41', '0.00', '69.93', '133.51']\n",
      "....................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:10\tRunning mean: 12.39\n",
      "Learner:1\tReward total:10\tRunning mean: 11.81\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0001832\n",
      "Learner:3\tReward total:36\tRunning mean: 29.15\n",
      "Learner:4\tReward total:17\tRunning mean: 27.67\n",
      "Learner:5\tReward total:0\tRunning mean: 4.814e-06\n",
      "Learner:6\tReward total:4\tRunning mean: 19.64\n",
      "Learner:7\tReward total:58\tRunning mean: 43.28\n",
      "Max Norms =  ['72.42', '21.31', '0.00', '112.49', '125.83', '0.00', '151.39', '116.48']\n",
      "....................................................................................................\n",
      "Episode 1100 complete\n",
      "Learner:0\tReward total:28\tRunning mean: 13.5\n",
      "Learner:1\tReward total:13\tRunning mean: 11.45\n",
      "Learner:2\tReward total:0\tRunning mean: 6.705e-05\n",
      "Learner:3\tReward total:39\tRunning mean: 29.57\n",
      "Learner:4\tReward total:36\tRunning mean: 28.26\n",
      "Learner:5\tReward total:0\tRunning mean: 1.762e-06\n",
      "Learner:6\tReward total:17\tRunning mean: 20.39\n",
      "Learner:7\tReward total:57\tRunning mean: 44.47\n",
      "Max Norms =  ['78.20', '0.99', '0.00', '144.38', '169.71', '0.00', '193.84', '117.54']\n",
      "....................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:30\tRunning mean: 12.39\n",
      "Learner:1\tReward total:18\tRunning mean: 11.81\n",
      "Learner:2\tReward total:0\tRunning mean: 2.454e-05\n",
      "Learner:3\tReward total:41\tRunning mean: 27.77\n",
      "Learner:4\tReward total:29\tRunning mean: 30.45\n",
      "Learner:5\tReward total:0\tRunning mean: 6.45e-07\n",
      "Learner:6\tReward total:21\tRunning mean: 23.33\n",
      "Learner:7\tReward total:43\tRunning mean: 41.2\n",
      "Max Norms =  ['53.60', '17.08', '0.00', '156.87', '185.93', '0.00', '74.52', '152.89']\n",
      "....................................................................................................\n",
      "Episode 1300 complete\n",
      "Learner:0\tReward total:10\tRunning mean: 13.72\n",
      "Learner:1\tReward total:11\tRunning mean: 11.19\n",
      "Learner:2\tReward total:0\tRunning mean: 8.983e-06\n",
      "Learner:3\tReward total:34\tRunning mean: 29.46\n",
      "Learner:4\tReward total:5\tRunning mean: 29.81\n",
      "Learner:5\tReward total:0\tRunning mean: 2.361e-07\n",
      "Learner:6\tReward total:41\tRunning mean: 26.35\n",
      "Learner:7\tReward total:47\tRunning mean: 40.3\n",
      "Max Norms =  ['85.02', '10.96', '0.00', '109.03', '113.91', '0.00', '131.35', '102.19']\n",
      "....................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:7\tRunning mean: 13.09\n",
      "Learner:1\tReward total:15\tRunning mean: 11.78\n",
      "Learner:2\tReward total:0\tRunning mean: 3.288e-06\n",
      "Learner:3\tReward total:23\tRunning mean: 26.15\n",
      "Learner:4\tReward total:39\tRunning mean: 30.38\n",
      "Learner:5\tReward total:0\tRunning mean: 8.641e-08\n",
      "Learner:6\tReward total:27\tRunning mean: 27.7\n",
      "Learner:7\tReward total:37\tRunning mean: 43.51\n",
      "Max Norms =  ['57.24', '3.52', '0.00', '147.65', '122.69', '0.00', '215.45', '124.58']\n",
      "....................................................................................................\n",
      "Episode 1500 complete\n",
      "Learner:0\tReward total:3\tRunning mean: 12.71\n",
      "Learner:1\tReward total:5\tRunning mean: 11.29\n",
      "Learner:2\tReward total:0\tRunning mean: 1.204e-06\n",
      "Learner:3\tReward total:39\tRunning mean: 25.53\n",
      "Learner:4\tReward total:47\tRunning mean: 32.3\n",
      "Learner:5\tReward total:0\tRunning mean: 3.163e-08\n",
      "Learner:6\tReward total:20\tRunning mean: 27.23\n",
      "Learner:7\tReward total:58\tRunning mean: 43.06\n",
      "Max Norms =  ['62.49', '6.44', '0.00', '157.30', '183.48', '0.00', '131.42', '150.07']\n",
      "....................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:18\tRunning mean: 14.39\n",
      "Learner:1\tReward total:2\tRunning mean: 10.88\n",
      "Learner:2\tReward total:0\tRunning mean: 4.405e-07\n",
      "Learner:3\tReward total:24\tRunning mean: 27.75\n",
      "Learner:4\tReward total:44\tRunning mean: 33.49\n",
      "Learner:5\tReward total:0\tRunning mean: 1.158e-08\n",
      "Learner:6\tReward total:26\tRunning mean: 25.49\n",
      "Learner:7\tReward total:31\tRunning mean: 43.28\n",
      "Max Norms =  ['63.14', '8.01', '0.00', '109.60', '154.46', '0.00', '128.64', '167.70']\n",
      "....................................................................................................\n",
      "Episode 1700 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 9.402\n",
      "Learner:1\tReward total:6\tRunning mean: 10.57\n",
      "Learner:2\tReward total:0\tRunning mean: 1.613e-07\n",
      "Learner:3\tReward total:17\tRunning mean: 32.32\n",
      "Learner:4\tReward total:34\tRunning mean: 31.4\n",
      "Learner:5\tReward total:0\tRunning mean: 4.238e-09\n",
      "Learner:6\tReward total:29\tRunning mean: 26.43\n",
      "Learner:7\tReward total:55\tRunning mean: 44.46\n",
      "Max Norms =  ['0.00', '8.67', '0.00', '136.88', '229.29', '0.00', '68.79', '154.36']\n",
      "....................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:19\tRunning mean: 11.82\n",
      "Learner:1\tReward total:4\tRunning mean: 10.19\n",
      "Learner:2\tReward total:0\tRunning mean: 5.902e-08\n",
      "Learner:3\tReward total:32\tRunning mean: 31.61\n",
      "Learner:4\tReward total:21\tRunning mean: 26.0\n",
      "Learner:5\tReward total:0\tRunning mean: 1.551e-09\n",
      "Learner:6\tReward total:12\tRunning mean: 29.0\n",
      "Learner:7\tReward total:48\tRunning mean: 43.16\n",
      "Max Norms =  ['118.32', '8.20', '0.00', '143.89', '92.20', '0.00', '169.51', '93.74']\n",
      "....................................................................................................\n",
      "Episode 1900 complete\n",
      "Learner:0\tReward total:16\tRunning mean: 14.84\n",
      "Learner:1\tReward total:11\tRunning mean: 9.838\n",
      "Learner:2\tReward total:0\tRunning mean: 2.16e-08\n",
      "Learner:3\tReward total:42\tRunning mean: 31.43\n",
      "Learner:4\tReward total:47\tRunning mean: 28.02\n",
      "Learner:5\tReward total:0\tRunning mean: 5.678e-10\n",
      "Learner:6\tReward total:22\tRunning mean: 28.73\n",
      "Learner:7\tReward total:22\tRunning mean: 40.7\n",
      "Max Norms =  ['127.48', '0.06', '0.00', '146.64', '132.61', '0.00', '134.17', '213.02']\n",
      "....................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:17\tRunning mean: 15.61\n",
      "Learner:1\tReward total:10\tRunning mean: 9.594\n",
      "Learner:2\tReward total:0\tRunning mean: 7.908e-09\n",
      "Learner:3\tReward total:19\tRunning mean: 27.4\n",
      "Learner:4\tReward total:32\tRunning mean: 30.12\n",
      "Learner:5\tReward total:0\tRunning mean: 2.078e-10\n",
      "Learner:6\tReward total:20\tRunning mean: 27.3\n",
      "Learner:7\tReward total:45\tRunning mean: 40.45\n",
      "Max Norms =  ['222.51', '0.21', '0.00', '183.16', '148.17', '0.00', '128.19', '123.42']\n",
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Load random agent 8\n",
      "....................................................................................................\n",
      "Episode 100 complete\n",
      "Learner:0\tReward total:1\tRunning mean: 0.2472\n",
      "Learner:1\tReward total:0\tRunning mean: 0.0\n",
      "Learner:2\tReward total:0\tRunning mean: 0.2347\n",
      "Learner:3\tReward total:0\tRunning mean: 0.1092\n",
      "Learner:4\tReward total:0\tRunning mean: 0.1455\n",
      "Learner:5\tReward total:0\tRunning mean: 0.08427\n",
      "Learner:6\tReward total:0\tRunning mean: 0.168\n",
      "Learner:7\tReward total:20\tRunning mean: 0.8756\n",
      "Max Norms =  ['123.81', '0.00', '136.53', '0.00', '137.00', '0.00', '0.00', '117.95']\n",
      "....................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.4107\n",
      "Learner:1\tReward total:0\tRunning mean: 0.008601\n",
      "Learner:2\tReward total:1\tRunning mean: 0.1955\n",
      "Learner:3\tReward total:0\tRunning mean: 0.6945\n",
      "Learner:4\tReward total:9\tRunning mean: 1.331\n",
      "Learner:5\tReward total:7\tRunning mean: 0.4025\n",
      "Learner:6\tReward total:0\tRunning mean: 0.9502\n",
      "Learner:7\tReward total:14\tRunning mean: 2.042\n",
      "Max Norms =  ['0.00', '0.00', '93.80', '0.00', '117.85', '111.79', '0.00', '113.35']\n",
      "....................................................................................................\n",
      "Episode 300 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.002\n",
      "Learner:1\tReward total:0\tRunning mean: 0.4825\n",
      "Learner:2\tReward total:0\tRunning mean: 0.2782\n",
      "Learner:3\tReward total:12\tRunning mean: 2.3\n",
      "Learner:4\tReward total:29\tRunning mean: 7.259\n",
      "Learner:5\tReward total:3\tRunning mean: 2.042\n",
      "Learner:6\tReward total:1\tRunning mean: 4.054\n",
      "Learner:7\tReward total:11\tRunning mean: 6.483\n",
      "Max Norms =  ['0.00', '0.00', '0.00', '108.91', '107.67', '79.23', '127.36', '117.79']\n",
      "....................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:1\tRunning mean: 1.286\n",
      "Learner:1\tReward total:6\tRunning mean: 1.845\n",
      "Learner:2\tReward total:0\tRunning mean: 0.3525\n",
      "Learner:3\tReward total:10\tRunning mean: 6.973\n",
      "Learner:4\tReward total:27\tRunning mean: 17.6\n",
      "Learner:5\tReward total:5\tRunning mean: 5.52\n",
      "Learner:6\tReward total:12\tRunning mean: 8.16\n",
      "Learner:7\tReward total:12\tRunning mean: 18.05\n",
      "Max Norms =  ['95.75', '86.72', '0.00', '80.99', '92.61', '83.35', '105.51', '164.68']\n",
      "....................................................................................................\n",
      "Episode 500 complete\n",
      "Learner:0\tReward total:4\tRunning mean: 1.82\n",
      "Learner:1\tReward total:9\tRunning mean: 5.163\n",
      "Learner:2\tReward total:0\tRunning mean: 0.4355\n",
      "Learner:3\tReward total:11\tRunning mean: 8.635\n",
      "Learner:4\tReward total:30\tRunning mean: 25.13\n",
      "Learner:5\tReward total:6\tRunning mean: 9.697\n",
      "Learner:6\tReward total:20\tRunning mean: 20.89\n",
      "Learner:7\tReward total:21\tRunning mean: 22.55\n",
      "Max Norms =  ['116.01', '149.24', '0.00', '59.65', '186.60', '127.38', '100.12', '106.86']\n",
      "....................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:5\tRunning mean: 2.918\n",
      "Learner:1\tReward total:4\tRunning mean: 6.286\n",
      "Learner:2\tReward total:1\tRunning mean: 0.8546\n",
      "Learner:3\tReward total:10\tRunning mean: 10.74\n",
      "Learner:4\tReward total:21\tRunning mean: 28.0\n",
      "Learner:5\tReward total:19\tRunning mean: 13.55\n",
      "Learner:6\tReward total:16\tRunning mean: 29.22\n",
      "Learner:7\tReward total:20\tRunning mean: 26.18\n",
      "Max Norms =  ['120.78', '82.43', '158.42', '52.89', '163.31', '102.78', '76.69', '208.19']\n",
      "....................................................................................................\n",
      "Episode 700 complete\n",
      "Learner:0\tReward total:2\tRunning mean: 4.44\n",
      "Learner:1\tReward total:11\tRunning mean: 7.894\n",
      "Learner:2\tReward total:7\tRunning mean: 3.742\n",
      "Learner:3\tReward total:9\tRunning mean: 11.16\n",
      "Learner:4\tReward total:28\tRunning mean: 27.47\n",
      "Learner:5\tReward total:36\tRunning mean: 16.36\n",
      "Learner:6\tReward total:26\tRunning mean: 31.42\n",
      "Learner:7\tReward total:16\tRunning mean: 29.98\n",
      "Max Norms =  ['103.35', '97.05', '134.70', '31.02', '129.80', '123.51', '144.65', '159.33']\n",
      "....................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:5\tRunning mean: 6.001\n",
      "Learner:1\tReward total:5\tRunning mean: 9.014\n",
      "Learner:2\tReward total:7\tRunning mean: 5.607\n",
      "Learner:3\tReward total:10\tRunning mean: 10.58\n",
      "Learner:4\tReward total:20\tRunning mean: 27.27\n",
      "Learner:5\tReward total:15\tRunning mean: 15.84\n",
      "Learner:6\tReward total:19\tRunning mean: 31.46\n",
      "Learner:7\tReward total:46\tRunning mean: 32.0\n",
      "Max Norms =  ['107.31', '49.90', '73.26', '70.58', '112.29', '169.10', '120.84', '161.25']\n",
      "....................................................................................................\n",
      "Episode 900 complete\n",
      "Learner:0\tReward total:5\tRunning mean: 10.25\n",
      "Learner:1\tReward total:0\tRunning mean: 8.569\n",
      "Learner:2\tReward total:5\tRunning mean: 7.292\n",
      "Learner:3\tReward total:18\tRunning mean: 12.86\n",
      "Learner:4\tReward total:24\tRunning mean: 24.54\n",
      "Learner:5\tReward total:22\tRunning mean: 18.71\n",
      "Learner:6\tReward total:23\tRunning mean: 30.05\n",
      "Learner:7\tReward total:45\tRunning mean: 33.09\n",
      "Max Norms =  ['155.54', '0.00', '88.82', '151.86', '88.69', '145.52', '105.98', '99.77']\n",
      "....................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:4\tRunning mean: 13.31\n",
      "Learner:1\tReward total:3\tRunning mean: 10.81\n",
      "Learner:2\tReward total:11\tRunning mean: 7.301\n",
      "Learner:3\tReward total:10\tRunning mean: 13.53\n",
      "Learner:4\tReward total:13\tRunning mean: 23.49\n",
      "Learner:5\tReward total:31\tRunning mean: 19.71\n",
      "Learner:6\tReward total:32\tRunning mean: 27.83\n",
      "Learner:7\tReward total:30\tRunning mean: 32.88\n",
      "Max Norms =  ['108.28', '134.10', '112.84', '105.53', '81.84', '83.73', '104.68', '136.54']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "....................................................................................................\n",
      "Episode 1100 complete\n",
      "Learner:0\tReward total:12\tRunning mean: 16.74\n",
      "Learner:1\tReward total:14\tRunning mean: 14.0\n",
      "Learner:2\tReward total:11\tRunning mean: 7.253\n",
      "Learner:3\tReward total:7\tRunning mean: 13.6\n",
      "Learner:4\tReward total:27\tRunning mean: 21.49\n",
      "Learner:5\tReward total:21\tRunning mean: 20.59\n",
      "Learner:6\tReward total:33\tRunning mean: 27.25\n",
      "Learner:7\tReward total:16\tRunning mean: 32.57\n",
      "Max Norms =  ['124.23', '154.31', '64.00', '87.47', '124.18', '111.13', '71.70', '160.70']\n",
      "....................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:15\tRunning mean: 16.63\n",
      "Learner:1\tReward total:12\tRunning mean: 12.9\n",
      "Learner:2\tReward total:15\tRunning mean: 7.719\n",
      "Learner:3\tReward total:23\tRunning mean: 15.53\n",
      "Learner:4\tReward total:32\tRunning mean: 23.25\n",
      "Learner:5\tReward total:15\tRunning mean: 21.51\n",
      "Learner:6\tReward total:40\tRunning mean: 26.73\n",
      "Learner:7\tReward total:7\tRunning mean: 30.54\n",
      "Max Norms =  ['90.14', '106.66', '71.54', '109.04', '92.31', '148.52', '71.81', '123.56']\n",
      "....................................................................................................\n",
      "Episode 1300 complete\n",
      "Learner:0\tReward total:15\tRunning mean: 16.63\n",
      "Learner:1\tReward total:23\tRunning mean: 13.68\n",
      "Learner:2\tReward total:26\tRunning mean: 8.846\n",
      "Learner:3\tReward total:15\tRunning mean: 16.77\n",
      "Learner:4\tReward total:23\tRunning mean: 21.97\n",
      "Learner:5\tReward total:4\tRunning mean: 20.14\n",
      "Learner:6\tReward total:22\tRunning mean: 22.12\n",
      "Learner:7\tReward total:25\tRunning mean: 31.62\n",
      "Max Norms =  ['153.63', '134.76', '132.20', '93.89', '95.45', '82.74', '48.70', '111.12']\n",
      "....................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:14\tRunning mean: 17.99\n",
      "Learner:1\tReward total:6\tRunning mean: 14.1\n",
      "Learner:2\tReward total:10\tRunning mean: 9.782\n",
      "Learner:3\tReward total:15\tRunning mean: 19.6\n",
      "Learner:4\tReward total:31\tRunning mean: 21.81\n",
      "Learner:5\tReward total:21\tRunning mean: 20.2\n",
      "Learner:6\tReward total:23\tRunning mean: 19.69\n",
      "Learner:7\tReward total:29\tRunning mean: 31.98\n",
      "Max Norms =  ['219.14', '153.39', '120.29', '66.25', '125.42', '123.17', '47.60', '140.18']\n",
      "....................................................................................................\n",
      "Episode 1500 complete\n",
      "Learner:0\tReward total:6\tRunning mean: 17.32\n",
      "Learner:1\tReward total:21\tRunning mean: 14.67\n",
      "Learner:2\tReward total:5\tRunning mean: 10.77\n",
      "Learner:3\tReward total:14\tRunning mean: 19.58\n",
      "Learner:4\tReward total:20\tRunning mean: 19.14\n",
      "Learner:5\tReward total:14\tRunning mean: 20.34\n",
      "Learner:6\tReward total:28\tRunning mean: 20.84\n",
      "Learner:7\tReward total:75\tRunning mean: 34.03\n",
      "Max Norms =  ['199.87', '120.08', '132.76', '85.95', '88.01', '160.71', '206.15', '268.89']\n",
      "....................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:12\tRunning mean: 18.38\n",
      "Learner:1\tReward total:14\tRunning mean: 13.61\n",
      "Learner:2\tReward total:20\tRunning mean: 11.91\n",
      "Learner:3\tReward total:15\tRunning mean: 17.14\n",
      "Learner:4\tReward total:20\tRunning mean: 21.12\n",
      "Learner:5\tReward total:27\tRunning mean: 19.45\n",
      "Learner:6\tReward total:15\tRunning mean: 19.16\n",
      "Learner:7\tReward total:37\tRunning mean: 34.63\n",
      "Max Norms =  ['147.64', '107.55', '73.99', '182.12', '90.21', '117.76', '39.01', '133.66']\n",
      "....................................................................................................\n",
      "Episode 1700 complete\n",
      "Learner:0\tReward total:15\tRunning mean: 16.28\n",
      "Learner:1\tReward total:6\tRunning mean: 14.27\n",
      "Learner:2\tReward total:9\tRunning mean: 11.43\n",
      "Learner:3\tReward total:7\tRunning mean: 16.43\n",
      "Learner:4\tReward total:30\tRunning mean: 20.78\n",
      "Learner:5\tReward total:23\tRunning mean: 21.76\n",
      "Learner:6\tReward total:26\tRunning mean: 19.35\n",
      "Learner:7\tReward total:14\tRunning mean: 36.56\n",
      "Max Norms =  ['230.42', '82.21', '94.59', '103.69', '365.43', '173.53', '2.74', '138.11']\n",
      "....................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:15\tRunning mean: 17.77\n",
      "Learner:1\tReward total:19\tRunning mean: 14.43\n",
      "Learner:2\tReward total:15\tRunning mean: 12.65\n",
      "Learner:3\tReward total:16\tRunning mean: 18.2\n",
      "Learner:4\tReward total:27\tRunning mean: 19.64\n",
      "Learner:5\tReward total:23\tRunning mean: 20.67\n",
      "Learner:6\tReward total:21\tRunning mean: 18.88\n",
      "Learner:7\tReward total:36\tRunning mean: 35.36\n",
      "Max Norms =  ['103.45', '72.18', '137.92', '140.68', '99.67', '147.47', '30.25', '162.77']\n",
      "....................................................................................................\n",
      "Episode 1900 complete\n",
      "Learner:0\tReward total:14\tRunning mean: 17.14\n",
      "Learner:1\tReward total:14\tRunning mean: 14.5\n",
      "Learner:2\tReward total:9\tRunning mean: 10.46\n",
      "Learner:3\tReward total:27\tRunning mean: 18.14\n",
      "Learner:4\tReward total:33\tRunning mean: 22.62\n",
      "Learner:5\tReward total:16\tRunning mean: 21.08\n",
      "Learner:6\tReward total:10\tRunning mean: 18.78\n",
      "Learner:7\tReward total:39\tRunning mean: 33.7\n",
      "Max Norms =  ['147.47', '100.67', '102.25', '89.04', '156.67', '76.00', '29.68', '252.88']\n",
      "....................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:22\tRunning mean: 17.05\n",
      "Learner:1\tReward total:12\tRunning mean: 13.7\n",
      "Learner:2\tReward total:5\tRunning mean: 9.631\n",
      "Learner:3\tReward total:10\tRunning mean: 18.06\n",
      "Learner:4\tReward total:26\tRunning mean: 23.77\n",
      "Learner:5\tReward total:17\tRunning mean: 23.01\n",
      "Learner:6\tReward total:17\tRunning mean: 17.53\n",
      "Learner:7\tReward total:40\tRunning mean: 29.69\n",
      "Max Norms =  ['152.25', '188.71', '110.18', '239.91', '183.21', '83.95', '100.25', '192.64']\n",
      "Learner agent 0\n",
      "Learner agent 1\n",
      "Learner agent 2\n",
      "Learner agent 3\n",
      "Learner agent 4\n",
      "Learner agent 5\n",
      "Learner agent 6\n",
      "Learner agent 7\n",
      "Load random agent 8\n",
      "....................................................................................................\n",
      "Episode 100 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.3182\n",
      "Learner:1\tReward total:0\tRunning mean: 0.1115\n",
      "Learner:2\tReward total:0\tRunning mean: 0.0\n",
      "Learner:3\tReward total:0\tRunning mean: 0.007857\n",
      "Learner:4\tReward total:0\tRunning mean: 0.1139\n",
      "Learner:5\tReward total:0\tRunning mean: 0.01502\n",
      "Learner:6\tReward total:0\tRunning mean: 0.3166\n",
      "Learner:7\tReward total:0\tRunning mean: 0.1151\n",
      "Max Norms =  ['124.63', '177.20', '0.00', '134.61', '114.91', '114.25', '0.00', '0.00']\n",
      "....................................................................................................\n",
      "Episode 200 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 0.6794\n",
      "Learner:1\tReward total:3\tRunning mean: 0.5428\n",
      "Learner:2\tReward total:0\tRunning mean: 0.3528\n",
      "Learner:3\tReward total:0\tRunning mean: 0.1704\n",
      "Learner:4\tReward total:0\tRunning mean: 0.3473\n",
      "Learner:5\tReward total:5\tRunning mean: 0.4642\n",
      "Learner:6\tReward total:0\tRunning mean: 0.9771\n",
      "Learner:7\tReward total:0\tRunning mean: 1.496\n",
      "Max Norms =  ['0.00', '83.82', '0.00', '133.86', '0.00', '90.99', '0.00', '0.00']\n",
      "....................................................................................................\n",
      "Episode 300 complete\n",
      "Learner:0\tReward total:0\tRunning mean: 1.402\n",
      "Learner:1\tReward total:0\tRunning mean: 1.942\n",
      "Learner:2\tReward total:27\tRunning mean: 2.974\n",
      "Learner:3\tReward total:0\tRunning mean: 0.9951\n",
      "Learner:4\tReward total:0\tRunning mean: 1.088\n",
      "Learner:5\tReward total:0\tRunning mean: 2.453\n",
      "Learner:6\tReward total:8\tRunning mean: 2.802\n",
      "Learner:7\tReward total:0\tRunning mean: 5.598\n",
      "Max Norms =  ['0.00', '119.07', '126.04', '0.00', '117.44', '0.00', '112.10', '0.00']\n",
      "....................................................................................................\n",
      "Episode 400 complete\n",
      "Learner:0\tReward total:6\tRunning mean: 4.959\n",
      "Learner:1\tReward total:11\tRunning mean: 7.101\n",
      "Learner:2\tReward total:12\tRunning mean: 5.674\n",
      "Learner:3\tReward total:8\tRunning mean: 3.194\n",
      "Learner:4\tReward total:1\tRunning mean: 1.875\n",
      "Learner:5\tReward total:36\tRunning mean: 10.04\n",
      "Learner:6\tReward total:3\tRunning mean: 3.416\n",
      "Learner:7\tReward total:23\tRunning mean: 22.02\n",
      "Max Norms =  ['89.47', '87.92', '136.67', '123.41', '80.81', '148.63', '89.42', '139.00']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "....................................................................................................\n",
      "Episode 500 complete\n",
      "Learner:0\tReward total:13\tRunning mean: 10.58\n",
      "Learner:1\tReward total:16\tRunning mean: 13.82\n",
      "Learner:2\tReward total:14\tRunning mean: 8.66\n",
      "Learner:3\tReward total:19\tRunning mean: 9.603\n",
      "Learner:4\tReward total:2\tRunning mean: 4.775\n",
      "Learner:5\tReward total:24\tRunning mean: 16.38\n",
      "Learner:6\tReward total:25\tRunning mean: 9.652\n",
      "Learner:7\tReward total:24\tRunning mean: 26.44\n",
      "Max Norms =  ['130.53', '89.42', '162.15', '148.17', '85.43', '123.41', '94.61', '150.16']\n",
      "....................................................................................................\n",
      "Episode 600 complete\n",
      "Learner:0\tReward total:33\tRunning mean: 13.8\n",
      "Learner:1\tReward total:13\tRunning mean: 15.76\n",
      "Learner:2\tReward total:20\tRunning mean: 15.18\n",
      "Learner:3\tReward total:14\tRunning mean: 13.38\n",
      "Learner:4\tReward total:12\tRunning mean: 8.299\n",
      "Learner:5\tReward total:17\tRunning mean: 17.94\n",
      "Learner:6\tReward total:27\tRunning mean: 15.55\n",
      "Learner:7\tReward total:19\tRunning mean: 27.13\n",
      "Max Norms =  ['59.49', '135.19', '129.43', '115.22', '60.68', '51.61', '206.58', '103.16']\n",
      "....................................................................................................\n",
      "Episode 700 complete\n",
      "Learner:0\tReward total:23\tRunning mean: 15.11\n",
      "Learner:1\tReward total:10\tRunning mean: 16.34\n",
      "Learner:2\tReward total:18\tRunning mean: 18.95\n",
      "Learner:3\tReward total:9\tRunning mean: 17.51\n",
      "Learner:4\tReward total:23\tRunning mean: 10.96\n",
      "Learner:5\tReward total:12\tRunning mean: 18.09\n",
      "Learner:6\tReward total:20\tRunning mean: 21.24\n",
      "Learner:7\tReward total:26\tRunning mean: 26.92\n",
      "Max Norms =  ['85.31', '85.86', '147.99', '92.65', '68.34', '43.17', '107.87', '133.00']\n",
      "....................................................................................................\n",
      "Episode 800 complete\n",
      "Learner:0\tReward total:33\tRunning mean: 15.54\n",
      "Learner:1\tReward total:8\tRunning mean: 16.5\n",
      "Learner:2\tReward total:15\tRunning mean: 15.65\n",
      "Learner:3\tReward total:9\tRunning mean: 17.84\n",
      "Learner:4\tReward total:10\tRunning mean: 13.03\n",
      "Learner:5\tReward total:12\tRunning mean: 18.28\n",
      "Learner:6\tReward total:23\tRunning mean: 24.02\n",
      "Learner:7\tReward total:34\tRunning mean: 26.56\n",
      "Max Norms =  ['70.37', '87.38', '99.06', '85.68', '45.32', '101.95', '119.37', '156.59']\n",
      "....................................................................................................\n",
      "Episode 900 complete\n",
      "Learner:0\tReward total:17\tRunning mean: 15.71\n",
      "Learner:1\tReward total:12\tRunning mean: 15.42\n",
      "Learner:2\tReward total:13\tRunning mean: 18.15\n",
      "Learner:3\tReward total:27\tRunning mean: 18.31\n",
      "Learner:4\tReward total:14\tRunning mean: 13.62\n",
      "Learner:5\tReward total:6\tRunning mean: 16.58\n",
      "Learner:6\tReward total:26\tRunning mean: 24.39\n",
      "Learner:7\tReward total:36\tRunning mean: 26.13\n",
      "Max Norms =  ['41.64', '99.52', '164.98', '98.82', '57.47', '87.45', '88.10', '77.08']\n",
      "....................................................................................................\n",
      "Episode 1000 complete\n",
      "Learner:0\tReward total:32\tRunning mean: 16.74\n",
      "Learner:1\tReward total:16\tRunning mean: 15.75\n",
      "Learner:2\tReward total:11\tRunning mean: 18.4\n",
      "Learner:3\tReward total:32\tRunning mean: 19.8\n",
      "Learner:4\tReward total:8\tRunning mean: 14.04\n",
      "Learner:5\tReward total:15\tRunning mean: 16.74\n",
      "Learner:6\tReward total:29\tRunning mean: 24.52\n",
      "Learner:7\tReward total:21\tRunning mean: 26.94\n",
      "Max Norms =  ['91.66', '128.82', '132.75', '115.03', '37.65', '104.68', '137.11', '109.92']\n",
      "....................................................................................................\n",
      "Episode 1100 complete\n",
      "Learner:0\tReward total:10\tRunning mean: 16.8\n",
      "Learner:1\tReward total:15\tRunning mean: 16.15\n",
      "Learner:2\tReward total:30\tRunning mean: 20.03\n",
      "Learner:3\tReward total:23\tRunning mean: 20.87\n",
      "Learner:4\tReward total:13\tRunning mean: 14.39\n",
      "Learner:5\tReward total:26\tRunning mean: 16.41\n",
      "Learner:6\tReward total:27\tRunning mean: 25.27\n",
      "Learner:7\tReward total:32\tRunning mean: 27.63\n",
      "Max Norms =  ['73.56', '63.36', '119.13', '115.69', '21.25', '87.92', '163.44', '137.88']\n",
      "....................................................................................................\n",
      "Episode 1200 complete\n",
      "Learner:0\tReward total:29\tRunning mean: 18.01\n",
      "Learner:1\tReward total:16\tRunning mean: 16.49\n",
      "Learner:2\tReward total:17\tRunning mean: 21.36\n",
      "Learner:3\tReward total:12\tRunning mean: 21.04\n",
      "Learner:4\tReward total:12\tRunning mean: 13.74\n",
      "Learner:5\tReward total:10\tRunning mean: 15.84\n",
      "Learner:6\tReward total:17\tRunning mean: 24.51\n",
      "Learner:7\tReward total:19\tRunning mean: 28.77\n",
      "Max Norms =  ['72.51', '55.76', '110.22', '65.04', '27.87', '119.79', '103.45', '124.60']\n",
      "....................................................................................................\n",
      "Episode 1300 complete\n",
      "Learner:0\tReward total:20\tRunning mean: 18.68\n",
      "Learner:1\tReward total:18\tRunning mean: 17.43\n",
      "Learner:2\tReward total:30\tRunning mean: 22.61\n",
      "Learner:3\tReward total:13\tRunning mean: 18.82\n",
      "Learner:4\tReward total:12\tRunning mean: 13.86\n",
      "Learner:5\tReward total:18\tRunning mean: 16.73\n",
      "Learner:6\tReward total:22\tRunning mean: 24.04\n",
      "Learner:7\tReward total:35\tRunning mean: 30.01\n",
      "Max Norms =  ['87.10', '57.51', '149.08', '70.19', '21.93', '59.90', '105.65', '106.61']\n",
      "....................................................................................................\n",
      "Episode 1400 complete\n",
      "Learner:0\tReward total:12\tRunning mean: 17.46\n",
      "Learner:1\tReward total:12\tRunning mean: 17.55\n",
      "Learner:2\tReward total:20\tRunning mean: 23.86\n",
      "Learner:3\tReward total:8\tRunning mean: 17.28\n",
      "Learner:4\tReward total:4\tRunning mean: 12.82\n",
      "Learner:5\tReward total:24\tRunning mean: 18.01\n",
      "Learner:6\tReward total:25\tRunning mean: 23.18\n",
      "Learner:7\tReward total:28\tRunning mean: 30.52\n",
      "Max Norms =  ['74.82', '81.64', '123.34', '114.64', '154.32', '24.11', '79.75', '144.77']\n",
      "....................................................................................................\n",
      "Episode 1500 complete\n",
      "Learner:0\tReward total:16\tRunning mean: 17.5\n",
      "Learner:1\tReward total:17\tRunning mean: 16.74\n",
      "Learner:2\tReward total:29\tRunning mean: 23.39\n",
      "Learner:3\tReward total:19\tRunning mean: 18.78\n",
      "Learner:4\tReward total:12\tRunning mean: 10.67\n",
      "Learner:5\tReward total:17\tRunning mean: 20.22\n",
      "Learner:6\tReward total:18\tRunning mean: 23.85\n",
      "Learner:7\tReward total:36\tRunning mean: 30.28\n",
      "Max Norms =  ['86.89', '83.84', '253.90', '121.57', '35.10', '72.49', '104.07', '140.91']\n",
      "....................................................................................................\n",
      "Episode 1600 complete\n",
      "Learner:0\tReward total:21\tRunning mean: 17.18\n",
      "Learner:1\tReward total:20\tRunning mean: 17.07\n",
      "Learner:2\tReward total:28\tRunning mean: 23.58\n",
      "Learner:3\tReward total:14\tRunning mean: 18.17\n",
      "Learner:4\tReward total:3\tRunning mean: 12.03\n",
      "Learner:5\tReward total:18\tRunning mean: 21.32\n",
      "Learner:6\tReward total:28\tRunning mean: 27.9\n",
      "Learner:7\tReward total:30\tRunning mean: 27.0\n",
      "Max Norms =  ['95.74', '151.83', '149.25', '104.25', '137.35', '66.49', '104.65', '118.45']\n",
      "....................................................................................................\n",
      "Episode 1700 complete\n",
      "Learner:0\tReward total:20\tRunning mean: 16.28\n",
      "Learner:1\tReward total:21\tRunning mean: 16.18\n",
      "Learner:2\tReward total:27\tRunning mean: 24.21\n",
      "Learner:3\tReward total:22\tRunning mean: 20.5\n",
      "Learner:4\tReward total:15\tRunning mean: 13.53\n",
      "Learner:5\tReward total:20\tRunning mean: 19.62\n",
      "Learner:6\tReward total:21\tRunning mean: 27.85\n",
      "Learner:7\tReward total:26\tRunning mean: 28.42\n",
      "Max Norms =  ['402.66', '121.66', '173.97', '208.07', '80.64', '196.63', '157.75', '178.06']\n",
      "....................................................................................................\n",
      "Episode 1800 complete\n",
      "Learner:0\tReward total:26\tRunning mean: 16.53\n",
      "Learner:1\tReward total:26\tRunning mean: 15.97\n",
      "Learner:2\tReward total:31\tRunning mean: 22.75\n",
      "Learner:3\tReward total:12\tRunning mean: 20.06\n",
      "Learner:4\tReward total:11\tRunning mean: 12.01\n",
      "Learner:5\tReward total:23\tRunning mean: 21.46\n",
      "Learner:6\tReward total:12\tRunning mean: 28.81\n",
      "Learner:7\tReward total:17\tRunning mean: 29.17\n",
      "Max Norms =  ['124.84', '147.23', '159.91', '164.72', '130.08', '139.03', '124.87', '85.41']\n",
      "....................................................................................................\n",
      "Episode 1900 complete\n",
      "Learner:0\tReward total:13\tRunning mean: 16.83\n",
      "Learner:1\tReward total:16\tRunning mean: 17.3\n",
      "Learner:2\tReward total:10\tRunning mean: 21.27\n",
      "Learner:3\tReward total:17\tRunning mean: 19.06\n",
      "Learner:4\tReward total:21\tRunning mean: 14.6\n",
      "Learner:5\tReward total:17\tRunning mean: 20.61\n",
      "Learner:6\tReward total:36\tRunning mean: 27.53\n",
      "Learner:7\tReward total:34\tRunning mean: 29.33\n",
      "Max Norms =  ['99.41', '79.15', '126.03', '85.81', '105.28', '86.41', '207.74', '208.00']\n",
      "....................................................................................................\n",
      "Episode 2000 complete\n",
      "Learner:0\tReward total:18\tRunning mean: 16.71\n",
      "Learner:1\tReward total:20\tRunning mean: 17.17\n",
      "Learner:2\tReward total:14\tRunning mean: 23.29\n",
      "Learner:3\tReward total:40\tRunning mean: 20.2\n",
      "Learner:4\tReward total:14\tRunning mean: 16.5\n",
      "Learner:5\tReward total:20\tRunning mean: 17.54\n",
      "Learner:6\tReward total:15\tRunning mean: 26.59\n",
      "Learner:7\tReward total:28\tRunning mean: 30.8\n",
      "Max Norms =  ['173.69', '85.05', '161.84', '194.27', '71.97', '48.47', '211.42', '210.61']\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import sys\n",
    "import pickle\n",
    "from collections import deque\n",
    "\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import numpy as np\n",
    "\n",
    "# Initialize agents parameters\n",
    "#   10 agents - 9 learning agents, 0 trained agents, 0 random agents\n",
    "num_learners = 9\n",
    "num_trained = 0\n",
    "num_rdn = 0\n",
    "num_statics = num_trained + num_rdn\n",
    "num_agents = num_learners + num_statics  \n",
    "\n",
    "# Initialize environment\n",
    "game = \"Gather\"\n",
    "num_actions = 8                       # 8 actions in Gathering\n",
    "\n",
    "# Initialize training parameters\n",
    "warm_start = False\n",
    "num_frames = 4      # environ observation consists of a list of 4 stacked frames per agent\n",
    "max_episodes = 5000\n",
    "max_frames = 300\n",
    "max_frames_ep = 0   # track highest number of frames an episode can last\n",
    "\n",
    "# These trainer parameters works for Atari Breakout\n",
    "gamma = 0.99  \n",
    "lr = 1e-3\n",
    "temp_start = 1.8  # Temperature for explore/exploit\n",
    "temp_end = 1.0\n",
    "log_interval = 200\n",
    "save_interval = 500\n",
    "\n",
    "def unpack_env_obs(env_obs):\n",
    "    \"\"\"\n",
    "    Gathering is a partially-observable Markov Game. env_obs returned by GatheringEnv is a numpy \n",
    "    array of dimension (num_agent, 800), which represents the agents' observations of the game.\n",
    "\n",
    "    The 800 elements (view_box) encodes 4 layers of 10x20 pixels frames in the format:\n",
    "    (viewbox_width, viewbox_depth, 4).\n",
    "    \n",
    "    This code reshapes the above into stacked frames that can be accepted by the Policy class:\n",
    "    (batch_idx, in_channel, width, height)\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    num_agents = len(env_obs)  # environ observations is a list of agents' observations\n",
    "    \n",
    "    obs = []\n",
    "    for i in range(num_agents):\n",
    "        x = env_obs[i]   # take the indexed agent's observation\n",
    "        x = torch.Tensor(x)   # Convert to tensor\n",
    "        \n",
    "        # Policy is a 3-layer CNN\n",
    "        x = x.view(1, 10, 20, -1)  # reshape into environment defined stacked frames\n",
    "        x = x.permute(0, 3, 1, 2)  # permute to Policy accepted stacked frames\n",
    "        obs.append(x)\n",
    "        \n",
    "    return obs  # return a list of Tensors\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "For now, we do not implement LSTM            \n",
    "# LSTM Change: Need to cycle hx and cx thru function\n",
    "def select_action(model, state, lstm_hc, cuda):\n",
    "    hx , cx = lstm_hc \n",
    "    num_frames, height, width = state.shape\n",
    "    state = torch.FloatTensor(state.reshape(-1, num_frames, height, width))\n",
    "\n",
    "    if cuda:\n",
    "        state = state.cuda()\n",
    "\n",
    "    probs, value, (hx, cx) = model((Variable(state), (hx, cx)))\n",
    "\n",
    "    m = torch.distributions.Categorical(probs)\n",
    "    action = m.sample()\n",
    "    log_prob = m.log_prob(action)\n",
    "    # LSTM Change: Need to cycle hx and cx thru function\n",
    "    return action.data[0], log_prob, value, (hx, cx)\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "def select_learner_action(model, obs, cuda):\n",
    "    \"\"\"\n",
    "    This code expects obs to be an array of stacked frames of the following dim:\n",
    "    (batch_idx, in_channel, width, height)\n",
    "    \n",
    "    This is inputted into model - the agent's Policy, which outputs a probability \n",
    "    distribution over available actions.\n",
    "    \n",
    "    Policy gradient is implemented using torch.distributions.Categorical. \n",
    "    \"\"\"\n",
    "    \n",
    "    # Policy is a 3-layer CNN\n",
    "    # _, num_frames, width, height = obs.shape\n",
    "    # obs = torch.FloatTensor(obs.reshape(-1, num_frames, width, height))\n",
    "    \n",
    "    # Policy is a 2-layer NN for now\n",
    "    # obs = obs.view(1, -1)\n",
    "   \n",
    "    if cuda:\n",
    "        obs = obs.cuda()\n",
    "      \n",
    "    probs = model(obs)\n",
    "    m = torch.distributions.Categorical(probs)\n",
    "    action = m.sample()\n",
    "    log_prob = m.log_prob(action)\n",
    "\n",
    "    return action.item(), log_prob \n",
    "\n",
    "\n",
    "def load_info(agents, narrate=False):\n",
    "    for i in range(num_agents):    \n",
    "        agents[i].load_info(info[i])\n",
    "        if narrate:\n",
    "            if agents[i].tagged:\n",
    "                print('frame {}, agent{} is tagged'.format(frame,i))\n",
    "            if agents[i].laser_fired:\n",
    "                print('frame {}, agent{} fires its laser'.format(frame,i))\n",
    "                print('and hit {} US and {} THEM'.format(agents[i].US_hit, agents[i].THEM_hit))\n",
    "    return\n",
    "\n",
    "\n",
    "# The main code starts here!!!\n",
    "\n",
    "# Cultural parameter search\n",
    "cultures =[\n",
    "           {'name':'pacifist', 'penalty':-0.01},\n",
    "           {'name':'pacifist', 'penalty':-0.1},\n",
    "           {'name':'pacifist', 'penalty':-1.0},\n",
    "           {'name':'pacifist', 'penalty':-10.0},\n",
    "           {'name':'pacifist', 'penalty':-100.0}\n",
    "          ]\n",
    "\n",
    "for culture in cultures:   # Go down the list of cultures\n",
    "\n",
    "    # Data structure for agents\n",
    "    agents = []\n",
    "    actions = []\n",
    "    log_probs = []\n",
    "    tags = []\n",
    "    rewards = []\n",
    "    optimizers = []\n",
    "\n",
    "    # Cold start\n",
    "    if warm_start is False:\n",
    "   \n",
    "        # Initialize learner agents, then load static agents (trained followed by random)\n",
    "        for i in range(num_learners):\n",
    "            print(\"Learner agent {}\".format(i))\n",
    "            agents.append(Policy(num_frames, num_actions, i)) # No weights loaded for learning agent\n",
    "            optimizers.append(optim.Adam(agents[i].parameters(), lr=lr))\n",
    "        \n",
    "            # set up optimizer - this works for Atari Breakout\n",
    "            # optimizers.append(optim.RMSprop(agents[i].parameters(), lr=lr, weight_decay=0.1)) \n",
    "        \n",
    "        for i in range(num_learners, num_learners+num_trained):\n",
    "            print (\"No trained agent exist yet!\")\n",
    "            raise\n",
    "            \"\"\"\n",
    "            Disable for now! No trained model exist!!!\n",
    "            agents.append(Policy(num_frames, num_actions, i))\n",
    "            agents[i].load_weights()         # load weight for static agent        \n",
    "            \"\"\"\n",
    "        for i in range(num_learners+num_trained, num_agents):\n",
    "            print(\"Load random agent {}\".format(i))\n",
    "            agents.append(Rdn_Policy())\n",
    "\n",
    "    \n",
    "        # Initialize all agent data\n",
    "        actions = [0 for i in range(num_agents)]\n",
    "        log_probs = [0 for i in range(num_agents)]\n",
    "        tags = [0 for i in range(num_agents)]\n",
    "        rewards = [0 for i in range(num_agents)]\n",
    "\n",
    "        # Keep track of rewards learned by learners\n",
    "        episode_reward = [0 for i in range(num_learners)]   # reward for an episode\n",
    "        running_reward = [None for i in range(num_learners)]   # running average\n",
    "        running_rewards = [[] for i in range(num_learners)]   # history of running averages\n",
    "        best_reward = [0 for i in range(num_learners)]    # best running average (for storing best_model)\n",
    "\n",
    "        # This is to support warm start for training\n",
    "        prior_eps = 0\n",
    "\n",
    "    # Warm start\n",
    "    if warm_start:\n",
    "        print (\"Cannot warm start\")\n",
    "        raise\n",
    "    \n",
    "        \"\"\"\n",
    "        # Disable for now!  Need to ensure model can support training on GPU and game playing\n",
    "        # on both CPU and GPU.\n",
    "    \n",
    "        data_file = 'results/{}.p'.format(game)\n",
    "\n",
    "        try:\n",
    "            with open(data_file, 'rb') as f:\n",
    "                running_rewards = pickle.load(f)\n",
    "                running_reward = running_rewards[-1]\n",
    "\n",
    "            prior_eps = len(running_rewards)\n",
    "\n",
    "            model_file = 'saved_models/actor_critic_{}_ep_{}.p'.format(game, prior_eps)\n",
    "            with open(model_file, 'rb') as f:\n",
    "                # Model Save and Load Update: Include both model and optim parameters\n",
    "                saved_model = pickle.load(f)\n",
    "                model, optimizer = saved_model\n",
    "\n",
    "        except OSError:\n",
    "            print('Saved file not found. Creating new cold start model.')\n",
    "            model = Policy(input_channels=num_frames, num_actions=num_actions)\n",
    "            optimizer = optim.RMSprop(model.parameters(), lr=lr,\n",
    "                                      weight_decay=0.1)\n",
    "            running_rewards = []\n",
    "            prior_eps = 0\n",
    "        \"\"\"\n",
    "\n",
    "    # Establish tribal association\n",
    "\n",
    "    tribes = []\n",
    "    tribes.append(Tribe(name='Vikings',color='blue', culture=culture, \\\n",
    "                    agents=[agents[0], agents[1], agents[2]]))\n",
    "    tribes.append(Tribe(name='Saxons', color='red', culture=culture, \\\n",
    "                    agents=[agents[3], agents[4], agents[5]]))\n",
    "    tribes.append(Tribe(name='Franks', color='purple', culture=culture, \\\n",
    "                    agents=[agents[6], agents[7], agents[8]]))\n",
    "    # tribes.append(Tribe(name='Crazies', color='yellow', agents=[agents[9]]))   # random agents are crazy!!!\n",
    "\n",
    "    # 9 agents in 4 tribes, used map defined in default.txt\n",
    "    agent_colors = [agent.color for agent in agents]\n",
    "    agent_tribes = [agent.tribe for agent in agents]\n",
    "    env = GatheringEnv(n_agents=num_agents,agent_colors=agent_colors, agent_tribes=agent_tribes, \\\n",
    "                       map_name='default')    \n",
    "   \n",
    "    \n",
    "    cuda = torch.cuda.is_available()\n",
    "\n",
    "    if cuda:\n",
    "        for i in range(num_learners):    # Learning agents need to utilize GPU\n",
    "            agents[i].cuda()\n",
    "\n",
    "        \n",
    "    for ep in range(max_episodes):\n",
    "    \n",
    "        print('.', end='')  # To show progress\n",
    "    \n",
    "        # Anneal temperature from temp_start to temp_end\n",
    "        for i in range(num_learners):    # For learning agents\n",
    "            agents[i].temperature = max(temp_end, temp_start - (temp_start - temp_end) * (ep / max_episodes))\n",
    "\n",
    "        env_obs = env.reset()  # Env return observations\n",
    "\n",
    "        # For Debug only\n",
    "        # print (len(env_obs))\n",
    "        # print (env_obs[0].shape)\n",
    "    \n",
    "        # Unpack observations into data structure compatible with agent Policy\n",
    "        agents_obs = unpack_env_obs(env_obs)\n",
    "\n",
    "        for i in range(num_learners):    # Reset agent info - laser tag statistics\n",
    "            agents[i].reset_info()   \n",
    "\n",
    "        # For Debug only\n",
    "        # print (len(agents_obs))\n",
    "        # print (agents_obs[0].shape)\n",
    "    \n",
    "        \"\"\"\n",
    "        For now, we do not stack observations, and we do not implement LSTM\n",
    "    \n",
    "        state = np.stack([state]*num_frames)\n",
    "\n",
    "        # LSTM change - reset LSTM hidden units when episode begins\n",
    "        cx = Variable(torch.zeros(1, 256))\n",
    "        hx = Variable(torch.zeros(1, 256))\n",
    "        if cuda:\n",
    "            cx = cx.cuda()\n",
    "            hx = hx.cuda()\n",
    "        \"\"\"\n",
    "\n",
    "        episode_reward = [0 for i in range(num_learners)]   # reward for an episode\n",
    "    \n",
    "        for frame in range(max_frames):\n",
    "\n",
    "            \"\"\"\n",
    "            For now, we do not implement LSTM\n",
    "            # Select action\n",
    "            # LSTM Change: Need to cycle hx and cx thru select_action\n",
    "            action, log_prob, value, (hx,cx)  = select_action(model, state, (hx,cx), cuda)        \n",
    "            \"\"\"\n",
    "\n",
    "            for i in range(num_learners):    # For learning agents\n",
    "                actions[i], log_probs[i] = select_learner_action(agents[i], agents_obs[i], cuda)\n",
    "                if actions[i] is 6:\n",
    "                    tags[i] += 1   # record a tag for accessing aggressiveness\n",
    "                agents[i].saved_actions.append((log_probs[i]))\n",
    "            \n",
    "                # Do not implement LSTM for now\n",
    "                # actions[i].saved_actions.append((log_prob, value))\n",
    "            \n",
    "            for i in range(num_learners, num_learners+num_trained):\n",
    "                print (\"No trained agent exist yet!\")\n",
    "                raise\n",
    "            for i in range(num_learners+num_trained, num_agents):   # For random agents\n",
    "                actions[i] = agents[i].select_action(agents_obs[i])\n",
    "                if actions[i] is 6:\n",
    "                    tags[i] += 1   # record a tag for accessing aggressiveness\n",
    "\n",
    "            # For Debug only\n",
    "            # if frame % 20 == 0:\n",
    "            #    print (actions) \n",
    "            #    print (log_probs)\n",
    "            \n",
    "            # Perform step        \n",
    "            env_obs, reward, done, info = env.step(actions)\n",
    "        \n",
    "            \"\"\"\n",
    "            For Debug only\n",
    "            print (env_obs)\n",
    "            print (reward)\n",
    "            print (done) \n",
    "            \"\"\"\n",
    "       \n",
    "            # Unpack observations into data structure compatible with agent Policy\n",
    "            agents_obs = unpack_env_obs(env_obs)\n",
    "            load_info(agents, narrate=False)   # Load agent info for AI agents\n",
    "\n",
    "            # For learner agents only, generate reward statistics and reward stack for policy gradient\n",
    "            for i in range(num_learners):\n",
    "                agents[i].rewards.append(reward[i])  # Stack rewards (for policy gradient)\n",
    "                episode_reward[i] += reward[i]   # accumulate episode reward \n",
    "            \n",
    "            \"\"\"\n",
    "            For now, we do not stack observation, may come in handy later on\n",
    "        \n",
    "            # Evict oldest diff add new diff to state\n",
    "            next_state = np.stack([next_state]*num_frames)\n",
    "            next_state[1:, :, :] = state[:-1, :, :]\n",
    "            state = next_state\n",
    "            \"\"\"\n",
    "\n",
    "            if any(done):\n",
    "                print(\"Done after {} frames\".format(frame))\n",
    "                break\n",
    "            \n",
    "        if frame > max_frames_ep:\n",
    "            max_frames_ep = frame    # Keep track of highest frames/episode\n",
    "\n",
    "        # Update reward statistics for learners\n",
    "        for i in range(num_learners):\n",
    "            if running_reward[i] is None:\n",
    "                running_reward[i] = episode_reward[i]\n",
    "            running_reward[i] = running_reward[i] * 0.99 + episode_reward[i] * 0.01\n",
    "            running_rewards[i].append(running_reward[i])\n",
    "\n",
    "        # Track Episode #, temp and highest frames/episode\n",
    "        if (ep+prior_eps+1) % log_interval == 0: \n",
    "            verbose_str = '\\nEpisode {} complete'.format(ep+prior_eps+1)\n",
    "            # verbose_str += '\\tTemp = {:.4}'.format(model.temperature)\n",
    "            # verbose_str += '\\tMax frames = {}'.format(max_frames_ep+1)\n",
    "            print(verbose_str)\n",
    "    \n",
    "            # Display rewards and running rewards for learning agents\n",
    "            for i in range(num_learners):\n",
    "                verbose_str = 'Learner:{}'.format(i)\n",
    "                verbose_str += '\\tReward total:{}'.format(episode_reward[i])\n",
    "                verbose_str += '\\tRunning mean: {:.4}'.format(running_reward[i])\n",
    "                print(verbose_str)\n",
    "    \n",
    "        # Update model\n",
    "        total_norms = finish_episode(agents[0:num_learners], optimizers[0:num_learners], gamma, cuda)\n",
    "\n",
    "        if (ep+prior_eps+1) % log_interval == 0:\n",
    "            print('Max Norms = ',[\"%0.2f\" % i for i in total_norms])\n",
    "        \n",
    "        if (ep+prior_eps+1) % save_interval == 0: \n",
    "            for i in range(num_learners):\n",
    "                model_dir = 'MA_models/3T-9L/'\n",
    "                results_dir = 'results/3T-9L/'\n",
    "\n",
    "                model_file = model_dir+'{}/p{}/MA{}_{}_ep{}.p'.format(culture['name'], culture['penalty'],\\\n",
    "                                 i, game, ep+prior_eps+1)\n",
    "                data_file = results_dir+'{}/p{}/MA{}_{}.p'.format(culture['name'], culture['penalty'],\\\n",
    "                                                      i, game)\n",
    "                \n",
    "                os.makedirs(os.path.dirname(model_file), exist_ok=True)\n",
    "                os.makedirs(os.path.dirname(data_file), exist_ok=True)\n",
    "                \n",
    "                with open(model_file, 'wb') as f:\n",
    "                    # Model Save and Load Update: Include both model and optim parameters \n",
    "                    pickle.dump((agents[i].cpu(), optimizers[i]), f)\n",
    "\n",
    "                if cuda:\n",
    "                    agents[i] = agents[i].cuda()\n",
    "\n",
    "                with open(data_file, 'wb') as f:\n",
    "                    pickle.dump(running_rewards[i], f)    \n",
    "            \n",
    "    env.close()  # Close the environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}