muzero_cartpole_v3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this is MuZero!\n",
    "# https://arxiv.org/abs/1911.08265\n",
    "# It works, but only for CartPole\n",
    "# Would be happy if it worked for Follower, LunarLander, or Acrobot\n",
    "# Also, while it's written, it's not using mcts.\n",
    "# Replace naive_search(which tries n**K) with mcts_search to use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import gym\n",
    "from tqdm import tqdm, trange\n",
    "import os,sys\n",
    "sys.path.append(os.getcwd())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.8.0\n"
     ]
    }
   ],
   "source": [
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[name: \"/device:CPU:0\"\n",
       " device_type: \"CPU\"\n",
       " memory_limit: 268435456\n",
       " locality {\n",
       " }\n",
       " incarnation: 438963871019809633\n",
       " xla_global_id: -1]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "\n",
    "device_lib.list_local_devices()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU')]\n",
      "tf.Tensor([1.4667784  0.22522028], shape=(2,), dtype=float32)\n",
      "tf.Tensor([1.4667784  0.22522028], shape=(2,), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(tf.config.list_physical_devices())\n",
    "with tf.device('/GPU'):\n",
    "  a = tf.random.normal(shape=(2,), dtype=tf.float32)\n",
    "  b = tf.nn.relu(a)\n",
    "\n",
    "print(a)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from tensorflow.python.framework.ops import disable_eager_execution\n",
    "# disable_eager_execution()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4,) 2\n"
     ]
    }
   ],
   "source": [
    "# Make Follower work! Will give interview to anyone who does.\n",
    "from muzero.follower import Follower\n",
    "#env = Follower()\n",
    "env = gym.make(\"CartPole-v0\")\n",
    "#env = gym.make(\"MountainCar-v0\")\n",
    "#env = gym.make(\"LunarLander-v2\")\n",
    "#env = gym.make(\"Acrobot-v1\")\n",
    "\n",
    "from muzero.model import MuModel\n",
    "m = MuModel(env.observation_space.shape, env.action_space.n, s_dim=128, K=3, lr=0.001)\n",
    "print(env.observation_space.shape, env.action_space.n)\n",
    "\n",
    "from muzero.game import Game, ReplayBuffer\n",
    "from muzero.mcts import naive_search, mcts_search\n",
    "replay_buffer = ReplayBuffer(50, 128, m.K)\n",
    "rews = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def play_game(env, m):\n",
    "  import random\n",
    "  game = Game(env, discount=0.997)\n",
    "  while not game.terminal():\n",
    "    game.env.render()\n",
    "    cc = random.random()\n",
    "    if cc < 0.05:\n",
    "      policy = [1/m.a_dim]*m.a_dim\n",
    "    else:\n",
    "      policy = naive_search(m, game.observation, T=1)\n",
    "    game.act_with_policy(policy)\n",
    "  return game"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-05-17 00:23:01.038410: W tensorflow/core/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8 8.0 Counter({0: 8}) 5.213472366333008\n",
      "19 19.0 Counter({0: 11, 1: 8}) 23.832164764404297\n",
      "119 119.0 Counter({1: 63, 0: 56}) 1879.95654296875\n",
      "12 12.0 Counter({1: 10, 0: 2}) 1328.507568359375\n",
      "200 200.0 Counter({0: 101, 1: 99}) 4219.6083984375\n",
      "33 33.0 Counter({1: 17, 0: 16}) 3474.5498046875\n",
      "30 30.0 Counter({0: 15, 1: 15}) 3689.006591796875\n",
      "55 55.0 Counter({0: 30, 1: 25}) 3851.533203125\n",
      "200 200.0 Counter({0: 103, 1: 97}) 4390.376953125\n",
      "200 200.0 Counter({0: 101, 1: 99}) 5540.93359375\n",
      "200 200.0 Counter({1: 101, 0: 99}) 6812.81494140625\n",
      "200 200.0 Counter({1: 101, 0: 99}) 6100.310546875\n",
      "200 200.0 Counter({1: 100, 0: 100}) 3947.413818359375\n"
     ]
    }
   ],
   "source": [
    "from muzero.model import reformat_batch\n",
    "import collections\n",
    "\n",
    "for j in range(30):\n",
    "  game = play_game(env, m)\n",
    "  replay_buffer.save_game(game)\n",
    "  for i in range(20):\n",
    "    m.train_on_batch(replay_buffer.sample_batch())\n",
    "  rew = sum(game.rewards)\n",
    "  rews.append(rew)\n",
    "  print(len(game.history), rew, collections.Counter(game.history), m.losses[-1][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(rews)\n",
    "plt.yscale('log')\n",
    "plt.plot([x[0] for x in m.losses])\n",
    "plt.plot([x[1] for x in m.losses])\n",
    "plt.plot([x[-3] for x in m.losses])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# can act?\n",
    "state = env.reset()\n",
    "for sn in range(2000):\n",
    "  p_0 = naive_search(m, state, debug=False, T=0.1)\n",
    "  #p_0, _ = mcts_search(m, state, 50)\n",
    "  a_1 = np.random.choice(list(range(len(p_0))), p=p_0)\n",
    "  _, v_0 = m.ft(m.ht(state))\n",
    "  \n",
    "  env.render()\n",
    "  state,r,done,_ = env.step(a_1)\n",
    "  print(a_1, v_0, r, p_0)\n",
    "  if done:\n",
    "    print(\"DONE\", sn)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}