workflowStochpid200_Ep15.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "aaf5d2db-5453-469c-9357-c37dedea7e89",
   "metadata": {},
   "source": [
    "# Sample Workflow for d3rlpy Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bfb6706a-eaec-4154-be04-e2b7c168479a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "import math\n",
    "import subprocess\n",
    "import os\n",
    "import d3rlpy\n",
    "plt.style.use('matplotlibrc')\n",
    "\n",
    "from Python.data_sampler import *"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "361d8a2f-129e-4b9c-81cd-55314fabeabd",
   "metadata": {},
   "source": [
    "## Building an MDPDataset\n",
    "\n",
    "We first read in a large batch of samples from the file. As `d3rlpy` wants it in the form (observations, actions, rewards, terminal flags), we go ahead and do that. Here's a helper function to get a dataset from a list of chunks of your choosing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3456d718-b8bb-460f-a429-9d99633145f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dataset(chunks : list, batch_size=30000, \n",
    "                path=\"collected_data/rl_deterministic.txt\") -> d3rlpy.dataset.MDPDataset :\n",
    "    random.seed(0)\n",
    "    samples = DataSampler(path_to_data=\"collected_data/rl_stochpid.txt\")\n",
    "    samples.setting(\"coarse\")\n",
    "    states = []\n",
    "    actions = []\n",
    "    rewards = []\n",
    "    next_states = []\n",
    "    for chunk in chunks:\n",
    "        samples.use_chunk(chunk)\n",
    "        samples.read_chunk()\n",
    "        [statesChunk, actionsChunk, rewardsChunk, nextStatesChunk] = samples.get_batch(batch_size)\n",
    "        states.append(statesChunk)\n",
    "        actions.append(actionsChunk)\n",
    "        rewards.append(rewardsChunk)\n",
    "        next_states.append(nextStatesChunk)\n",
    "    states = torch.cat(states)\n",
    "    actions = torch.cat(actions)\n",
    "    rewards = torch.cat(rewards)\n",
    "    next_states = torch.cat(next_states)\n",
    "    terminals = np.zeros(len(states))\n",
    "    terminals[::1111] = 1 #episode length 100, change if necessary\n",
    "    print(states.shape)\n",
    "    dataset = d3rlpy.dataset.MDPDataset(states.numpy(), \n",
    "                                        actions.numpy(), \n",
    "                                        rewards.numpy(), terminals)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eeaff27f-fd9a-4ad2-81c6-fca5f5fe6ce9",
   "metadata": {},
   "source": [
    "We can build the dataset from there, just like this, and split into train and test sets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f543168b-2603-47ed-bff8-d0bafb40bb0f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -4.75891077e-02 -3.69999953e-02\n",
      "  2.00999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  4.50429671e-01 -4.92727243e-01 -5.31666025e-03]\n",
      "Read chunk # -1 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.25610892e-01 -3.35999953e-02\n",
      " -2.42000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -3.08749986e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 2 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.08389108e-01  3.32000047e-02\n",
      " -2.02000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -4.25137655e-01 -1.65270603e-01  6.00000000e-01]\n",
      "Read chunk # 5 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -8.62891077e-02 -4.03999953e-02\n",
      " -1.76000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.96411621e-01 -3.05278115e-01  6.00000000e-01]\n",
      "Read chunk # 8 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -9.61891077e-02 -3.73999953e-02\n",
      " -9.50001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -1.42721664e-01 -2.59917132e-01  6.00000000e-01]\n",
      "Read chunk # 11 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.50189108e-01  3.62000047e-02\n",
      "  1.78999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -5.98450077e-02  6.00000000e-01]\n",
      "Read chunk # 14 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.05010892e-01 -5.19999531e-03\n",
      "  2.41999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01  1.88888739e-01 -6.00000000e-01]\n",
      "Read chunk # 17 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.06910892e-01 -4.19999953e-02\n",
      "  2.73999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  5.39942807e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 20 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.99210892e-01 -4.41999953e-02\n",
      " -1.77000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -7.33679917e-02  4.73334231e-01 -6.00000000e-01]\n",
      "Read chunk # 23 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.71589108e-01 -1.13999953e-02\n",
      "  2.14999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  5.45208655e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 26 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.36910892e-01  4.92000047e-02\n",
      " -7.80001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -3.78465429e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 29 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.88891077e-02 -5.79999953e-02\n",
      "  1.62999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -7.84491968e-02  2.01303439e-01]\n",
      "Read chunk # 32 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.80910892e-01  4.34000047e-02\n",
      "  1.49999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -8.68303371e-02  5.01639755e-01 -6.00000000e-01]\n",
      "Read chunk # 35 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.29989108e-01 -5.93999953e-02\n",
      "  1.87999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.00266707e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 38 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -4.37889108e-01  4.72000047e-02\n",
      " -6.90001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -5.04730803e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 41 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -7.03891077e-02 -1.45999953e-02\n",
      "  2.71999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  5.45547944e-01 -1.42260606e-01  1.10586690e-01]\n",
      "Read chunk # 44 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.56189108e-01  5.80000047e-02\n",
      "  1.50999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  4.51481072e-01 -2.88867043e-01  6.00000000e-01]\n",
      "Read chunk # 47 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.59710892e-01 -1.19999531e-03\n",
      "  1.38999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -1.86128266e-01  3.12352813e-01 -6.00000000e-01]\n",
      "Read chunk # 50 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.02189108e-01 -2.81999953e-02\n",
      "  1.33999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  3.85653844e-01 -2.69594451e-01  6.00000000e-01]\n",
      "Read chunk # 53 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.85310892e-01  5.70000047e-02\n",
      "  2.41999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  3.22471359e-01  4.41606904e-02 -6.00000000e-01]\n",
      "Read chunk # 56 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.50789108e-01  5.46000047e-02\n",
      " -1.72000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01 -3.35221789e-01  4.09875775e-01]\n",
      "Read chunk # 59 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.63489108e-01  1.62000047e-02\n",
      " -6.70001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -4.45065024e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 62 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -4.18389108e-01  3.44000047e-02\n",
      " -1.64000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  5.06280885e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 65 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.84108923e-02 -5.73999953e-02\n",
      "  2.49999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  5.48457250e-01 -5.06518229e-02 -6.00000000e-01]\n",
      "Read chunk # 68 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.16889108e-01 -1.95999953e-02\n",
      " -2.45000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01 -1.32698757e-01  6.00000000e-01]\n",
      "Read chunk # 71 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.03010892e-01  5.10000047e-02\n",
      "  2.51999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  4.52271790e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 74 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.91910892e-01  3.96000047e-02\n",
      " -2.36000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.92711681e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 77 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.77110892e-01 -5.45999953e-02\n",
      "  7.99998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  2.09994786e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 80 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.81410892e-01  4.20000469e-03\n",
      "  2.96999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 83 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.17689108e-01  4.08000047e-02\n",
      "  1.31999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  2.84891032e-01 -2.71061791e-01  6.00000000e-01]\n",
      "Read chunk # 86 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.43510892e-01  4.96000047e-02\n",
      " -2.00000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -3.07244996e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 89 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.24289108e-01  2.46000047e-02\n",
      "  1.88999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  1.59045766e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 92 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -4.43589108e-01  1.56000047e-02\n",
      " -1.34000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -3.56328529e-01 -3.71697889e-01  6.00000000e-01]\n",
      "Read chunk # 95 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.09789108e-01 -4.57999953e-02\n",
      " -2.20000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.84885686e-01 -2.78683294e-01  6.00000000e-01]\n",
      "Read chunk # 98 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.59889108e-01 -1.85999953e-02\n",
      " -2.70001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  2.58249611e-03 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 101 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.24010892e-01  2.32000047e-02\n",
      " -2.93000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 104 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.58110892e-01  3.32000047e-02\n",
      " -2.87000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01  3.85730338e-01 -6.00000000e-01]\n",
      "Read chunk # 107 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.77489108e-01  2.02000047e-02\n",
      " -6.50001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01 -3.49166569e-01  6.00000000e-01]\n",
      "Read chunk # 110 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.52891077e-02  3.98000047e-02\n",
      "  1.18999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  2.70574617e-01 -4.20063674e-01  6.00000000e-01]\n",
      "Read chunk # 113 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.05910892e-01 -3.91999953e-02\n",
      " -1.24000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.38197550e-01  4.97490391e-01 -6.00000000e-01]\n",
      "Read chunk # 116 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.99389108e-01  4.69199998e-09\n",
      " -3.40001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.39297056e-01  9.94067892e-02  6.00000000e-01]\n",
      "Read chunk # 119 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.39610892e-01  1.32000047e-02\n",
      "  2.50999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -7.01468987e-02  3.28419297e-01 -6.00000000e-01]\n",
      "Read chunk # 122 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.21089229e-03 -2.79999953e-02\n",
      "  2.45999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -1.74596735e-02 -1.04237546e-01]\n",
      "Read chunk # 125 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -4.07889108e-01  2.08000047e-02\n",
      "  2.89999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 128 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.44710892e-01  4.96000047e-02\n",
      " -1.33000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.28175455e-01  3.01721417e-01 -6.00000000e-01]\n",
      "Read chunk # 131 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.05310892e-01  3.72000047e-02\n",
      "  2.13999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  4.62211617e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 134 out of 4999\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  9.19108923e-02 -3.75999953e-02\n",
      " -1.29000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -4.10756296e-01  9.23267200e-02 -6.00000000e-01]\n",
      "Read chunk # 137 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.67589108e-01 -4.39999953e-02\n",
      "  2.89999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -3.07944815e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 140 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.20789108e-01 -3.09999953e-02\n",
      " -2.24000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01 -2.74665987e-01  6.00000000e-01]\n",
      "Read chunk # 143 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.02189108e-01  5.78000047e-02\n",
      "  5.39998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  1.40938487e-01 -2.83568202e-01  6.00000000e-01]\n",
      "Read chunk # 146 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -4.51891077e-02  2.50000047e-02\n",
      " -1.63000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -3.19573802e-01 -9.69405338e-02  2.33193033e-01]\n",
      "Read chunk # 149 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.96910892e-01  3.54000047e-02\n",
      " -1.00000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -4.00345524e-01  2.96168675e-01 -6.00000000e-01]\n",
      "Read chunk # 152 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.38789108e-01 -2.49999953e-02\n",
      " -2.11000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01 -2.01289443e-01  6.00000000e-01]\n",
      "Read chunk # 155 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.26710892e-01 -2.59999953e-02\n",
      "  2.88999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  4.36702158e-01  8.01277139e-02 -6.00000000e-01]\n",
      "Read chunk # 158 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.26410892e-01  1.64000047e-02\n",
      " -2.90000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.05298324e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 161 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.47889108e-01 -4.59999531e-03\n",
      "  2.85999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01  1.59959033e-01  6.00000000e-01]\n",
      "Read chunk # 164 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -4.16589108e-01  4.18000047e-02\n",
      " -5.50001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -5.07637733e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 167 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.08810892e-01 -2.45999953e-02\n",
      " -6.40001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  4.40229373e-02  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 170 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.16189108e-01  3.18000047e-02\n",
      " -1.86000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 173 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.23110892e-01  1.00000469e-03\n",
      "  2.96999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01  3.28335875e-01 -6.00000000e-01]\n",
      "Read chunk # 176 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.48108923e-02  1.80000469e-03\n",
      " -1.27000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01  4.59822805e-01 -6.00000000e-01]\n",
      "Read chunk # 179 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -4.24089108e-01 -2.85999953e-02\n",
      "  2.97999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 182 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.50689108e-01 -5.41999953e-02\n",
      "  2.95999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 185 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.14510892e-01  3.62000047e-02\n",
      " -1.83000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 188 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.48910892e-01 -2.91999953e-02\n",
      " -1.31000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01  5.05562706e-01 -6.00000000e-01]\n",
      "Read chunk # 191 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -8.29891077e-02 -5.09999953e-02\n",
      "  1.86999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  2.55508053e-01 -4.43781082e-01  4.05012232e-01]\n",
      "Read chunk # 194 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.31891077e-02 -3.79999531e-03\n",
      "  1.82999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -2.61015587e-01  1.26236120e-01]\n",
      "Read chunk # 197 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.29710892e-01  2.54000047e-02\n",
      " -1.92000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 200 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.89891077e-02  1.38000047e-02\n",
      "  1.55999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -5.13215135e-01  3.00794900e-01]\n",
      "Read chunk # 203 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.21289108e-01 -7.99995308e-04\n",
      " -2.20000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.75665250e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 206 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.57689108e-01  2.42000047e-02\n",
      "  2.37999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  5.83548139e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 209 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.43810892e-01  5.14000047e-02\n",
      " -1.30001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.70996225e-01  6.00000000e-01 -5.87902697e-01]\n",
      "Read chunk # 212 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.34689108e-01 -4.35999953e-02\n",
      " -1.36000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 215 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.26589108e-01 -4.69999953e-02\n",
      " -1.50001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  4.81812048e-03  1.54826348e-03  6.00000000e-01]\n",
      "Read chunk # 218 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.48010892e-01 -5.19999953e-02\n",
      " -2.13000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  7.08863912e-03  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 221 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -4.57891077e-02  5.12000047e-02\n",
      " -9.10001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -4.65655744e-01 -3.32196737e-01  4.37241574e-01]\n",
      "Read chunk # 224 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -4.15989108e-01  5.66000047e-02\n",
      " -1.41000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01 -4.52867603e-01  6.00000000e-01]\n",
      "Read chunk # 227 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.36289108e-01 -2.53999953e-02\n",
      "  1.71999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 230 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -4.01589108e-01  2.80000047e-02\n",
      "  2.68999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -4.51054866e-01  6.00000000e-01]\n",
      "Read chunk # 233 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  7.57108923e-02 -1.07999953e-02\n",
      " -8.20001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -1.44534291e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 236 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.48889108e-01  5.22000047e-02\n",
      " -8.80001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -5.39525779e-01 -2.65920002e-01  6.00000000e-01]\n",
      "Read chunk # 239 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.14410892e-01  1.20000469e-03\n",
      " -2.86000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 242 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.81110892e-01  1.88000047e-02\n",
      "  1.83999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.51417538e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 245 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -4.29489108e-01  1.24000047e-02\n",
      "  1.97999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  3.73149828e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 248 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.79989108e-01  2.20000469e-03\n",
      " -1.09000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -1.56992418e-01 -2.96852309e-01  6.00000000e-01]\n",
      "Read chunk # 251 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.40210892e-01  5.78000047e-02\n",
      " -1.90000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 254 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.50489108e-01  1.96000047e-02\n",
      " -1.07000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  5.59731026e-02 -1.94939649e-01  6.00000000e-01]\n",
      "Read chunk # 257 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.23789108e-01  3.50000047e-02\n",
      " -2.99000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01 -1.69292322e-01  6.00000000e-01]\n",
      "Read chunk # 260 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.13189108e-01 -5.13999953e-02\n",
      "  1.89999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -1.66280664e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 263 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.50889108e-01 -7.59999531e-03\n",
      " -1.83000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 266 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.52189108e-01  1.00000047e-02\n",
      "  2.39999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 269 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.22510892e-01 -1.79999531e-03\n",
      "  2.79998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  1.81232156e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 272 out of 4999\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.49210892e-01 -3.03999953e-02\n",
      "  1.49998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  1.79615166e-01  5.81541855e-01 -6.00000000e-01]\n",
      "Read chunk # 275 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.37810892e-01 -5.05999953e-02\n",
      " -1.02000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  1.09901302e-02  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 278 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  7.48108923e-02 -5.23999953e-02\n",
      "  2.36999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -3.68605982e-02 -4.87448936e-01]\n",
      "Read chunk # 281 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.21610892e-01  8.00000469e-03\n",
      "  2.97999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01  2.91952737e-02 -6.00000000e-01]\n",
      "Read chunk # 284 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.48310892e-01  5.00000047e-02\n",
      "  2.96999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01  3.59901780e-01 -6.00000000e-01]\n",
      "Read chunk # 287 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.67710892e-01 -4.85999953e-02\n",
      " -8.40001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  1.81703176e-02  3.82277322e-01 -6.00000000e-01]\n",
      "Read chunk # 290 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.04389108e-01  5.24000047e-02\n",
      " -2.83000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 293 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.41289108e-01  3.60000047e-02\n",
      " -1.40001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -8.27789575e-02 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 296 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.17510892e-01 -3.65999953e-02\n",
      " -1.66000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -3.00240960e-01  2.90765602e-01 -6.00000000e-01]\n",
      "Read chunk # 299 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.73289108e-01 -5.45999953e-02\n",
      " -1.00000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.16993578e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 302 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  6.49108923e-02  5.90000047e-02\n",
      "  9.49998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  7.10391075e-02  2.54105374e-01 -5.00664430e-01]\n",
      "Read chunk # 305 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.38389108e-01  2.34000047e-02\n",
      " -1.60001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -3.07355592e-02 -5.98605161e-01  6.00000000e-01]\n",
      "Read chunk # 308 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.30010892e-01 -5.63999953e-02\n",
      " -1.00000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  8.46207718e-03  6.00000000e-01 -5.80351397e-01]\n",
      "Read chunk # 311 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.36891077e-02  5.04000047e-02\n",
      "  8.99986580e-04  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.52208393e-01  6.92762005e-02  1.56656693e-01]\n",
      "Read chunk # 314 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.10810892e-01 -5.97999953e-02\n",
      "  1.98999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  5.01115535e-01 -8.61781267e-02 -6.00000000e-01]\n",
      "Read chunk # 317 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.05110892e-01 -2.47999953e-02\n",
      " -8.10001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.86964731e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 320 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.22910892e-01  2.00000047e-02\n",
      "  1.27999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  4.59289728e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 323 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.52010892e-01  5.68000047e-02\n",
      " -5.60001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.00695431e-01  2.31423962e-01 -6.00000000e-01]\n",
      "Read chunk # 326 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.49110892e-01  1.76000047e-02\n",
      "  2.81999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -1.34419574e-01 -6.00000000e-01]\n",
      "Read chunk # 329 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -4.23489108e-01  3.76000047e-02\n",
      "  6.19998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  9.58279765e-02  3.17116545e-02  6.00000000e-01]\n",
      "Read chunk # 332 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.76789108e-01  2.14000047e-02\n",
      "  2.16999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  3.59804672e-01 -3.51601165e-01  6.00000000e-01]\n",
      "Read chunk # 335 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.50310892e-01  6.00004692e-04\n",
      "  1.55999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  2.71425846e-01  5.62564812e-01 -6.00000000e-01]\n",
      "Read chunk # 338 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.09489108e-01  1.18000047e-02\n",
      "  2.79998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -5.72494312e-02 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 341 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.80010892e-01  2.98000047e-02\n",
      " -2.80000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 344 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.45310892e-01 -2.93999953e-02\n",
      " -2.75000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 347 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.07710892e-01 -3.39999953e-02\n",
      "  2.04999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01  3.85309036e-01 -6.00000000e-01]\n",
      "Read chunk # 350 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.93610892e-01 -2.01999953e-02\n",
      "  2.11999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01  3.77359095e-01 -6.00000000e-01]\n",
      "Read chunk # 353 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  9.11089229e-03  4.48000047e-02\n",
      " -2.09000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -1.70878049e-01  3.49786872e-01  1.45655484e-01]\n",
      "Read chunk # 356 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.43389108e-01 -2.71999953e-02\n",
      "  1.36999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  5.14610654e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 359 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -4.46889108e-01 -7.39999531e-03\n",
      "  1.69998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  1.88330148e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 362 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.40089108e-01  4.00000469e-03\n",
      " -2.10001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -8.76858206e-02 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 365 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.30389108e-01 -5.85999953e-02\n",
      "  8.09998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  3.42118666e-01  2.63720877e-02  6.00000000e-01]\n",
      "Read chunk # 368 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.08389108e-01 -4.21999953e-02\n",
      " -1.03000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -1.50809711e-01 -5.32694771e-01  6.00000000e-01]\n",
      "Read chunk # 371 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -6.28891077e-02 -1.59999531e-03\n",
      "  9.99865802e-05  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  1.05695718e-02 -1.89067652e-01  2.30149487e-01]\n",
      "Read chunk # 374 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -7.33891077e-02  2.76000047e-02\n",
      "  2.10999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -6.00000000e-01  4.90958138e-01]\n",
      "Read chunk # 377 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.51410892e-01  1.16000047e-02\n",
      " -6.30001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.11237961e-02  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 380 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.68389108e-01 -4.81999953e-02\n",
      "  4.69998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 383 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.02089108e-01  2.94000047e-02\n",
      "  2.85999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  4.70683052e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 386 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -9.85891077e-02  2.40000047e-02\n",
      "  1.54999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  5.25769160e-01 -3.53723681e-01  4.73708513e-01]\n",
      "Read chunk # 389 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.86610892e-01  5.90000047e-02\n",
      " -2.92000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 392 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.00110892e-01  4.40000469e-03\n",
      "  1.51999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  4.91762511e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 395 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.47510892e-01 -5.19999531e-03\n",
      "  8.19998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  9.97151393e-02  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 398 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.36010892e-01  3.60000469e-03\n",
      "  2.71999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 401 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.33410892e-01  5.72000047e-02\n",
      " -2.17000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -5.65014856e-01  3.98708063e-01 -6.00000000e-01]\n",
      "Read chunk # 404 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.04089108e-01 -2.37999953e-02\n",
      " -1.25000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -3.35348279e-01 -7.34814169e-02  6.00000000e-01]\n",
      "Read chunk # 407 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.31210892e-01  5.64000047e-02\n",
      "  1.64999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  2.39621452e-01  5.45026826e-01 -6.00000000e-01]\n",
      "Read chunk # 410 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.03010892e-01 -4.73999953e-02\n",
      " -1.20001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  5.32425768e-02  2.79664163e-01 -6.00000000e-01]\n",
      "Read chunk # 413 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.94689108e-01  1.92000047e-02\n",
      " -7.40001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -1.15756192e-01 -5.77515100e-01  6.00000000e-01]\n",
      "Read chunk # 416 out of 4999\n",
      "start\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.00000000e+00  7.95731469e+08  1.10510892e-01 -5.01999953e-02\n",
      " -2.71000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -4.67950082e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 419 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.13310892e-01 -2.69999953e-02\n",
      " -5.80001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -1.68944324e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 422 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.29489108e-01 -5.43999953e-02\n",
      "  1.76999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  4.79725512e-01 -3.26286803e-01  6.00000000e-01]\n",
      "Read chunk # 425 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.33210892e-01  2.12000047e-02\n",
      " -1.99000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01  5.26419148e-01 -6.00000000e-01]\n",
      "Read chunk # 428 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.94489108e-01 -3.47999953e-02\n",
      "  2.14999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -3.44267910e-01  6.00000000e-01]\n",
      "Read chunk # 431 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -4.06689108e-01 -3.01999953e-02\n",
      " -1.11000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  3.16202763e-02 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 434 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.72989108e-01 -2.57999953e-02\n",
      " -1.97000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -5.32948988e-01 -1.60593091e-01  6.00000000e-01]\n",
      "Read chunk # 437 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.24610892e-01 -3.39999953e-02\n",
      " -2.63000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  4.79065041e-02  4.78462151e-01 -6.00000000e-01]\n",
      "Read chunk # 440 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.85108923e-02  3.66000047e-02\n",
      "  6.29998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -4.91595501e-02 -4.47642958e-02 -3.77784896e-01]\n",
      "Read chunk # 443 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.22410892e-01 -3.11999953e-02\n",
      "  9.19998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  2.26573046e-01  4.05336790e-01 -6.00000000e-01]\n",
      "Read chunk # 446 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.13789108e-01 -4.69999953e-02\n",
      " -8.60001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -1.30889834e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 449 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -9.19891077e-02 -4.91999953e-02\n",
      " -1.87000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.76006838e-01  4.82024170e-02  1.06681578e-01]\n",
      "Read chunk # 452 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.13410892e-01 -4.35999953e-02\n",
      " -2.24000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -5.79667359e-01  6.00000000e-01 -5.83572140e-01]\n",
      "Read chunk # 455 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.35210892e-01  3.26000047e-02\n",
      " -1.93000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 458 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -9.43891077e-02  2.10000047e-02\n",
      "  2.93999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 461 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.10892292e-04  3.64000047e-02\n",
      " -5.00001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.64499871e-01  1.94654762e-01  2.53659250e-01]\n",
      "Read chunk # 464 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.51389108e-01 -4.71999953e-02\n",
      "  2.04999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 467 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.19389108e-01 -5.61999953e-02\n",
      " -1.83000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.31365978e-01 -3.13119816e-01  6.00000000e-01]\n",
      "Read chunk # 470 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.87710892e-01 -4.39999953e-02\n",
      "  1.92999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  3.64064042e-01  4.01946007e-01 -6.00000000e-01]\n",
      "Read chunk # 473 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.02710892e-01 -1.89999953e-02\n",
      " -1.53000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -5.65702577e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 476 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.21989108e-01 -5.97999953e-02\n",
      "  8.69998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  4.34425950e-01  2.29266661e-02  4.90639112e-01]\n",
      "Read chunk # 479 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.28610892e-01  5.54000047e-02\n",
      "  2.24999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 482 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.17910892e-01  2.06000047e-02\n",
      " -1.32000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 485 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.55910892e-01  3.40000047e-02\n",
      "  1.59999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  4.26866809e-01  4.49172343e-01 -6.00000000e-01]\n",
      "Read chunk # 488 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.97789108e-01  5.44000047e-02\n",
      " -2.57000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.70053884e-01 -5.93878507e-01  6.00000000e-01]\n",
      "Read chunk # 491 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.82189108e-01 -5.37999953e-02\n",
      " -9.50001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -1.64140546e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 494 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.38589108e-01 -4.73999953e-02\n",
      " -5.00001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -8.67049447e-02 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 497 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.23710892e-01  4.02000047e-02\n",
      "  1.92999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  5.79238375e-01  3.63239427e-01 -6.00000000e-01]\n",
      "Read chunk # 500 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.99289108e-01 -3.69999953e-02\n",
      "  2.18999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.02595006e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 503 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.27610892e-01 -2.69999953e-02\n",
      " -2.42000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -4.47210887e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 506 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -9.19891077e-02 -4.19999531e-03\n",
      "  2.19998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  7.05966752e-02 -5.47924738e-02  2.36330862e-01]\n",
      "Read chunk # 509 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.29589108e-01  4.46000047e-02\n",
      "  1.89999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  3.64467258e-01 -4.88968803e-01  6.00000000e-01]\n",
      "Read chunk # 512 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.37889108e-01 -8.99999531e-03\n",
      "  1.97999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -5.49984046e-01  5.90675362e-01]\n",
      "Read chunk # 515 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  9.82108923e-02  2.84000047e-02\n",
      " -1.73000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 518 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.81789108e-01 -2.33999953e-02\n",
      " -2.73000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 521 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.35610892e-01 -1.11999953e-02\n",
      " -2.99000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -5.81947142e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 524 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  5.80108923e-02 -5.79999953e-02\n",
      " -2.13000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -4.69506856e-01  3.92174298e-01  1.12031119e-01]\n",
      "Read chunk # 527 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.78210892e-01  4.68000047e-02\n",
      " -9.70001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 530 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.60210892e-01 -1.49999953e-02\n",
      "  4.19998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.25177022e-02  3.21207732e-01 -6.00000000e-01]\n",
      "Read chunk # 533 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.01108923e-02  4.98000047e-02\n",
      " -2.20000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -3.31290230e-01  1.31805500e-01 -1.74241358e-01]\n",
      "Read chunk # 536 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.89289108e-01 -2.41999953e-02\n",
      " -1.32000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -1.81775096e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 539 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.78108923e-02 -2.91999953e-02\n",
      "  1.03999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01  2.09695008e-02 -6.00000000e-01]\n",
      "Read chunk # 542 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.78489108e-01 -2.81999953e-02\n",
      " -2.57000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 545 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  3.79310892e-01  5.60000047e-02\n",
      "  2.09998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.77325909e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 548 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.85489108e-01 -3.01999953e-02\n",
      "  2.79999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  1.34206595e-01 -2.61331967e-01  6.00000000e-01]\n",
      "Read chunk # 551 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.64589108e-01 -2.21999953e-02\n",
      "  2.19998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  2.96296333e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 554 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.12089108e-01 -1.93999953e-02\n",
      " -1.47000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01 -2.76501657e-01  6.00000000e-01]\n",
      "Read chunk # 557 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  4.44410892e-01  9.20000469e-03\n",
      "  1.42999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  4.55158024e-01  6.00000000e-01 -6.00000000e-01]\n",
      "Read chunk # 560 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.43189108e-01  4.52000047e-02\n",
      " -2.51000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -5.41423855e-01 -1.38633448e-01  6.00000000e-01]\n",
      "Read chunk # 563 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -1.69089108e-01 -5.73999953e-02\n",
      "  1.87999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  5.94362624e-01 -5.96389084e-01  6.00000000e-01]\n",
      "Read chunk # 566 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -4.18891077e-02 -2.57999953e-02\n",
      " -1.15000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -1.52135913e-01 -2.93324567e-02  2.87400114e-01]\n",
      "Read chunk # 569 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.41989108e-01  5.00000469e-03\n",
      " -2.87000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -6.00000000e-01 -4.14867070e-02  6.00000000e-01]\n",
      "Read chunk # 572 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.52789108e-01  1.78000047e-02\n",
      " -7.80001342e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -2.89679268e-01 -7.12466655e-02  6.00000000e-01]\n",
      "Read chunk # 575 out of 4999\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  1.59410892e-01  1.36000047e-02\n",
      " -1.09000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -4.35316098e-01  5.03258928e-01 -6.00000000e-01]\n",
      "Read chunk # 578 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.04189108e-01  1.82000047e-02\n",
      "  1.36999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -4.92474183e-02 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 581 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.86089108e-01 -5.23999953e-02\n",
      "  8.59998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  6.00000000e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 584 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -3.02589108e-01 -4.71999953e-02\n",
      " -1.54000134e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -1.09668859e-01 -6.00000000e-01  6.00000000e-01]\n",
      "Read chunk # 587 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08  2.02108923e-02  4.70000047e-02\n",
      "  1.75999866e-02  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  1.48938817e-01  4.91111161e-02 -1.57500779e-01]\n",
      "Read chunk # 590 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -2.05689108e-01  4.76000047e-02\n",
      "  7.99998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      "  3.55953068e-02 -5.59553297e-01  6.00000000e-01]\n",
      "Read chunk # 593 out of 4999\n",
      "start\n",
      "[ 0.00000000e+00  7.95731469e+08 -9.07891077e-02  3.08000047e-02\n",
      "  3.49998658e-03  0.00000000e+00 -5.33423489e+00 -1.57091618e+00\n",
      " -1.20284015e-01 -5.72056059e-01  6.00000000e-01]\n",
      "Read chunk # 596 out of 4999\n",
      "torch.Size([222000, 6])\n"
     ]
    }
   ],
   "source": [
    "dataset = get_dataset([i+(i-1)*2 for i in range(200)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8014463e-ab16-4498-ae33-63b91854534c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The behavior policy value statistics are:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean': -135.4019,\n",
       " 'std': 100.196686,\n",
       " 'min': -394.1042,\n",
       " 'max': 0.0,\n",
       " 'histogram': (array([ 4, 10,  3,  1,  7,  5,  3,  5,  5,  4,  2,  7,  4, 15, 20, 29, 31,\n",
       "         37,  7,  1], dtype=int64),\n",
       "  array([-394.1042  , -374.399   , -354.69376 , -334.98856 , -315.28336 ,\n",
       "         -295.57812 , -275.87292 , -256.16772 , -236.46251 , -216.75731 ,\n",
       "         -197.0521  , -177.34688 , -157.64168 , -137.93646 , -118.231255,\n",
       "          -98.52605 ,  -78.82084 ,  -59.115627,  -39.41042 ,  -19.70521 ,\n",
       "            0.      ], dtype=float32))}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"The behavior policy value statistics are:\")\n",
    "dataset.compute_stats()['return']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f051bee1-1f8b-4acc-a658-d80a5b4df54b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "train_episodes, test_episodes = train_test_split(dataset, test_size=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1025ba17-f95f-426b-adc1-9312e5e180fa",
   "metadata": {},
   "source": [
    "## Setting up an Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f5ef782b-6847-4fa4-8d95-3fa8904866bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from d3rlpy.algos import CQL\n",
    "\n",
    "from d3rlpy.preprocessing import MinMaxActionScaler\n",
    "action_scaler = MinMaxActionScaler(minimum=-0.6, maximum=0.6)\n",
    "#cql = CQL(action_scaler=action_scaler)\n",
    "\n",
    "model = CQL(q_func_factory='mean', #qr -> quantile regression q function, but you don't have to use this\n",
    "            reward_scaler='standard',\n",
    "            action_scaler=action_scaler,\n",
    "          actor_learning_rate=1e-5, \n",
    "          critic_learning_rate=0.0003, \n",
    "            use_gpu=False) #change it to true if you have one\n",
    "model.build_with_dataset(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2fd44e0e-0ab2-4ab0-8f96-7a8eb382cda0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.055221074319288195\n"
     ]
    }
   ],
   "source": [
    "from d3rlpy.metrics.scorer import td_error_scorer\n",
    "from d3rlpy.metrics.scorer import average_value_estimation_scorer\n",
    "from d3rlpy.metrics.scorer import initial_state_value_estimation_scorer\n",
    "\n",
    "# calculate metrics with test dataset\n",
    "ave_error_init = average_value_estimation_scorer(model, test_episodes)\n",
    "print(ave_error_init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8f69d169-0f59-4e77-8654-c5d79dae2b1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Reusing TensorBoard on port 6006 (pid 9160), started 9 days, 13:54:19 ago. (Use '!kill 9160' to kill it.)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "      <iframe id=\"tensorboard-frame-629f6fbed82c07cd\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
       "      </iframe>\n",
       "      <script>\n",
       "        (function() {\n",
       "          const frame = document.getElementById(\"tensorboard-frame-629f6fbed82c07cd\");\n",
       "          const url = new URL(\"/\", window.location);\n",
       "          const port = 6006;\n",
       "          if (port) {\n",
       "            url.port = port;\n",
       "          }\n",
       "          frame.src = url;\n",
       "        })();\n",
       "      </script>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%load_ext tensorboard\n",
    "%tensorboard --logdir runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9b0384d5-a843-4d30-a355-404215f27fa4",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-17 12:00.21 [debug    ] RoundIterator is selected.\n",
      "2022-04-17 12:00.21 [info     ] Directory is created at d3rlpy_logs\\CQL_20220417120021\n",
      "2022-04-17 12:00.21 [debug    ] Fitting action scaler...       action_scaler=min_max\n",
      "2022-04-17 12:00.21 [debug    ] Fitting reward scaler...       reward_scaler=standard\n",
      "2022-04-17 12:00.21 [warning  ] Skip building models since they're already built.\n",
      "2022-04-17 12:00.21 [info     ] Parameters are saved to d3rlpy_logs\\CQL_20220417120021\\params.json params={'action_scaler': {'type': 'min_max', 'params': {'minimum': array(-0.6), 'maximum': array(0.6)}}, 'actor_encoder_factory': {'type': 'default', 'params': {'activation': 'relu', 'use_batch_norm': False, 'dropout_rate': None}}, 'actor_learning_rate': 1e-05, 'actor_optim_factory': {'optim_cls': 'Adam', 'betas': (0.9, 0.999), 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'alpha_learning_rate': 0.0001, 'alpha_optim_factory': {'optim_cls': 'Adam', 'betas': (0.9, 0.999), 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'alpha_threshold': 10.0, 'batch_size': 256, 'conservative_weight': 5.0, 'critic_encoder_factory': {'type': 'default', 'params': {'activation': 'relu', 'use_batch_norm': False, 'dropout_rate': None}}, 'critic_learning_rate': 0.0003, 'critic_optim_factory': {'optim_cls': 'Adam', 'betas': (0.9, 0.999), 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'gamma': 0.99, 'generated_maxlen': 100000, 'initial_alpha': 1.0, 'initial_temperature': 1.0, 'n_action_samples': 10, 'n_critics': 2, 'n_frames': 1, 'n_steps': 1, 'q_func_factory': {'type': 'mean', 'params': {'bootstrap': False, 'share_encoder': False}}, 'real_ratio': 1.0, 'reward_scaler': {'type': 'standard', 'params': {'mean': -0.1253796968196785, 'std': 0.1152335874082873, 'eps': 0.001, 'multiplier': 1.0}}, 'scaler': None, 'soft_q_backup': False, 'target_reduction_type': 'min', 'tau': 0.005, 'temp_learning_rate': 0.0001, 'temp_optim_factory': {'optim_cls': 'Adam', 'betas': (0.9, 0.999), 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False}, 'use_gpu': None, 'algorithm': 'CQL', 'observation_shape': (6,), 'action_size': 3}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f24c0c90dd34a319a151eacc3061230",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/15:   0%|          | 0/693 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-17 12:02.30 [info     ] CQL_20220417120021: epoch=1 step=693 epoch=1 metrics={'time_sample_batch': 0.000581503946543772, 'time_algorithm_update': 0.18238733375571334, 'temp_loss': 4.65626538677133, 'temp': 0.9668053239798993, 'alpha_loss': -12.911016746344849, 'alpha': 1.0322156683535115, 'critic_loss': 14.74611774461094, 'actor_loss': -0.42827668672892527, 'time_step': 0.18320987406919184, 'td_error': 6.001556134833017, 'init_value': -4.348829746246338, 'ave_value': -0.5191810367349148} step=693\n",
      "2022-04-17 12:02.30 [info     ] Model parameters are saved to d3rlpy_logs\\CQL_20220417120021\\model_693.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63373ebdcb1e46d48bfefbf36caaa333",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/15:   0%|          | 0/693 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-17 12:04.29 [info     ] CQL_20220417120021: epoch=2 step=1386 epoch=2 metrics={'time_sample_batch': 0.0004270537763102918, 'time_algorithm_update': 0.16782348118131124, 'temp_loss': 4.042389003745405, 'temp': 0.9060278358569565, 'alpha_loss': -7.260059795682392, 'alpha': 1.0884678769765306, 'critic_loss': 10.052958227683284, 'actor_loss': 3.121363379330002, 'time_step': 0.1684828299980659, 'td_error': 14.85021098793301, 'init_value': -13.9783935546875, 'ave_value': -4.502754552398767} step=1386\n",
      "2022-04-17 12:04.29 [info     ] Model parameters are saved to d3rlpy_logs\\CQL_20220417120021\\model_1386.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ba2303b3b0243f6888c804f82b4ec6d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/15:   0%|          | 0/693 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-17 12:06.27 [info     ] CQL_20220417120021: epoch=3 step=2079 epoch=3 metrics={'time_sample_batch': 0.0005929741852555268, 'time_algorithm_update': 0.16659873010104181, 'temp_loss': 3.576845687533182, 'temp': 0.8512001107265423, 'alpha_loss': -4.1661767718767875, 'alpha': 1.1331465830878606, 'critic_loss': 9.84127060641114, 'actor_loss': 9.652977783098537, 'time_step': 0.16735042844499862, 'td_error': 19.60398619671991, 'init_value': -26.02377700805664, 'ave_value': -9.833783397163375} step=2079\n",
      "2022-04-17 12:06.27 [info     ] Model parameters are saved to d3rlpy_logs\\CQL_20220417120021\\model_2079.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "486eab6ea88d49bfb75814ccb06b8262",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/15:   0%|          | 0/693 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-17 12:08.25 [info     ] CQL_20220417120021: epoch=4 step=2772 epoch=4 metrics={'time_sample_batch': 0.0006621565137590681, 'time_algorithm_update': 0.16742711734634114, 'temp_loss': 3.1692982800251133, 'temp': 0.8011528140882737, 'alpha_loss': -2.8076866900260744, 'alpha': 1.17197868473086, 'critic_loss': 13.866958395915287, 'actor_loss': 17.794004983888215, 'time_step': 0.16825243095298867, 'td_error': 22.42609883305729, 'init_value': -39.32931900024414, 'ave_value': -16.39575589178855} step=2772\n",
      "2022-04-17 12:08.25 [info     ] Model parameters are saved to d3rlpy_logs\\CQL_20220417120021\\model_2772.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dbe4e0aa3afb4c19988016b918b31b2a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 5/15:   0%|          | 0/693 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-17 12:10.23 [info     ] CQL_20220417120021: epoch=5 step=3465 epoch=5 metrics={'time_sample_batch': 0.0006348157857919668, 'time_algorithm_update': 0.16598991054133075, 'temp_loss': 2.820546483580684, 'temp': 0.7550066305273367, 'alpha_loss': -2.2495455466329104, 'alpha': 1.2119492885870335, 'critic_loss': 21.397209364270406, 'actor_loss': 26.36379223227673, 'time_step': 0.166740145346131, 'td_error': 25.293200149778745, 'init_value': -54.10169219970703, 'ave_value': -22.602784375771563} step=3465\n",
      "2022-04-17 12:10.23 [info     ] Model parameters are saved to d3rlpy_logs\\CQL_20220417120021\\model_3465.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5deac6cc56c4426494ab24fbcf17844b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 6/15:   0%|          | 0/693 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-17 12:12.21 [info     ] CQL_20220417120021: epoch=6 step=4158 epoch=6 metrics={'time_sample_batch': 0.0006377026116177117, 'time_algorithm_update': 0.16696134033313217, 'temp_loss': 2.4986630112233787, 'temp': 0.7121622214069614, 'alpha_loss': -1.8040631297069654, 'alpha': 1.2553659978539053, 'critic_loss': 32.08857310626758, 'actor_loss': 34.95188240502648, 'time_step': 0.16790203889899097, 'td_error': 29.275144914172746, 'init_value': -70.87553405761719, 'ave_value': -30.787530470795485} step=4158\n",
      "2022-04-17 12:12.21 [info     ] Model parameters are saved to d3rlpy_logs\\CQL_20220417120021\\model_4158.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c3a07b99c02404bb6e5ad773dfc034f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 7/15:   0%|          | 0/693 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-17 12:14.17 [info     ] CQL_20220417120021: epoch=7 step=4851 epoch=7 metrics={'time_sample_batch': 0.0004905364214083849, 'time_algorithm_update': 0.16463896963331434, 'temp_loss': 2.1880219995889485, 'temp': 0.6725771279218049, 'alpha_loss': -1.3982097943018434, 'alpha': 1.3002332290701708, 'critic_loss': 44.02269009766297, 'actor_loss': 43.06395434920406, 'time_step': 0.1653632488085594, 'td_error': 33.69266521438974, 'init_value': -85.70948791503906, 'ave_value': -36.35884067366008} step=4851\n",
      "2022-04-17 12:14.17 [info     ] Model parameters are saved to d3rlpy_logs\\CQL_20220417120021\\model_4851.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f798a01559ec4061a7962d67cd131ce4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 8/15:   0%|          | 0/693 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-17 12:16.15 [info     ] CQL_20220417120021: epoch=8 step=5544 epoch=8 metrics={'time_sample_batch': 0.00048186940720243025, 'time_algorithm_update': 0.16577298816664393, 'temp_loss': 1.9091968126971313, 'temp': 0.6357587360028408, 'alpha_loss': -0.8422430705562675, 'alpha': 1.3415798963750423, 'critic_loss': 59.289354796361444, 'actor_loss': 50.756713201129244, 'time_step': 0.16642943777219213, 'td_error': 39.98317461586636, 'init_value': -102.40438079833984, 'ave_value': -42.78646596258656} step=5544\n",
      "2022-04-17 12:16.15 [info     ] Model parameters are saved to d3rlpy_logs\\CQL_20220417120021\\model_5544.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "adde24b1db2b4955ab789543b2b2137a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 9/15:   0%|          | 0/693 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-17 12:18.11 [info     ] CQL_20220417120021: epoch=9 step=6237 epoch=9 metrics={'time_sample_batch': 0.0006204067714630612, 'time_algorithm_update': 0.16425267476884145, 'temp_loss': 1.6732670479866678, 'temp': 0.6014262222796464, 'alpha_loss': -0.4084108769298971, 'alpha': 1.3685963265651577, 'critic_loss': 76.6834666949158, 'actor_loss': 58.06314557008069, 'time_step': 0.16500294053709352, 'td_error': 47.60893605526132, 'init_value': -118.98560333251953, 'ave_value': -49.00421224516956} step=6237\n",
      "2022-04-17 12:18.11 [info     ] Model parameters are saved to d3rlpy_logs\\CQL_20220417120021\\model_6237.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0271d5e2da654e6380b63762d27fb634",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 10/15:   0%|          | 0/693 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-17 12:20.08 [info     ] CQL_20220417120021: epoch=10 step=6930 epoch=10 metrics={'time_sample_batch': 0.0006665818791024785, 'time_algorithm_update': 0.16571338352186854, 'temp_loss': 1.469061935274804, 'temp': 0.5690988520695189, 'alpha_loss': 0.07688553110907788, 'alpha': 1.377646206605314, 'critic_loss': 92.84677293015076, 'actor_loss': 65.00190007703817, 'time_step': 0.16653869781659278, 'td_error': 54.852616630438845, 'init_value': -133.86863708496094, 'ave_value': -54.96286857938301} step=6930\n",
      "2022-04-17 12:20.08 [info     ] Model parameters are saved to d3rlpy_logs\\CQL_20220417120021\\model_6930.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3821aaefa3214dfda6a6a826f9ef34df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 11/15:   0%|          | 0/693 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-17 12:22.04 [info     ] CQL_20220417120021: epoch=11 step=7623 epoch=11 metrics={'time_sample_batch': 0.0006224964604233251, 'time_algorithm_update': 0.16451068082757153, 'temp_loss': 1.3229033974514035, 'temp': 0.5382011694481534, 'alpha_loss': 0.4170242162034303, 'alpha': 1.3621412262772068, 'critic_loss': 109.27623535509922, 'actor_loss': 71.64886636445017, 'time_step': 0.16530630151602785, 'td_error': 63.17940015315356, 'init_value': -149.45184326171875, 'ave_value': -61.386242799283785} step=7623\n",
      "2022-04-17 12:22.04 [info     ] Model parameters are saved to d3rlpy_logs\\CQL_20220417120021\\model_7623.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bbdc036552bc4d2b9a3b8c459ea8be8a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 12/15:   0%|          | 0/693 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-17 12:24.02 [info     ] CQL_20220417120021: epoch=12 step=8316 epoch=12 metrics={'time_sample_batch': 0.0004761088290083804, 'time_algorithm_update': 0.1660835650060084, 'temp_loss': 1.185880459404267, 'temp': 0.5087536117559216, 'alpha_loss': 0.5833540028707839, 'alpha': 1.324088328844541, 'critic_loss': 127.83576335122575, 'actor_loss': 77.80929886899125, 'time_step': 0.1666751042072907, 'td_error': 71.62691609080991, 'init_value': -164.76126098632812, 'ave_value': -66.63057370241908} step=8316\n",
      "2022-04-17 12:24.02 [info     ] Model parameters are saved to d3rlpy_logs\\CQL_20220417120021\\model_8316.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b7436ba34fb442496ac8de2b2c02c1c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 13/15:   0%|          | 0/693 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-17 12:25.58 [info     ] CQL_20220417120021: epoch=13 step=9009 epoch=13 metrics={'time_sample_batch': 0.0005237670859905204, 'time_algorithm_update': 0.16389727420449085, 'temp_loss': 1.0804106128680242, 'temp': 0.48042657059680505, 'alpha_loss': 0.6573837014482307, 'alpha': 1.2808677620357938, 'critic_loss': 146.20540157059398, 'actor_loss': 83.85198626717792, 'time_step': 0.16469662874119967, 'td_error': 79.7875179208428, 'init_value': -177.63638305664062, 'ave_value': -70.9167471174476} step=9009\n",
      "2022-04-17 12:25.58 [info     ] Model parameters are saved to d3rlpy_logs\\CQL_20220417120021\\model_9009.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2d23f514b8644c7b3d9be93c37365d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 14/15:   0%|          | 0/693 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-17 12:27.54 [info     ] CQL_20220417120021: epoch=14 step=9702 epoch=14 metrics={'time_sample_batch': 0.0005626912412877379, 'time_algorithm_update': 0.1646141273118717, 'temp_loss': 0.988661247414428, 'temp': 0.45337048633102045, 'alpha_loss': 0.6154817443819445, 'alpha': 1.2362774607079026, 'critic_loss': 166.17844334206023, 'actor_loss': 89.60769694054454, 'time_step': 0.16539324172819503, 'td_error': 88.38481607945064, 'init_value': -192.35400390625, 'ave_value': -76.79703487137917} step=9702\n",
      "2022-04-17 12:27.54 [info     ] Model parameters are saved to d3rlpy_logs\\CQL_20220417120021\\model_9702.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a84eab962ce44c68a5f044de7514230",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 15/15:   0%|          | 0/693 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-17 12:29.51 [info     ] CQL_20220417120021: epoch=15 step=10395 epoch=15 metrics={'time_sample_batch': 0.0005222808403026146, 'time_algorithm_update': 0.16569094293217293, 'temp_loss': 0.8922837355092146, 'temp': 0.4275907404495008, 'alpha_loss': 0.6358140777708482, 'alpha': 1.1962082086703478, 'critic_loss': 183.75308277699855, 'actor_loss': 94.97248449504461, 'time_step': 0.16642965417231662, 'td_error': 97.42876166185621, 'init_value': -205.39906311035156, 'ave_value': -82.70192911476404} step=10395\n",
      "2022-04-17 12:29.51 [info     ] Model parameters are saved to d3rlpy_logs\\CQL_20220417120021\\model_10395.pt\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(1,\n",
       "  {'time_sample_batch': 0.000581503946543772,\n",
       "   'time_algorithm_update': 0.18238733375571334,\n",
       "   'temp_loss': 4.65626538677133,\n",
       "   'temp': 0.9668053239798993,\n",
       "   'alpha_loss': -12.911016746344849,\n",
       "   'alpha': 1.0322156683535115,\n",
       "   'critic_loss': 14.74611774461094,\n",
       "   'actor_loss': -0.42827668672892527,\n",
       "   'time_step': 0.18320987406919184,\n",
       "   'td_error': 6.001556134833017,\n",
       "   'init_value': -4.348829746246338,\n",
       "   'ave_value': -0.5191810367349148}),\n",
       " (2,\n",
       "  {'time_sample_batch': 0.0004270537763102918,\n",
       "   'time_algorithm_update': 0.16782348118131124,\n",
       "   'temp_loss': 4.042389003745405,\n",
       "   'temp': 0.9060278358569565,\n",
       "   'alpha_loss': -7.260059795682392,\n",
       "   'alpha': 1.0884678769765306,\n",
       "   'critic_loss': 10.052958227683284,\n",
       "   'actor_loss': 3.121363379330002,\n",
       "   'time_step': 0.1684828299980659,\n",
       "   'td_error': 14.85021098793301,\n",
       "   'init_value': -13.9783935546875,\n",
       "   'ave_value': -4.502754552398767}),\n",
       " (3,\n",
       "  {'time_sample_batch': 0.0005929741852555268,\n",
       "   'time_algorithm_update': 0.16659873010104181,\n",
       "   'temp_loss': 3.576845687533182,\n",
       "   'temp': 0.8512001107265423,\n",
       "   'alpha_loss': -4.1661767718767875,\n",
       "   'alpha': 1.1331465830878606,\n",
       "   'critic_loss': 9.84127060641114,\n",
       "   'actor_loss': 9.652977783098537,\n",
       "   'time_step': 0.16735042844499862,\n",
       "   'td_error': 19.60398619671991,\n",
       "   'init_value': -26.02377700805664,\n",
       "   'ave_value': -9.833783397163375}),\n",
       " (4,\n",
       "  {'time_sample_batch': 0.0006621565137590681,\n",
       "   'time_algorithm_update': 0.16742711734634114,\n",
       "   'temp_loss': 3.1692982800251133,\n",
       "   'temp': 0.8011528140882737,\n",
       "   'alpha_loss': -2.8076866900260744,\n",
       "   'alpha': 1.17197868473086,\n",
       "   'critic_loss': 13.866958395915287,\n",
       "   'actor_loss': 17.794004983888215,\n",
       "   'time_step': 0.16825243095298867,\n",
       "   'td_error': 22.42609883305729,\n",
       "   'init_value': -39.32931900024414,\n",
       "   'ave_value': -16.39575589178855}),\n",
       " (5,\n",
       "  {'time_sample_batch': 0.0006348157857919668,\n",
       "   'time_algorithm_update': 0.16598991054133075,\n",
       "   'temp_loss': 2.820546483580684,\n",
       "   'temp': 0.7550066305273367,\n",
       "   'alpha_loss': -2.2495455466329104,\n",
       "   'alpha': 1.2119492885870335,\n",
       "   'critic_loss': 21.397209364270406,\n",
       "   'actor_loss': 26.36379223227673,\n",
       "   'time_step': 0.166740145346131,\n",
       "   'td_error': 25.293200149778745,\n",
       "   'init_value': -54.10169219970703,\n",
       "   'ave_value': -22.602784375771563}),\n",
       " (6,\n",
       "  {'time_sample_batch': 0.0006377026116177117,\n",
       "   'time_algorithm_update': 0.16696134033313217,\n",
       "   'temp_loss': 2.4986630112233787,\n",
       "   'temp': 0.7121622214069614,\n",
       "   'alpha_loss': -1.8040631297069654,\n",
       "   'alpha': 1.2553659978539053,\n",
       "   'critic_loss': 32.08857310626758,\n",
       "   'actor_loss': 34.95188240502648,\n",
       "   'time_step': 0.16790203889899097,\n",
       "   'td_error': 29.275144914172746,\n",
       "   'init_value': -70.87553405761719,\n",
       "   'ave_value': -30.787530470795485}),\n",
       " (7,\n",
       "  {'time_sample_batch': 0.0004905364214083849,\n",
       "   'time_algorithm_update': 0.16463896963331434,\n",
       "   'temp_loss': 2.1880219995889485,\n",
       "   'temp': 0.6725771279218049,\n",
       "   'alpha_loss': -1.3982097943018434,\n",
       "   'alpha': 1.3002332290701708,\n",
       "   'critic_loss': 44.02269009766297,\n",
       "   'actor_loss': 43.06395434920406,\n",
       "   'time_step': 0.1653632488085594,\n",
       "   'td_error': 33.69266521438974,\n",
       "   'init_value': -85.70948791503906,\n",
       "   'ave_value': -36.35884067366008}),\n",
       " (8,\n",
       "  {'time_sample_batch': 0.00048186940720243025,\n",
       "   'time_algorithm_update': 0.16577298816664393,\n",
       "   'temp_loss': 1.9091968126971313,\n",
       "   'temp': 0.6357587360028408,\n",
       "   'alpha_loss': -0.8422430705562675,\n",
       "   'alpha': 1.3415798963750423,\n",
       "   'critic_loss': 59.289354796361444,\n",
       "   'actor_loss': 50.756713201129244,\n",
       "   'time_step': 0.16642943777219213,\n",
       "   'td_error': 39.98317461586636,\n",
       "   'init_value': -102.40438079833984,\n",
       "   'ave_value': -42.78646596258656}),\n",
       " (9,\n",
       "  {'time_sample_batch': 0.0006204067714630612,\n",
       "   'time_algorithm_update': 0.16425267476884145,\n",
       "   'temp_loss': 1.6732670479866678,\n",
       "   'temp': 0.6014262222796464,\n",
       "   'alpha_loss': -0.4084108769298971,\n",
       "   'alpha': 1.3685963265651577,\n",
       "   'critic_loss': 76.6834666949158,\n",
       "   'actor_loss': 58.06314557008069,\n",
       "   'time_step': 0.16500294053709352,\n",
       "   'td_error': 47.60893605526132,\n",
       "   'init_value': -118.98560333251953,\n",
       "   'ave_value': -49.00421224516956}),\n",
       " (10,\n",
       "  {'time_sample_batch': 0.0006665818791024785,\n",
       "   'time_algorithm_update': 0.16571338352186854,\n",
       "   'temp_loss': 1.469061935274804,\n",
       "   'temp': 0.5690988520695189,\n",
       "   'alpha_loss': 0.07688553110907788,\n",
       "   'alpha': 1.377646206605314,\n",
       "   'critic_loss': 92.84677293015076,\n",
       "   'actor_loss': 65.00190007703817,\n",
       "   'time_step': 0.16653869781659278,\n",
       "   'td_error': 54.852616630438845,\n",
       "   'init_value': -133.86863708496094,\n",
       "   'ave_value': -54.96286857938301}),\n",
       " (11,\n",
       "  {'time_sample_batch': 0.0006224964604233251,\n",
       "   'time_algorithm_update': 0.16451068082757153,\n",
       "   'temp_loss': 1.3229033974514035,\n",
       "   'temp': 0.5382011694481534,\n",
       "   'alpha_loss': 0.4170242162034303,\n",
       "   'alpha': 1.3621412262772068,\n",
       "   'critic_loss': 109.27623535509922,\n",
       "   'actor_loss': 71.64886636445017,\n",
       "   'time_step': 0.16530630151602785,\n",
       "   'td_error': 63.17940015315356,\n",
       "   'init_value': -149.45184326171875,\n",
       "   'ave_value': -61.386242799283785}),\n",
       " (12,\n",
       "  {'time_sample_batch': 0.0004761088290083804,\n",
       "   'time_algorithm_update': 0.1660835650060084,\n",
       "   'temp_loss': 1.185880459404267,\n",
       "   'temp': 0.5087536117559216,\n",
       "   'alpha_loss': 0.5833540028707839,\n",
       "   'alpha': 1.324088328844541,\n",
       "   'critic_loss': 127.83576335122575,\n",
       "   'actor_loss': 77.80929886899125,\n",
       "   'time_step': 0.1666751042072907,\n",
       "   'td_error': 71.62691609080991,\n",
       "   'init_value': -164.76126098632812,\n",
       "   'ave_value': -66.63057370241908}),\n",
       " (13,\n",
       "  {'time_sample_batch': 0.0005237670859905204,\n",
       "   'time_algorithm_update': 0.16389727420449085,\n",
       "   'temp_loss': 1.0804106128680242,\n",
       "   'temp': 0.48042657059680505,\n",
       "   'alpha_loss': 0.6573837014482307,\n",
       "   'alpha': 1.2808677620357938,\n",
       "   'critic_loss': 146.20540157059398,\n",
       "   'actor_loss': 83.85198626717792,\n",
       "   'time_step': 0.16469662874119967,\n",
       "   'td_error': 79.7875179208428,\n",
       "   'init_value': -177.63638305664062,\n",
       "   'ave_value': -70.9167471174476}),\n",
       " (14,\n",
       "  {'time_sample_batch': 0.0005626912412877379,\n",
       "   'time_algorithm_update': 0.1646141273118717,\n",
       "   'temp_loss': 0.988661247414428,\n",
       "   'temp': 0.45337048633102045,\n",
       "   'alpha_loss': 0.6154817443819445,\n",
       "   'alpha': 1.2362774607079026,\n",
       "   'critic_loss': 166.17844334206023,\n",
       "   'actor_loss': 89.60769694054454,\n",
       "   'time_step': 0.16539324172819503,\n",
       "   'td_error': 88.38481607945064,\n",
       "   'init_value': -192.35400390625,\n",
       "   'ave_value': -76.79703487137917}),\n",
       " (15,\n",
       "  {'time_sample_batch': 0.0005222808403026146,\n",
       "   'time_algorithm_update': 0.16569094293217293,\n",
       "   'temp_loss': 0.8922837355092146,\n",
       "   'temp': 0.4275907404495008,\n",
       "   'alpha_loss': 0.6358140777708482,\n",
       "   'alpha': 1.1962082086703478,\n",
       "   'critic_loss': 183.75308277699855,\n",
       "   'actor_loss': 94.97248449504461,\n",
       "   'time_step': 0.16642965417231662,\n",
       "   'td_error': 97.42876166185621,\n",
       "   'init_value': -205.39906311035156,\n",
       "   'ave_value': -82.70192911476404})]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(train_episodes,\n",
    "        eval_episodes=test_episodes,\n",
    "        n_epochs=15, \n",
    "        tensorboard_dir='runs',\n",
    "        scorers={\n",
    "            'td_error': td_error_scorer,\n",
    "            'init_value': initial_state_value_estimation_scorer,\n",
    "            'ave_value': average_value_estimation_scorer\n",
    "        })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "69ff770c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Aero\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\d3rlpy\\preprocessing\\action_scalers.py:204: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n",
      "  minimum = torch.tensor(\n",
      "C:\\Users\\Aero\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\d3rlpy\\preprocessing\\action_scalers.py:207: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n",
      "  maximum = torch.tensor(\n"
     ]
    }
   ],
   "source": [
    "model.save_model('cqlStochpid200model_Ep15check.pt')\n",
    "model.save_policy('cqlStochpid200_Ep15check.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bbb0460-f723-4b67-a553-57887d11dbc9",
   "metadata": {},
   "source": [
    "## Off-Policy Evaluation\n",
    "\n",
    "We do get some metrics on a test set of initial state value and average value. However, these estimates (using the critic's Q-function) of model performance are biased. They're useful for validation during training, but not much else. Instead, we fit a Q-function to the data (or a separate dataset, as I've done here) separately and evaluate the model's performance on it.\n",
    "\n",
    "Feel free to change the chunks and number of steps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "de79cadb-28ef-42f8-b230-756f3ee11bb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from d3rlpy.ope import FQE\n",
    "# # metrics to evaluate with\n",
    "# from d3rlpy.metrics.scorer import soft_opc_scorer\n",
    "\n",
    "\n",
    "# ope_dataset = get_dataset([i*2 for i in range(100)], path=\"collected_data/rl_deterministic.txt\") #change if you'd prefer different chunks\n",
    "# ope_train_episodes, ope_test_episodes = train_test_split(ope_dataset, test_size=0.2)\n",
    "\n",
    "# fqe = FQE(algo=model, action_scaler = action_scaler, use_gpu=False) #change this if you have one!\n",
    "# fqe.fit(ope_train_episodes, eval_episodes=ope_test_episodes,\n",
    "#         tensorboard_dir='runs',\n",
    "#         n_epochs=100, n_steps_per_epoch=10000, #change if overfitting/underfitting\n",
    "#         scorers={\n",
    "#            'init_value': initial_state_value_estimation_scorer,\n",
    "#             'ave_value': average_value_estimation_scorer,\n",
    "#            'soft_opc': soft_opc_scorer(return_threshold=0)\n",
    "#         })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5d639afd-476b-4844-9921-f001f81abf34",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from d3rlpy.ope import FQE\n",
    "# # metrics to evaluate with\n",
    "# from d3rlpy.metrics.scorer import soft_opc_scorer\n",
    "\n",
    "\n",
    "# ope_dataset = get_dataset([i*2 for i in range(100)], path=\"collected_data/rl_stochastic.txt\") #change if you'd prefer different chunks\n",
    "# ope_train_episodes, ope_test_episodes = train_test_split(ope_dataset, test_size=0.2)\n",
    "\n",
    "# fqe = FQE(algo=model, action_scaler = action_scaler, use_gpu=False) #change this if you have one!\n",
    "# fqe.fit(ope_train_episodes, eval_episodes=ope_test_episodes,\n",
    "#         tensorboard_dir='runs',\n",
    "#         n_epochs=100, n_steps_per_epoch=10000, #change if overfitting/underfitting\n",
    "#         scorers={\n",
    "#            'init_value': initial_state_value_estimation_scorer,\n",
    "#             'ave_value': average_value_estimation_scorer,\n",
    "#            'soft_opc': soft_opc_scorer(return_threshold=0)\n",
    "#         })"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}