My first Ray project w XGBoost.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2f787c2d",
   "metadata": {},
   "source": [
    "## My first Ray project w XGBoost\n",
    "\n",
    "Trying out some getting started, configuring the environment and so on :)\n",
    "To try it out, run env-setting notebook before. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "43d8e0c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-19 08:27:43,210\tWARNING services.py:1780 -- WARNING: The object store is using /tmp instead of /dev/shm because /dev/shm has only 67108864 bytes available. This will harm performance! You may be able to free up space by deleting files in /dev/shm. If you are inside a Docker container, you can increase /dev/shm size by passing '--shm-size=5.04gb' to 'docker run' (or add it to the run_options list in a Ray cluster config). Make sure to set this to more than 30% of available RAM.\n",
      "2023-03-19 08:27:43,341\tINFO worker.py:1553 -- Started a local Ray instance.\n",
      "2023-03-19 08:27:51,734\tWARNING read_api.py:330 -- ⚠️  The number of blocks in this dataset (1) limits its parallelism to 1 concurrent tasks. This is much less than the number of available CPU slots in the cluster. Use `.repartition(n)` to increase the number of dataset blocks.\n",
      "Read progress: 100%|██████████| 1/1 [00:01<00:00,  1.99s/it]\n",
      "Read progress: 100%|██████████| 1/1 [00:00<00:00, 578.76it/s]\n",
      "Read progress: 100%|██████████| 1/1 [00:00<00:00, 645.87it/s]\n"
     ]
    }
   ],
   "source": [
    "import ray\n",
    "\n",
    "# Load data.\n",
    "dataset = ray.data.read_csv(\"s3://anonymous@air-example-data/breast_cancer.csv\")\n",
    "\n",
    "# Split data into train and validation.\n",
    "train_dataset, valid_dataset = dataset.train_test_split(test_size=0.3)\n",
    "\n",
    "# Create a test dataset by dropping the target column.\n",
    "test_dataset = valid_dataset.drop_columns(cols=[\"target\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5e81f21c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a preprocessor to scale some columns.\n",
    "from ray.data.preprocessors import StandardScaler\n",
    "\n",
    "preprocessor = StandardScaler(columns=[\"mean radius\", \"mean texture\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2a4d674",
   "metadata": {},
   "source": [
    "## train a model with XGBoostTrainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4faaf7fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-19 08:29:20,033\tINFO tensorboardx.py:170 -- pip install \"ray[tune]\" to see TensorBoard files.\n",
      "2023-03-19 08:29:20,034\tWARNING callback.py:108 -- The TensorboardX logger cannot be instantiated because either TensorboardX or one of it's dependencies is not installed. Please make sure you have the latest version of TensorboardX installed: `pip install -U tensorboardx`\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"tuneStatus\">\n",
       "  <div style=\"display: flex;flex-direction: row\">\n",
       "    <div style=\"display: flex;flex-direction: column;\">\n",
       "      <h3>Tune Status</h3>\n",
       "      <table>\n",
       "<tbody>\n",
       "<tr><td>Current time:</td><td>2023-03-19 08:29:28</td></tr>\n",
       "<tr><td>Running for: </td><td>00:00:08.83        </td></tr>\n",
       "<tr><td>Memory:      </td><td>5.4/15.6 GiB       </td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "    </div>\n",
       "    <div class=\"vDivider\"></div>\n",
       "    <div class=\"systemInfo\">\n",
       "      <h3>System Info</h3>\n",
       "      Using FIFO scheduling algorithm.<br>Resources requested: 0/8 CPUs, 0/0 GPUs, 0.0/9.16 GiB heap, 0.0/4.58 GiB objects\n",
       "    </div>\n",
       "    \n",
       "  </div>\n",
       "  <div class=\"hDivider\"></div>\n",
       "  <div class=\"trialStatus\">\n",
       "    <h3>Trial Status</h3>\n",
       "    <table>\n",
       "<thead>\n",
       "<tr><th>Trial name                </th><th>status    </th><th>loc            </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  train-logloss</th><th style=\"text-align: right;\">  train-error</th><th style=\"text-align: right;\">  valid-logloss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>XGBoostTrainer_25070_00000</td><td>TERMINATED</td><td>172.17.0.2:1120</td><td style=\"text-align: right;\">    21</td><td style=\"text-align: right;\">         6.78411</td><td style=\"text-align: right;\">      0.0184957</td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">      0.0897979</td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "  </div>\n",
       "</div>\n",
       "<style>\n",
       ".tuneStatus {\n",
       "  color: var(--jp-ui-font-color1);\n",
       "}\n",
       ".tuneStatus .systemInfo {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus td {\n",
       "  white-space: nowrap;\n",
       "}\n",
       ".tuneStatus .trialStatus {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".tuneStatus .hDivider {\n",
       "  border-bottom-width: var(--jp-border-width);\n",
       "  border-bottom-color: var(--jp-border-color0);\n",
       "  border-bottom-style: solid;\n",
       "}\n",
       ".tuneStatus .vDivider {\n",
       "  border-left-width: var(--jp-border-width);\n",
       "  border-left-color: var(--jp-border-color0);\n",
       "  border-left-style: solid;\n",
       "  margin: 0.5em 1em 0.5em 1em;\n",
       "}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1120)\u001b[0m 2023-03-19 08:29:22,056\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[aggregate]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1120)\u001b[0m 2023-03-19 08:29:23,356\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[StandardScaler]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1120)\u001b[0m 2023-03-19 08:29:23,412\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[StandardScaler]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1120)\u001b[0m 2023-03-19 08:29:23,481\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[repartition]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1120)\u001b[0m 2023-03-19 08:29:23,557\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[repartition]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1120)\u001b[0m 2023-03-19 08:29:26,177\tINFO tracker.py:218 -- start listen on 172.17.0.2:38593\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1120)\u001b[0m 2023-03-19 08:29:26,230\tINFO tracker.py:382 -- @tracker All of 2 nodes getting started\n",
      "\u001b[2m\u001b[36m(_RemoteRayXGBoostActor pid=1229)\u001b[0m [08:29:26] task [xgboost.ray]:139753678664848 got new rank 1\n",
      "\u001b[2m\u001b[36m(_RemoteRayXGBoostActor pid=1228)\u001b[0m [08:29:26] task [xgboost.ray]:140072041780128 got new rank 0\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1120)\u001b[0m 2023-03-19 08:29:27,536\tINFO tracker.py:388 -- @tracker All nodes finishes job\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"trialProgress\">\n",
       "  <h3>Trial Progress</h3>\n",
       "  <table>\n",
       "<thead>\n",
       "<tr><th>Trial name                </th><th>date               </th><th>done  </th><th>episodes_total  </th><th>experiment_id                   </th><th style=\"text-align: right;\">  experiment_tag</th><th>hostname    </th><th style=\"text-align: right;\">  iterations_since_restore</th><th>node_ip   </th><th style=\"text-align: right;\">  pid</th><th>should_checkpoint  </th><th style=\"text-align: right;\">  time_since_restore</th><th style=\"text-align: right;\">  time_this_iter_s</th><th style=\"text-align: right;\">  time_total_s</th><th style=\"text-align: right;\">  timestamp</th><th style=\"text-align: right;\">  timesteps_since_restore</th><th>timesteps_total  </th><th style=\"text-align: right;\">  train-error</th><th style=\"text-align: right;\">  train-logloss</th><th style=\"text-align: right;\">  training_iteration</th><th style=\"text-align: right;\">   trial_id</th><th style=\"text-align: right;\">  valid-error</th><th style=\"text-align: right;\">  valid-logloss</th><th style=\"text-align: right;\">  warmup_time</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>XGBoostTrainer_25070_00000</td><td>2023-03-19_08-29-28</td><td>True  </td><td>                </td><td>fab94c57893144a28929554696a4ff72</td><td style=\"text-align: right;\">               0</td><td>b10e8c16a849</td><td style=\"text-align: right;\">                        21</td><td>172.17.0.2</td><td style=\"text-align: right;\"> 1120</td><td>True               </td><td style=\"text-align: right;\">             6.78411</td><td style=\"text-align: right;\">          0.072351</td><td style=\"text-align: right;\">       6.78411</td><td style=\"text-align: right;\"> 1679214568</td><td style=\"text-align: right;\">                        0</td><td>                 </td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">      0.0184957</td><td style=\"text-align: right;\">                  21</td><td style=\"text-align: right;\">25070_00000</td><td style=\"text-align: right;\">    0.0411765</td><td style=\"text-align: right;\">      0.0897979</td><td style=\"text-align: right;\">   0.00931907</td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "</div>\n",
       "<style>\n",
       ".trialProgress {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  color: var(--jp-ui-font-color1);\n",
       "}\n",
       ".trialProgress h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".trialProgress td {\n",
       "  white-space: nowrap;\n",
       "}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-19 08:29:28,883\tINFO tune.py:798 -- Total run time: 8.86 seconds (8.81 seconds for the tuning loop).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'train-logloss': 0.01849572784173766, 'train-error': 0.0, 'valid-logloss': 0.08979789356372374, 'valid-error': 0.04117647058823529, 'time_this_iter_s': 0.07235097885131836, 'should_checkpoint': True, 'done': True, 'timesteps_total': None, 'episodes_total': None, 'training_iteration': 21, 'trial_id': '25070_00000', 'experiment_id': 'fab94c57893144a28929554696a4ff72', 'date': '2023-03-19_08-29-28', 'timestamp': 1679214568, 'time_total_s': 6.784107208251953, 'pid': 1120, 'hostname': 'b10e8c16a849', 'node_ip': '172.17.0.2', 'config': {}, 'time_since_restore': 6.784107208251953, 'timesteps_since_restore': 0, 'iterations_since_restore': 21, 'warmup_time': 0.009319067001342773, 'experiment_tag': '0'}\n"
     ]
    }
   ],
   "source": [
    "from ray.air.config import ScalingConfig\n",
    "from ray.train.xgboost import XGBoostTrainer\n",
    "\n",
    "trainer = XGBoostTrainer(\n",
    "    scaling_config=ScalingConfig(\n",
    "        # Number of workers to use for data parallelism.\n",
    "        num_workers=2,\n",
    "        # Whether to use GPU acceleration.\n",
    "        use_gpu=False,\n",
    "        # Make sure to leave some CPUs free for Ray Data operations.\n",
    "        _max_cpu_fraction_per_node=0.9,\n",
    "    ),\n",
    "    label_column=\"target\",\n",
    "    num_boost_round=20,\n",
    "    params={\n",
    "        # XGBoost specific params\n",
    "        \"objective\": \"binary:logistic\",\n",
    "        # \"tree_method\": \"gpu_hist\",  # uncomment this to use GPUs.\n",
    "        \"eval_metric\": [\"logloss\", \"error\"],\n",
    "    },\n",
    "    datasets={\"train\": train_dataset, \"valid\": valid_dataset},\n",
    "    preprocessor=preprocessor,\n",
    ")\n",
    "result = trainer.fit()\n",
    "print(result.metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8fa95a8",
   "metadata": {},
   "source": [
    "## Tune hyperparameters and find the best model with Ray Tune "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3a2dfe6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ray import tune\n",
    "\n",
    "param_space = {\"params\": {\"max_depth\": tune.randint(1, 9)}}\n",
    "metric = \"train-logloss\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "eb75f8e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-19 08:36:33,785\tWARNING callback.py:108 -- The TensorboardX logger cannot be instantiated because either TensorboardX or one of it's dependencies is not installed. Please make sure you have the latest version of TensorboardX installed: `pip install -U tensorboardx`\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"tuneStatus\">\n",
       "  <div style=\"display: flex;flex-direction: row\">\n",
       "    <div style=\"display: flex;flex-direction: column;\">\n",
       "      <h3>Tune Status</h3>\n",
       "      <table>\n",
       "<tbody>\n",
       "<tr><td>Current time:</td><td>2023-03-19 08:36:55</td></tr>\n",
       "<tr><td>Running for: </td><td>00:00:21.97        </td></tr>\n",
       "<tr><td>Memory:      </td><td>5.2/15.6 GiB       </td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "    </div>\n",
       "    <div class=\"vDivider\"></div>\n",
       "    <div class=\"systemInfo\">\n",
       "      <h3>System Info</h3>\n",
       "      Using FIFO scheduling algorithm.<br>Resources requested: 0/8 CPUs, 0/0 GPUs, 0.0/9.16 GiB heap, 0.0/4.58 GiB objects\n",
       "    </div>\n",
       "    \n",
       "  </div>\n",
       "  <div class=\"hDivider\"></div>\n",
       "  <div class=\"trialStatus\">\n",
       "    <h3>Trial Status</h3>\n",
       "    <table>\n",
       "<thead>\n",
       "<tr><th>Trial name                </th><th>status    </th><th>loc            </th><th style=\"text-align: right;\">  params/max_depth</th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  train-logloss</th><th style=\"text-align: right;\">  train-error</th><th style=\"text-align: right;\">  valid-logloss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>XGBoostTrainer_27907_00000</td><td>TERMINATED</td><td>172.17.0.2:1715</td><td style=\"text-align: right;\">                 7</td><td style=\"text-align: right;\">    21</td><td style=\"text-align: right;\">         7.03373</td><td style=\"text-align: right;\">      0.0184957</td><td style=\"text-align: right;\">    0        </td><td style=\"text-align: right;\">      0.0897979</td></tr>\n",
       "<tr><td>XGBoostTrainer_27907_00001</td><td>TERMINATED</td><td>172.17.0.2:1769</td><td style=\"text-align: right;\">                 8</td><td style=\"text-align: right;\">    21</td><td style=\"text-align: right;\">         6.82536</td><td style=\"text-align: right;\">      0.0184957</td><td style=\"text-align: right;\">    0        </td><td style=\"text-align: right;\">      0.0897979</td></tr>\n",
       "<tr><td>XGBoostTrainer_27907_00002</td><td>TERMINATED</td><td>172.17.0.2:1771</td><td style=\"text-align: right;\">                 4</td><td style=\"text-align: right;\">    21</td><td style=\"text-align: right;\">         7.55822</td><td style=\"text-align: right;\">      0.0183889</td><td style=\"text-align: right;\">    0        </td><td style=\"text-align: right;\">      0.101024 </td></tr>\n",
       "<tr><td>XGBoostTrainer_27907_00003</td><td>TERMINATED</td><td>172.17.0.2:2529</td><td style=\"text-align: right;\">                 1</td><td style=\"text-align: right;\">    21</td><td style=\"text-align: right;\">         7.95038</td><td style=\"text-align: right;\">      0.0955215</td><td style=\"text-align: right;\">    0.0175879</td><td style=\"text-align: right;\">      0.112144 </td></tr>\n",
       "<tr><td>XGBoostTrainer_27907_00004</td><td>TERMINATED</td><td>172.17.0.2:2621</td><td style=\"text-align: right;\">                 8</td><td style=\"text-align: right;\">    21</td><td style=\"text-align: right;\">         6.04832</td><td style=\"text-align: right;\">      0.0184957</td><td style=\"text-align: right;\">    0        </td><td style=\"text-align: right;\">      0.0897979</td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "  </div>\n",
       "</div>\n",
       "<style>\n",
       ".tuneStatus {\n",
       "  color: var(--jp-ui-font-color1);\n",
       "}\n",
       ".tuneStatus .systemInfo {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus td {\n",
       "  white-space: nowrap;\n",
       "}\n",
       ".tuneStatus .trialStatus {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".tuneStatus .hDivider {\n",
       "  border-bottom-width: var(--jp-border-width);\n",
       "  border-bottom-color: var(--jp-border-color0);\n",
       "  border-bottom-style: solid;\n",
       "}\n",
       ".tuneStatus .vDivider {\n",
       "  border-left-width: var(--jp-border-width);\n",
       "  border-left-color: var(--jp-border-color0);\n",
       "  border-left-style: solid;\n",
       "  margin: 0.5em 1em 0.5em 1em;\n",
       "}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1715)\u001b[0m 2023-03-19 08:36:36,191\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[aggregate]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1715)\u001b[0m 2023-03-19 08:36:36,717\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[StandardScaler]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1715)\u001b[0m 2023-03-19 08:36:36,893\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[StandardScaler]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1715)\u001b[0m 2023-03-19 08:36:36,969\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[repartition]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1715)\u001b[0m 2023-03-19 08:36:38,100\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[repartition]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1769)\u001b[0m 2023-03-19 08:36:38,842\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[aggregate]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1769)\u001b[0m 2023-03-19 08:36:38,893\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[StandardScaler]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1769)\u001b[0m 2023-03-19 08:36:38,946\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[StandardScaler]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1771)\u001b[0m 2023-03-19 08:36:38,944\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[aggregate]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1769)\u001b[0m 2023-03-19 08:36:39,060\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[repartition]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1769)\u001b[0m 2023-03-19 08:36:39,211\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[repartition]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1771)\u001b[0m 2023-03-19 08:36:39,217\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[StandardScaler]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1771)\u001b[0m 2023-03-19 08:36:39,491\tWARNING plan.py:523 -- Warning: The Ray cluster currently does not have any available CPUs. The Dataset job will hang unless more CPUs are freed up. A common reason is that cluster resources are used by Actors or Tune trials; see the following link for more details: https://docs.ray.io/en/master/data/dataset-internals.html#datasets-and-tune\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1769)\u001b[0m 2023-03-19 08:36:39,529\tWARNING plan.py:523 -- Warning: The Ray cluster currently does not have any available CPUs. The Dataset job will hang unless more CPUs are freed up. A common reason is that cluster resources are used by Actors or Tune trials; see the following link for more details: https://docs.ray.io/en/master/data/dataset-internals.html#datasets-and-tune\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1771)\u001b[0m 2023-03-19 08:36:39,502\tWARNING plan.py:523 -- Warning: The Ray cluster currently does not have any available CPUs. The Dataset job will hang unless more CPUs are freed up. A common reason is that cluster resources are used by Actors or Tune trials; see the following link for more details: https://docs.ray.io/en/master/data/dataset-internals.html#datasets-and-tune\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1771)\u001b[0m 2023-03-19 08:36:39,503\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[StandardScaler]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1771)\u001b[0m 2023-03-19 08:36:39,677\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[repartition]\n",
      "\u001b[2m\u001b[36m(_RemoteRayXGBoostActor pid=1809)\u001b[0m 2023-03-19 08:36:40,421\tWARNING plan.py:523 -- Warning: The Ray cluster currently does not have any available CPUs. The Dataset job will hang unless more CPUs are freed up. A common reason is that cluster resources are used by Actors or Tune trials; see the following link for more details: https://docs.ray.io/en/master/data/dataset-internals.html#datasets-and-tune\n",
      "\u001b[2m\u001b[36m(_RemoteRayXGBoostActor pid=1809)\u001b[0m 2023-03-19 08:36:40,450\tWARNING plan.py:523 -- Warning: The Ray cluster currently does not have any available CPUs. The Dataset job will hang unless more CPUs are freed up. A common reason is that cluster resources are used by Actors or Tune trials; see the following link for more details: https://docs.ray.io/en/master/data/dataset-internals.html#datasets-and-tune\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1715)\u001b[0m 2023-03-19 08:36:40,634\tINFO tracker.py:218 -- start listen on 172.17.0.2:43853\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1771)\u001b[0m 2023-03-19 08:36:40,551\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[repartition]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1715)\u001b[0m 2023-03-19 08:36:40,697\tINFO tracker.py:382 -- @tracker All of 2 nodes getting started\n",
      "\u001b[2m\u001b[36m(_RemoteRayXGBoostActor pid=1809)\u001b[0m [08:36:40] task [xgboost.ray]:139696816522240 got new rank 0\n",
      "\u001b[2m\u001b[36m(_RemoteRayXGBoostActor pid=1813)\u001b[0m [08:36:40] task [xgboost.ray]:140144023909376 got new rank 1\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"trialProgress\">\n",
       "  <h3>Trial Progress</h3>\n",
       "  <table>\n",
       "<thead>\n",
       "<tr><th>Trial name                </th><th>date               </th><th>done  </th><th>episodes_total  </th><th>experiment_id                   </th><th>experiment_tag  </th><th>hostname    </th><th style=\"text-align: right;\">  iterations_since_restore</th><th>node_ip   </th><th style=\"text-align: right;\">  pid</th><th>should_checkpoint  </th><th style=\"text-align: right;\">  time_since_restore</th><th style=\"text-align: right;\">  time_this_iter_s</th><th style=\"text-align: right;\">  time_total_s</th><th style=\"text-align: right;\">  timestamp</th><th style=\"text-align: right;\">  timesteps_since_restore</th><th>timesteps_total  </th><th style=\"text-align: right;\">  train-error</th><th style=\"text-align: right;\">  train-logloss</th><th style=\"text-align: right;\">  training_iteration</th><th style=\"text-align: right;\">   trial_id</th><th style=\"text-align: right;\">  valid-error</th><th style=\"text-align: right;\">  valid-logloss</th><th style=\"text-align: right;\">  warmup_time</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>XGBoostTrainer_27907_00000</td><td>2023-03-19_08-36-43</td><td>True  </td><td>                </td><td>5d631b388306456b93e7b3c5ac0b871d</td><td>0_max_depth=7   </td><td>b10e8c16a849</td><td style=\"text-align: right;\">                        21</td><td>172.17.0.2</td><td style=\"text-align: right;\"> 1715</td><td>True               </td><td style=\"text-align: right;\">             7.03373</td><td style=\"text-align: right;\">          0.175378</td><td style=\"text-align: right;\">       7.03373</td><td style=\"text-align: right;\"> 1679215003</td><td style=\"text-align: right;\">                        0</td><td>                 </td><td style=\"text-align: right;\">    0        </td><td style=\"text-align: right;\">      0.0184957</td><td style=\"text-align: right;\">                  21</td><td style=\"text-align: right;\">27907_00000</td><td style=\"text-align: right;\">    0.0411765</td><td style=\"text-align: right;\">      0.0897979</td><td style=\"text-align: right;\">   0.0110359 </td></tr>\n",
       "<tr><td>XGBoostTrainer_27907_00001</td><td>2023-03-19_08-36-45</td><td>True  </td><td>                </td><td>75eada908e9b48ccb7d6d55bc19e4d59</td><td>1_max_depth=8   </td><td>b10e8c16a849</td><td style=\"text-align: right;\">                        21</td><td>172.17.0.2</td><td style=\"text-align: right;\"> 1769</td><td>True               </td><td style=\"text-align: right;\">             6.82536</td><td style=\"text-align: right;\">          0.79186 </td><td style=\"text-align: right;\">       6.82536</td><td style=\"text-align: right;\"> 1679215005</td><td style=\"text-align: right;\">                        0</td><td>                 </td><td style=\"text-align: right;\">    0        </td><td style=\"text-align: right;\">      0.0184957</td><td style=\"text-align: right;\">                  21</td><td style=\"text-align: right;\">27907_00001</td><td style=\"text-align: right;\">    0.0411765</td><td style=\"text-align: right;\">      0.0897979</td><td style=\"text-align: right;\">   0.0125062 </td></tr>\n",
       "<tr><td>XGBoostTrainer_27907_00002</td><td>2023-03-19_08-36-46</td><td>True  </td><td>                </td><td>15c817e68ebe4160bb70760ab928af0b</td><td>2_max_depth=4   </td><td>b10e8c16a849</td><td style=\"text-align: right;\">                        21</td><td>172.17.0.2</td><td style=\"text-align: right;\"> 1771</td><td>True               </td><td style=\"text-align: right;\">             7.55822</td><td style=\"text-align: right;\">          0.34967 </td><td style=\"text-align: right;\">       7.55822</td><td style=\"text-align: right;\"> 1679215006</td><td style=\"text-align: right;\">                        0</td><td>                 </td><td style=\"text-align: right;\">    0        </td><td style=\"text-align: right;\">      0.0183889</td><td style=\"text-align: right;\">                  21</td><td style=\"text-align: right;\">27907_00002</td><td style=\"text-align: right;\">    0.0470588</td><td style=\"text-align: right;\">      0.101024 </td><td style=\"text-align: right;\">   0.0109482 </td></tr>\n",
       "<tr><td>XGBoostTrainer_27907_00003</td><td>2023-03-19_08-36-54</td><td>True  </td><td>                </td><td>2c8b438a0e224dcda910589a73952d9e</td><td>3_max_depth=1   </td><td>b10e8c16a849</td><td style=\"text-align: right;\">                        21</td><td>172.17.0.2</td><td style=\"text-align: right;\"> 2529</td><td>True               </td><td style=\"text-align: right;\">             7.95038</td><td style=\"text-align: right;\">          0.15171 </td><td style=\"text-align: right;\">       7.95038</td><td style=\"text-align: right;\"> 1679215014</td><td style=\"text-align: right;\">                        0</td><td>                 </td><td style=\"text-align: right;\">    0.0175879</td><td style=\"text-align: right;\">      0.0955215</td><td style=\"text-align: right;\">                  21</td><td style=\"text-align: right;\">27907_00003</td><td style=\"text-align: right;\">    0.0294118</td><td style=\"text-align: right;\">      0.112144 </td><td style=\"text-align: right;\">   0.010232  </td></tr>\n",
       "<tr><td>XGBoostTrainer_27907_00004</td><td>2023-03-19_08-36-55</td><td>True  </td><td>                </td><td>ac7882ab8bbd4b428137c31b1cad868a</td><td>4_max_depth=8   </td><td>b10e8c16a849</td><td style=\"text-align: right;\">                        21</td><td>172.17.0.2</td><td style=\"text-align: right;\"> 2621</td><td>True               </td><td style=\"text-align: right;\">             6.04832</td><td style=\"text-align: right;\">          1.02244 </td><td style=\"text-align: right;\">       6.04832</td><td style=\"text-align: right;\"> 1679215015</td><td style=\"text-align: right;\">                        0</td><td>                 </td><td style=\"text-align: right;\">    0        </td><td style=\"text-align: right;\">      0.0184957</td><td style=\"text-align: right;\">                  21</td><td style=\"text-align: right;\">27907_00004</td><td style=\"text-align: right;\">    0.0411765</td><td style=\"text-align: right;\">      0.0897979</td><td style=\"text-align: right;\">   0.00957203</td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "</div>\n",
       "<style>\n",
       ".trialProgress {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  color: var(--jp-ui-font-color1);\n",
       "}\n",
       ".trialProgress h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".trialProgress td {\n",
       "  white-space: nowrap;\n",
       "}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1769)\u001b[0m 2023-03-19 08:36:42,798\tINFO tracker.py:218 -- start listen on 172.17.0.2:33999\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1769)\u001b[0m 2023-03-19 08:36:42,845\tINFO tracker.py:382 -- @tracker All of 2 nodes getting started\n",
      "\u001b[2m\u001b[36m(_RemoteRayXGBoostActor pid=2065)\u001b[0m [08:36:42] task [xgboost.ray]:139705094182272 got new rank 1\n",
      "\u001b[2m\u001b[36m(_RemoteRayXGBoostActor pid=2066)\u001b[0m [08:36:42] task [xgboost.ray]:139889656390656 got new rank 0\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1715)\u001b[0m 2023-03-19 08:36:43,013\tINFO tracker.py:388 -- @tracker All nodes finishes job\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1771)\u001b[0m 2023-03-19 08:36:43,063\tINFO tracker.py:218 -- start listen on 172.17.0.2:33733\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1771)\u001b[0m 2023-03-19 08:36:43,097\tINFO tracker.py:382 -- @tracker All of 2 nodes getting started\n",
      "\u001b[2m\u001b[36m(_RemoteRayXGBoostActor pid=2107)\u001b[0m [08:36:43] task [xgboost.ray]:140605256799376 got new rank 0\n",
      "\u001b[2m\u001b[36m(_RemoteRayXGBoostActor pid=2108)\u001b[0m [08:36:43] task [xgboost.ray]:140186808337744 got new rank 1\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1769)\u001b[0m 2023-03-19 08:36:44,539\tINFO tracker.py:388 -- @tracker All nodes finishes job\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=1771)\u001b[0m 2023-03-19 08:36:44,777\tINFO tracker.py:388 -- @tracker All nodes finishes job\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=2529)\u001b[0m 2023-03-19 08:36:46,635\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[aggregate]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=2529)\u001b[0m 2023-03-19 08:36:48,036\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[StandardScaler]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=2529)\u001b[0m 2023-03-19 08:36:48,089\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[StandardScaler]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=2529)\u001b[0m 2023-03-19 08:36:48,182\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[repartition]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=2529)\u001b[0m 2023-03-19 08:36:48,277\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[repartition]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=2621)\u001b[0m 2023-03-19 08:36:49,671\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[aggregate]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=2621)\u001b[0m 2023-03-19 08:36:50,257\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[StandardScaler]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=2621)\u001b[0m 2023-03-19 08:36:50,308\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[StandardScaler]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=2621)\u001b[0m 2023-03-19 08:36:50,401\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[repartition]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=2529)\u001b[0m 2023-03-19 08:36:51,481\tINFO tracker.py:218 -- start listen on 172.17.0.2:59231\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=2529)\u001b[0m 2023-03-19 08:36:51,528\tINFO tracker.py:382 -- @tracker All of 2 nodes getting started\n",
      "\u001b[2m\u001b[36m(_RemoteRayXGBoostActor pid=2724)\u001b[0m [08:36:51] task [xgboost.ray]:140383140320112 got new rank 1\n",
      "\u001b[2m\u001b[36m(_RemoteRayXGBoostActor pid=2723)\u001b[0m [08:36:51] task [xgboost.ray]:140542965745024 got new rank 0\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=2621)\u001b[0m 2023-03-19 08:36:51,624\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[repartition]\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=2621)\u001b[0m 2023-03-19 08:36:52,955\tINFO tracker.py:218 -- start listen on 172.17.0.2:54449\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=2621)\u001b[0m 2023-03-19 08:36:52,990\tINFO tracker.py:382 -- @tracker All of 2 nodes getting started\n",
      "\u001b[2m\u001b[36m(_RemoteRayXGBoostActor pid=3002)\u001b[0m [08:36:52] task [xgboost.ray]:139634598441696 got new rank 0\n",
      "\u001b[2m\u001b[36m(_RemoteRayXGBoostActor pid=3003)\u001b[0m [08:36:52] task [xgboost.ray]:140366510822400 got new rank 1\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=2529)\u001b[0m 2023-03-19 08:36:53,143\tINFO tracker.py:388 -- @tracker All nodes finishes job\n",
      "\u001b[2m\u001b[36m(XGBoostTrainer pid=2621)\u001b[0m 2023-03-19 08:36:54,392\tINFO tracker.py:388 -- @tracker All nodes finishes job\n",
      "2023-03-19 08:36:55,774\tINFO tune.py:798 -- Total run time: 21.99 seconds (21.95 seconds for the tuning loop).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best result: Result(metrics={'train-logloss': 0.01838890816981263, 'train-error': 0.0, 'valid-logloss': 0.10102374425212689, 'valid-error': 0.04705882352941176, 'should_checkpoint': True, 'done': True, 'trial_id': '27907_00002', 'experiment_tag': '2_max_depth=4'}, error=None, log_dir=PosixPath('/home/jovyan/ray_results/XGBoostTrainer_2023-03-19_08-36-33/XGBoostTrainer_27907_00002_2_max_depth=4_2023-03-19_08-36-36'))\n"
     ]
    }
   ],
   "source": [
    "from ray.tune.tuner import Tuner, TuneConfig\n",
    "\n",
    "tuner = Tuner(\n",
    "    trainer,\n",
    "    param_space=param_space,\n",
    "    tune_config=TuneConfig(num_samples=5, metric=metric, mode=\"min\"),\n",
    ")\n",
    "result_grid = tuner.fit()\n",
    "best_result = result_grid.get_best_result()\n",
    "print(\"Best result:\", best_result)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "107d4b86",
   "metadata": {},
   "source": [
    "## use the train model for batch prediction "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "94ea6558",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-19 08:37:07,938\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[MapBatches(<lambda>)]\n",
      "MapBatches(<lambda>): 100%|██████████| 1/1 [00:00<00:00,  1.22it/s]\n",
      "2023-03-19 08:37:08,792\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> TaskPoolMapOperator[StandardScaler]\n",
      "StandardScaler: 100%|██████████| 1/1 [00:00<00:00, 39.61it/s]\n",
      "2023-03-19 08:37:08,839\tINFO bulk_executor.py:39 -- Executing DAG InputDataBuffer[Input] -> ActorPoolMapOperator[MapBatches(ScoringWrapper)]\n",
      "MapBatches(ScoringWrapper), 0 actors: 100%|██████████| 1/1 [00:01<00:00,  1.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'predictions': 0.9966920614242554}\n",
      "{'predictions': 0.9931760430335999}\n",
      "{'predictions': 0.0034648359287530184}\n",
      "{'predictions': 0.9966920614242554}\n",
      "{'predictions': 0.9965646862983704}\n",
      "{'predictions': 0.9956005811691284}\n",
      "{'predictions': 0.9950228929519653}\n",
      "{'predictions': 0.9943311214447021}\n",
      "{'predictions': 0.4793323874473572}\n",
      "{'predictions': 0.9818810820579529}\n",
      "{'predictions': 0.0034648359287530184}\n",
      "{'predictions': 0.996193528175354}\n",
      "{'predictions': 0.9557499885559082}\n",
      "{'predictions': 0.993036687374115}\n",
      "{'predictions': 0.9940920472145081}\n",
      "{'predictions': 0.22775237262248993}\n",
      "{'predictions': 0.4834454357624054}\n",
      "{'predictions': 0.9949895739555359}\n",
      "{'predictions': 0.9798774123191833}\n",
      "{'predictions': 0.0034648359287530184}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from ray.train.batch_predictor import BatchPredictor\n",
    "from ray.train.xgboost import XGBoostPredictor\n",
    "\n",
    "# You can also create a checkpoint from a trained model using\n",
    "# `XGBoostCheckpoint.from_model`.\n",
    "checkpoint = best_result.checkpoint\n",
    "\n",
    "batch_predictor = BatchPredictor.from_checkpoint(checkpoint, XGBoostPredictor)\n",
    "\n",
    "predicted_probabilities = batch_predictor.predict(test_dataset)\n",
    "predicted_probabilities.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25b3762c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}