demos_jupyter_notebooks/convnets_cifar100/trainer_cifar100_shufflenetv2_labelsmoothing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## Import dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "\n",
    "import pandas\n",
    "import torch\n",
    "\n",
    "from proselflc.trainer.trainer_cnn_vision_derivedgrad import Trainer\n",
    "\n",
    "import pprint\n",
    "print = pprint.PrettyPrinter(indent=4).pprint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add params configurations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{   'batch_accumu_steps': 10,\n",
      "    'batch_size': 128,\n",
      "    'classes_per_batch': 64,\n",
      "    'counter': 'iteration',\n",
      "    'data_name': 'cifar100',\n",
      "    'device': 'gpu',\n",
      "    'epsilon': 0.25,\n",
      "    'eval_interval': 500,\n",
      "    'gamma': 0.1,\n",
      "    'logit_soften_T': None,\n",
      "    'loss_name': 'labelsmoothing',\n",
      "    'lr': 0.2,\n",
      "    'lr_scheduler': 'WarmupMultiStepSchedule',\n",
      "    'milestones': [20000, 30000],\n",
      "    'momentum': 0.9,\n",
      "    'network_name': 'shufflenetv2',\n",
      "    'num_classes': 100,\n",
      "    'num_workers': 8,\n",
      "    'sampler': 'BalancedBatchSampler',\n",
      "    'symmetric_noise_rate': 0.4,\n",
      "    'total_epochs': 100,\n",
      "    'warmup_epochs': 0,\n",
      "    'weight_decay': 0.001}\n"
     ]
    }
   ],
   "source": [
    "params = {}\n",
    "params.update(\n",
    "    {\n",
    "        \"data_name\": \"cifar100\",\n",
    "        \"num_classes\": 100,  # 1000\n",
    "        \"device\": \"gpu\",\n",
    "        #\n",
    "        \"num_workers\": 8,\n",
    "        #\n",
    "        \"counter\": \"iteration\",\n",
    "        \"classes_per_batch\": 8,\n",
    "        #\n",
    "        # To get deterministic results, please uncomment the line below\n",
    "        # to set seed parameter.\n",
    "        # \"seed\": 123,\n",
    "    }\n",
    ")\n",
    "\n",
    "# data\n",
    "params[\"symmetric_noise_rate\"] = 0.4\n",
    "\n",
    "# network\n",
    "params[\"network_name\"] = \"shufflenetv2\"\n",
    "\n",
    "# for the demo purpose, I set the total epochs to be small.\n",
    "params[\"total_epochs\"] = 100\n",
    "params[\"eval_interval\"] = 500 # iterations\n",
    "\n",
    "# batch\n",
    "params[\"batch_size\"] = 128\n",
    "params[\"sampler\"] = \"BalancedBatchSampler\"\n",
    "params[\"classes_per_batch\"] = 64\n",
    "\n",
    "# learning rate\n",
    "params[\"lr\"] = 0.2\n",
    "params[\"weight_decay\"] = 1e-3\n",
    "params[\"lr_scheduler\"] = \"WarmupMultiStepSchedule\"\n",
    "params[\"warmup_epochs\"] = 0\n",
    "params[\"milestones\"] = [20000, 30000]\n",
    "\n",
    "# optimisation\n",
    "params[\"momentum\"] = 0.9\n",
    "params[\"batch_accumu_steps\"] = 10\n",
    "params[\"gamma\"] = 0.1\n",
    "\n",
    "# loss settings\n",
    "params[\"loss_name\"] = \"labelsmoothing\"\n",
    "params[\"logit_soften_T\"] = None\n",
    "params[\"epsilon\"] = 0.25 # can also try 0.2, 0.5, etc\n",
    "\n",
    "print(params)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Extra configurations to get deterministic results"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_VISIBLE_DEVICES=0\n",
      "env: CUBLAS_WORKSPACE_CONFIG=:4096:8\n"
     ]
    }
   ],
   "source": [
    "%env CUDA_VISIBLE_DEVICES=0\n",
    "%env CUBLAS_WORKSPACE_CONFIG=:4096:8\n",
    "params[\"seed\"] = 123"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Create the folder to store intermediate and final results\n",
    "* First, run `sudo mkdir /home/proselflc_experiments/ && sudo chmod -R 777 /home/proselflc_experiments/` so that so you have the write permission.\n",
    "* Or set `WORK_DIR = \"/home/your_username/proselflc_experiments/\"`"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "WORK_DIR = \"/home/proselflc_experiments/\"\n",
    "#\n",
    "# use the time as a unique experiment identifier\n",
    "dt_string = time.strftime(\"%Y%m%d-%H%M%S\")\n",
    "summary_writer_dir = (\n",
    "    params[\"loss_name\"]\n",
    "    + \"_\"\n",
    "    + dt_string\n",
    ")\n",
    "params[\"summary_writer_dir\"] = (\n",
    "    WORK_DIR\n",
    "    + \"/\"\n",
    "    + params[\"data_name\"]\n",
    "    + \"_symmetric_noise_rate_\"\n",
    "    + str(params[\"symmetric_noise_rate\"])\n",
    "    + \"/\"\n",
    "    + params[\"network_name\"]\n",
    "    + \"/\"\n",
    "    + summary_writer_dir\n",
    ")\n",
    "if not os.path.exists(params[\"summary_writer_dir\"]):\n",
    "    os.makedirs(params[\"summary_writer_dir\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Init the trainer, and store the params configurations\n",
    "For each experiment, we have one unique result folder to store the params configurations and learning curves.\n",
    "Therefore, you can revisit any specific experiment whenever you need without losing any details."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(params=params)\n",
    "# params[\"milestones\"] was a list of integers, we convert it to a string before sinking.\n",
    "params[\"milestones\"] = str(params[\"milestones\"])\n",
    "dataframe = pandas.DataFrame(params, index=[0])\n",
    "dataframe.to_csv(\n",
    "    params[\"summary_writer_dir\"] + \"/params.csv\",\n",
    "    encoding=\"utf-8\",\n",
    "    index=False,\n",
    "    sep=\"\\t\",\n",
    "    mode=\"w\",  #\n",
    ")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Run the trainer and save the final model"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating Network.....\n",
      "Iteration= (500,2)/(39000, 100), lr=0.2000, batch_mean_epsilon=-1.0000, valid batch size=-1.0000, batch_mean_gtrust=-1.0000, batch_mean_etrust=-1.0000\n",
      "clean_test: Loss= 4.3131, Accuracy= 0.0512, Entropy= 0.9701, Max_p= 0.0320,\n",
      "noisy_subset: Loss= 4.7096, Accuracy= 0.0095, Entropy= 0.9752, Max_p= 0.0293,\n",
      "clean_subset: Loss= 4.3329, Accuracy= 0.0500, Entropy= 0.9750, Max_p= 0.0295,\n",
      "cleaned_noisy_subset: Loss= 4.3421, Accuracy= 0.0473, Entropy= 0.9753, Max_p= 0.0292,\n",
      "Evaluating Network.....\n",
      "Iteration= (1000,3)/(39000, 100), lr=0.2000, batch_mean_epsilon=-1.0000, valid batch size=-1.0000, batch_mean_gtrust=-1.0000, batch_mean_etrust=-1.0000\n",
      "clean_test: Loss= 4.0713, Accuracy= 0.1026, Entropy= 0.9564, Max_p= 0.0491,\n",
      "noisy_subset: Loss= 4.7364, Accuracy= 0.0101, Entropy= 0.9638, Max_p= 0.0428,\n",
      "clean_subset: Loss= 4.0994, Accuracy= 0.0999, Entropy= 0.9638, Max_p= 0.0429,\n",
      "cleaned_noisy_subset: Loss= 4.1158, Accuracy= 0.0972, Entropy= 0.9639, Max_p= 0.0428,\n",
      "Evaluating Network.....\n",
      "Iteration= (1500,4)/(39000, 100), lr=0.2000, batch_mean_epsilon=-1.0000, valid batch size=-1.0000, batch_mean_gtrust=-1.0000, batch_mean_etrust=-1.0000\n",
      "clean_test: Loss= 3.8660, Accuracy= 0.1368, Entropy= 0.9403, Max_p= 0.0641,\n",
      "noisy_subset: Loss= 4.7688, Accuracy= 0.0092, Entropy= 0.9509, Max_p= 0.0543,\n",
      "clean_subset: Loss= 3.8971, Accuracy= 0.1382, Entropy= 0.9502, Max_p= 0.0549,\n",
      "cleaned_noisy_subset: Loss= 3.9328, Accuracy= 0.1308, Entropy= 0.9510, Max_p= 0.0543,\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "Input \u001B[0;32mIn [6]\u001B[0m, in \u001B[0;36m<cell line: 1>\u001B[0;34m()\u001B[0m\n\u001B[0;32m----> 1\u001B[0m \u001B[43mtrainer\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtrain\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m      2\u001B[0m torch\u001B[38;5;241m.\u001B[39msave(\n\u001B[1;32m      3\u001B[0m     trainer\u001B[38;5;241m.\u001B[39mnetwork,\n\u001B[1;32m      4\u001B[0m     params[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124msummary_writer_dir\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;241m+\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m/model.pt\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m      5\u001B[0m )\n\u001B[1;32m      6\u001B[0m \u001B[38;5;28mprint\u001B[39m(\n\u001B[1;32m      7\u001B[0m     \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mThe experiment is finished with details sinked in \u001B[39m\u001B[38;5;132;01m{}\u001B[39;00m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;241m.\u001B[39mformat(\n\u001B[1;32m      8\u001B[0m         params[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124msummary_writer_dir\u001B[39m\u001B[38;5;124m\"\u001B[39m]\n\u001B[1;32m      9\u001B[0m     )\n\u001B[1;32m     10\u001B[0m )\n",
      "File \u001B[0;32m~/.local/share/virtualenvs/ProSelfLC_tpami2release-X5r1NEDs/lib/python3.8/site-packages/torch/utils/data/dataloader.py:518\u001B[0m, in \u001B[0;36m_BaseDataLoaderIter.__next__\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m    517\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m__next__\u001B[39m(\u001B[38;5;28mself\u001B[39m) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m Any:\n\u001B[0;32m--> 518\u001B[0m     \u001B[38;5;28;01mwith\u001B[39;00m torch\u001B[38;5;241m.\u001B[39mautograd\u001B[38;5;241m.\u001B[39mprofiler\u001B[38;5;241m.\u001B[39mrecord_function(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_profile_name):\n\u001B[1;32m    519\u001B[0m         \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_sampler_iter \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m    520\u001B[0m             \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_reset()\n",
      "File \u001B[0;32m~/.local/share/virtualenvs/ProSelfLC_tpami2release-X5r1NEDs/lib/python3.8/site-packages/torch/autograd/profiler.py:614\u001B[0m, in \u001B[0;36mrecord_function.__enter__\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m    613\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m__enter__\u001B[39m(\u001B[38;5;28mself\u001B[39m):\n\u001B[0;32m--> 614\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mhandle \u001B[38;5;241m=\u001B[39m \u001B[43mtorch\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mops\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mprofiler\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_record_function_enter\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mname\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    615\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\n",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "source": [
    "trainer.train()\n",
    "torch.save(\n",
    "    trainer.network,\n",
    "    params[\"summary_writer_dir\"] + \"/model.pt\",\n",
    ")\n",
    "print(\n",
    "    \"The experiment is finished with details sinked in {}\".format(\n",
    "        params[\"summary_writer_dir\"]\n",
    "    )\n",
    ")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}