src/full-training-100.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pylab as plt\n",
    "\n",
    "from modules.utils import load_cifar10, load_cifar100\n",
    "from modules.cnn_with_spectral_pooling import CNN_Spectral_Pool\n",
    "\n",
    "% matplotlib inline\n",
    "% load_ext autoreload\n",
    "% autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# INSTRUCTIONS - PLEASE READ CAREFULLY\n",
    "\n",
    "### GPU Required\n",
    "\n",
    "The code will only run on a machine with a GPU since it processes images channels-first.\n",
    "\n",
    "### 32 GB RAM Required\n",
    "\n",
    "Unlike the CIFAR-10 dataset, the CIFAR-100 dataset does not come in batches. Therefore, loading the dataset requires at least 32 GB of RAM.\n",
    "\n",
    "### Unzipping model Required\n",
    "\n",
    "You need to run \n",
    "\n",
    "```tar -zxvf best_model_100.tar.gz```\n",
    "\n",
    "from the src/ directory before proceeding."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Please make sure you are running on a machine with a GPU, at least 32 GB RAM, and have already unzipped the best_model_100.tar.gz file before proceeding.\n"
     ]
    }
   ],
   "source": [
    "print('Please make sure you are running on a machine with a GPU, at least 32 GB RAM, and have already unzipped the best_model_100.tar.gz file before proceeding.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download CIFAR data, if necessary, and load it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file already downloaded..\n"
     ]
    }
   ],
   "source": [
    "xtrain, ytrain, xtest, ytest = load_cifar100(get_test_data=True, channels_last=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((50000, 3, 32, 32), (50000,), (10000, 3, 32, 32), (10000,))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xtrain.shape, ytrain.shape, xtest.shape, ytest.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Best result from manual tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building tf graph...\n",
      "Adding conv layer 1 | Input size: 32 | Input channels: 3 | #filters: 128 | filter size: 3\n",
      "Adding spectral pool layer 1 | Input size: 32 | filter size: (25,25) | Freq Dropout Bounds: (3.4125,13.0)\n",
      "Adding conv layer 2 | Input size: 25 | Input channels: 128 | #filters: 160 | filter size: 3\n",
      "Adding spectral pool layer 2 | Input size: 25 | filter size: (19,19) | Freq Dropout Bounds: (2.25,10.0)\n",
      "Adding conv layer 3 | Input size: 19 | Input channels: 160 | #filters: 192 | filter size: 3\n",
      "Adding spectral pool layer 3 | Input size: 19 | filter size: (15,15) | Freq Dropout Bounds: (1.5,8.0)\n",
      "Adding conv layer 4 | Input size: 15 | Input channels: 192 | #filters: 224 | filter size: 3\n",
      "Adding spectral pool layer 4 | Input size: 15 | filter size: (11,11) | Freq Dropout Bounds: (0.8999999999999999,6.0)\n",
      "Adding conv layer 5 | Input size: 11 | Input channels: 224 | #filters: 224 | filter size: 1\n",
      "Adding conv layer 6 | Input size: 11 | Input channels: 224 | #filters: 100 | filter size: 1\n",
      "Adding final softmax layer using global averaging\n",
      "(?, 100)\n",
      "number of batches for testing: 20\n",
      "Loading pre-trained model\n",
      "INFO:tensorflow:Restoring parameters from model/best_model_cifar_100/test_1513439725.3974495\n",
      "Test accuracy: 48.170\n"
     ]
    }
   ],
   "source": [
    "# Manual tuning\n",
    "tf.reset_default_graph()\n",
    "cnn = CNN_Spectral_Pool(M=4,\n",
    "                        verbose=True,\n",
    "                        num_output=100,\n",
    "                        gamma=0.79)\n",
    "\n",
    "cnn.calc_test_accuracy(xtest, ytest, 'best_model_cifar_100/test_1513439725.3974495')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Only rerun the rest of the notebook to train from scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from modules.cnn_with_spectral_pooling import CNN_Spectral_Pool\n",
    "validation_size = 4096\n",
    "\n",
    "cnn = CNN_Spectral_Pool(M=7,\n",
    "                        num_output=100,\n",
    "                        verbose=True,\n",
    "                        learning_rate=1.09e-3,\n",
    "                        l2_norm=2.7e-5,\n",
    "                        lr_reduction_factor=0.5,\n",
    "                        lr_reduction_epochs=[11,21,31,41],\n",
    "                        gamma=0.6336)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building tf graph...\n",
      "Adding conv layer 1 | Input size: 32 | Input channels: 3 | #filters: 128 | filter size: 3\n",
      "Adding spectral pool layer 1 | Input size: 32 | filter size: (20,20) | Freq Dropout Bounds: (3.0642857142857145,11.0)\n",
      "Adding conv layer 2 | Input size: 20 | Input channels: 128 | #filters: 160 | filter size: 3\n",
      "Adding spectral pool layer 2 | Input size: 20 | filter size: (12,12) | Freq Dropout Bounds: (1.7999999999999998,7.0)\n",
      "Adding conv layer 3 | Input size: 12 | Input channels: 160 | #filters: 192 | filter size: 3\n",
      "Adding spectral pool layer 3 | Input size: 12 | filter size: (7,7) | Freq Dropout Bounds: (0.9428571428571428,4.0)\n",
      "Adding conv layer 4 | Input size: 7 | Input channels: 192 | #filters: 224 | filter size: 3\n",
      "Adding spectral pool layer 4 | Input size: 7 | filter size: (4,4) | Freq Dropout Bounds: (0.6428571428571429,3.0)\n",
      "Adding conv layer 5 | Input size: 4 | Input channels: 224 | #filters: 256 | filter size: 3\n",
      "Adding spectral pool layer 5 | Input size: 4 | filter size: (3,3) | Freq Dropout Bounds: (0.3857142857142857,2.0)\n",
      "Adding conv layer 6 | Input size: 3 | Input channels: 256 | #filters: 288 | filter size: 3\n",
      "Adding spectral pool layer 6 | Input size: 3 | filter size: (3,3) | Freq Dropout Bounds: (0.34285714285714286,2.0)\n",
      "Adding conv layer 7 | Input size: 3 | Input channels: 288 | #filters: 288 | filter size: 3\n",
      "Adding spectral pool layer 7 | Input size: 3 | filter size: (3,3) | Freq Dropout Bounds: (0.3,2.0)\n",
      "Adding conv layer 8 | Input size: 3 | Input channels: 288 | #filters: 288 | filter size: 1\n",
      "Adding conv layer 9 | Input size: 3 | Input channels: 288 | #filters: 100 | filter size: 1\n",
      "Adding final softmax layer using global averaging\n",
      "(?, 100)\n",
      "number of batches for training: 179 validation: 16\n",
      "training epoch 1 \n",
      "45824/45904 loss: 4.520456790924072 | training accuracy: 2.115% | validation accuracy : 3.320%\n",
      "\n",
      "\tBest validation accuracy! iteration:179 accuracy: 3.3203125%\n",
      "\n",
      "training epoch 2 \n",
      "45824/45904 loss: 4.092227935791016 | training accuracy: 5.781% | validation accuracy : 7.520%\n",
      "\n",
      "\tBest validation accuracy! iteration:358 accuracy: 7.51953125%\n",
      "\n",
      "training epoch 3 \n",
      "45824/45904 loss: 3.858351707458496 | training accuracy: 8.884% | validation accuracy : 9.253%\n",
      "\n",
      "\tBest validation accuracy! iteration:537 accuracy: 9.2529296875%\n",
      "\n",
      "training epoch 4 \n",
      "45824/45904 loss: 3.6939642429351807 | training accuracy: 11.511% | validation accuracy : 12.915%\n",
      "\n",
      "\tBest validation accuracy! iteration:716 accuracy: 12.9150390625%\n",
      "\n",
      "training epoch 5 \n",
      "45824/45904 loss: 3.530968427658081 | training accuracy: 14.514% | validation accuracy : 16.382%\n",
      "\n",
      "\tBest validation accuracy! iteration:895 accuracy: 16.3818359375%\n",
      "\n",
      "training epoch 6 \n",
      "45824/45904 loss: 3.412588357925415 | training accuracy: 16.779% | validation accuracy : 19.043%\n",
      "\n",
      "\tBest validation accuracy! iteration:1074 accuracy: 19.04296875%\n",
      "\n",
      "training epoch 7 \n",
      "45824/45904 loss: 3.291534662246704 | training accuracy: 18.911% | validation accuracy : 20.557%\n",
      "\n",
      "\tBest validation accuracy! iteration:1253 accuracy: 20.556640625%\n",
      "\n",
      "training epoch 8 \n",
      "45824/45904 loss: 3.178992509841919 | training accuracy: 21.161% | validation accuracy : 21.826%\n",
      "\n",
      "\tBest validation accuracy! iteration:1432 accuracy: 21.826171875%\n",
      "\n",
      "training epoch 9 \n",
      "45824/45904 loss: 3.1168339252471924 | training accuracy: 22.368% | validation accuracy : 22.461%\n",
      "\n",
      "\tBest validation accuracy! iteration:1611 accuracy: 22.4609375%\n",
      "\n",
      "training epoch 10 \n",
      "45824/45904 loss: 3.0122599601745605 | training accuracy: 24.297% | validation accuracy : 22.998%\n",
      "\n",
      "\tBest validation accuracy! iteration:1790 accuracy: 22.998046875%\n",
      "\n",
      "training epoch 11 \n",
      "\tLearning rate reduced to 5.4500e-04 at epoch 11\n",
      "45824/45904 loss: 2.8512001037597656 | training accuracy: 27.658% | validation accuracy : 26.025%\n",
      "\n",
      "\tBest validation accuracy! iteration:1969 accuracy: 26.025390625%\n",
      "\n",
      "training epoch 12 \n",
      "45824/45904 loss: 2.779961347579956 | training accuracy: 28.832% | validation accuracy : 27.026%\n",
      "\n",
      "\tBest validation accuracy! iteration:2148 accuracy: 27.0263671875%\n",
      "\n",
      "training epoch 13 \n",
      "45824/45904 loss: 2.7145726680755615 | training accuracy: 30.021% | validation accuracy : 27.466%\n",
      "\n",
      "\tBest validation accuracy! iteration:2327 accuracy: 27.4658203125%\n",
      "\n",
      "training epoch 14 \n",
      "45824/45904 loss: 2.6201624870300293 | training accuracy: 32.149% | validation accuracy : 28.516%\n",
      "\n",
      "\tBest validation accuracy! iteration:2506 accuracy: 28.515625%\n",
      "\n",
      "training epoch 15 \n",
      "45824/45904 loss: 2.6028361320495605 | training accuracy: 32.490% | validation accuracy : 28.833%\n",
      "\n",
      "\tBest validation accuracy! iteration:2685 accuracy: 28.8330078125%\n",
      "\n",
      "training epoch 16 \n",
      "45824/45904 loss: 2.514836311340332 | training accuracy: 34.517% | validation accuracy : 29.126%\n",
      "\n",
      "\tBest validation accuracy! iteration:2864 accuracy: 29.1259765625%\n",
      "\n",
      "training epoch 17 \n",
      "45824/45904 loss: 2.476057767868042 | training accuracy: 35.121% | validation accuracy : 28.931%\n",
      "training epoch 18 \n",
      "45824/45904 loss: 2.4203014373779297 | training accuracy: 36.276% | validation accuracy : 28.833%\n",
      "training epoch 19 \n",
      "45824/45904 loss: 2.38010573387146 | training accuracy: 37.094% | validation accuracy : 29.736%\n",
      "\n",
      "\tBest validation accuracy! iteration:3401 accuracy: 29.736328125%\n",
      "\n",
      "training epoch 20 \n",
      "45824/45904 loss: 2.312600612640381 | training accuracy: 38.729% | validation accuracy : 29.980%\n",
      "\n",
      "\tBest validation accuracy! iteration:3580 accuracy: 29.98046875%\n",
      "\n",
      "training epoch 21 \n",
      "\tLearning rate reduced to 2.7250e-04 at epoch 21\n",
      "45824/45904 loss: 2.266238212585449 | training accuracy: 39.872% | validation accuracy : 30.347%\n",
      "\n",
      "\tBest validation accuracy! iteration:3759 accuracy: 30.3466796875%\n",
      "\n",
      "training epoch 22 \n",
      "45824/45904 loss: 2.1767210960388184 | training accuracy: 41.937% | validation accuracy : 31.323%\n",
      "\n",
      "\tBest validation accuracy! iteration:3938 accuracy: 31.3232421875%\n",
      "\n",
      "training epoch 23 \n",
      "45824/45904 loss: 2.1159002780914307 | training accuracy: 43.152% | validation accuracy : 31.226%\n",
      "training epoch 24 \n",
      "45824/45904 loss: 2.074190616607666 | training accuracy: 44.101% | validation accuracy : 31.738%\n",
      "\n",
      "\tBest validation accuracy! iteration:4296 accuracy: 31.73828125%\n",
      "\n",
      "training epoch 25 \n",
      "45824/45904 loss: 2.0260658264160156 | training accuracy: 45.262% | validation accuracy : 31.909%\n",
      "\n",
      "\tBest validation accuracy! iteration:4475 accuracy: 31.9091796875%\n",
      "\n",
      "training epoch 26 \n",
      "45824/45904 loss: 1.9809658527374268 | training accuracy: 46.415% | validation accuracy : 32.324%\n",
      "\n",
      "\tBest validation accuracy! iteration:4654 accuracy: 32.32421875%\n",
      "\n",
      "training epoch 27 \n",
      "45824/45904 loss: 1.9285094738006592 | training accuracy: 47.573% | validation accuracy : 32.031%\n",
      "training epoch 28 \n",
      "45824/45904 loss: 1.898651123046875 | training accuracy: 48.141% | validation accuracy : 32.202%\n",
      "training epoch 29 \n",
      "45824/45904 loss: 1.900774598121643 | training accuracy: 48.115% | validation accuracy : 32.422%\n",
      "\n",
      "\tBest validation accuracy! iteration:5191 accuracy: 32.421875%\n",
      "\n",
      "training epoch 30 \n",
      "45824/45904 loss: 1.8484783172607422 | training accuracy: 49.265% | validation accuracy : 31.982%\n",
      "training epoch 31 \n",
      "\tLearning rate reduced to 1.3625e-04 at epoch 31\n",
      "45824/45904 loss: 1.836740493774414 | training accuracy: 49.895% | validation accuracy : 32.471%\n",
      "\n",
      "\tBest validation accuracy! iteration:5549 accuracy: 32.470703125%\n",
      "\n",
      "training epoch 32 \n",
      "45824/45904 loss: 1.745876669883728 | training accuracy: 51.772% | validation accuracy : 32.422%\n",
      "training epoch 33 \n",
      "45824/45904 loss: 1.7236874103546143 | training accuracy: 52.440% | validation accuracy : 32.837%\n",
      "\n",
      "\tBest validation accuracy! iteration:5907 accuracy: 32.8369140625%\n",
      "\n",
      "training epoch 34 \n",
      "45824/45904 loss: 1.6608601808547974 | training accuracy: 53.937% | validation accuracy : 32.715%\n",
      "training epoch 35 \n",
      "45824/45904 loss: 1.6183983087539673 | training accuracy: 55.045% | validation accuracy : 32.617%\n",
      "training epoch 36 \n",
      "45824/45904 loss: 1.6207776069641113 | training accuracy: 54.884% | validation accuracy : 32.568%\n",
      "training epoch 37 \n",
      "45824/45904 loss: 1.559953212738037 | training accuracy: 56.521% | validation accuracy : 32.568%\n",
      "training epoch 38 \n",
      "45824/45904 loss: 1.5400631427764893 | training accuracy: 56.972% | validation accuracy : 32.764%\n",
      "training epoch 39 \n",
      "45824/45904 loss: 1.5240812301635742 | training accuracy: 57.498% | validation accuracy : 32.397%\n",
      "training epoch 40 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "45824/45904 loss: 1.5056647062301636 | training accuracy: 57.922% | validation accuracy : 32.666%\n",
      "training epoch 41 \n",
      "\tLearning rate reduced to 6.8125e-05 at epoch 41\n",
      "45824/45904 loss: 1.5088940858840942 | training accuracy: 57.893% | validation accuracy : 32.690%\n",
      "training epoch 42 \n",
      "45824/45904 loss: 1.4472187757492065 | training accuracy: 59.565% | validation accuracy : 33.154%\n",
      "\n",
      "\tBest validation accuracy! iteration:7518 accuracy: 33.154296875%\n",
      "\n",
      "training epoch 43 \n",
      "45824/45904 loss: 1.3882346153259277 | training accuracy: 60.846% | validation accuracy : 32.739%\n",
      "training epoch 44 \n",
      "45824/45904 loss: 1.4145677089691162 | training accuracy: 60.278% | validation accuracy : 32.593%\n",
      "training epoch 45 \n",
      "45824/45904 loss: 1.4203919172286987 | training accuracy: 60.324% | validation accuracy : 32.812%\n",
      "training epoch 46 \n",
      "45824/45904 loss: 1.4002615213394165 | training accuracy: 60.488% | validation accuracy : 32.788%\n",
      "training epoch 47 \n",
      "45824/45904 loss: 1.3819750547409058 | training accuracy: 61.343% | validation accuracy : 32.446%\n",
      "training epoch 48 \n",
      "45824/45904 loss: 1.3736293315887451 | training accuracy: 61.666% | validation accuracy : 32.666%\n",
      "training epoch 49 \n",
      "45824/45904 loss: 1.3127983808517456 | training accuracy: 63.192% | validation accuracy : 32.642%\n",
      "training epoch 50 \n",
      "45824/45904 loss: 1.35208261013031 | training accuracy: 62.210% | validation accuracy : 32.446%\n",
      "Best validation accuracy: 33.154%; Model name: 'optimal_hyperparams/optimal_hyperparams_1513542042.6826596'.\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "cnn.train(xtrain[:-validation_size],\n",
    "          ytrain[:-validation_size],\n",
    "          xtrain[-validation_size:],\n",
    "          ytrain[-validation_size:],\n",
    "          batch_size=256,\n",
    "          epochs=50,\n",
    "          extra_conv_layer=True,\n",
    "          use_global_averaging=True,\n",
    "          model_name='optimal_hyperparams'\n",
    ")\n",
    "full_model_name = cnn.full_model_name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate accuracy on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building tf graph...\n",
      "Adding conv layer 1 | Input size: 32 | Input channels: 3 | #filters: 128 | filter size: 3\n",
      "Adding spectral pool layer 1 | Input size: 32 | filter size: (20,20) | Freq Dropout Bounds: (3.0642857142857145,11.0)\n",
      "Adding conv layer 2 | Input size: 20 | Input channels: 128 | #filters: 160 | filter size: 3\n",
      "Adding spectral pool layer 2 | Input size: 20 | filter size: (12,12) | Freq Dropout Bounds: (1.7999999999999998,7.0)\n",
      "Adding conv layer 3 | Input size: 12 | Input channels: 160 | #filters: 192 | filter size: 3\n",
      "Adding spectral pool layer 3 | Input size: 12 | filter size: (7,7) | Freq Dropout Bounds: (0.9428571428571428,4.0)\n",
      "Adding conv layer 4 | Input size: 7 | Input channels: 192 | #filters: 224 | filter size: 3\n",
      "Adding spectral pool layer 4 | Input size: 7 | filter size: (4,4) | Freq Dropout Bounds: (0.6428571428571429,3.0)\n",
      "Adding conv layer 5 | Input size: 4 | Input channels: 224 | #filters: 256 | filter size: 3\n",
      "Adding spectral pool layer 5 | Input size: 4 | filter size: (3,3) | Freq Dropout Bounds: (0.3857142857142857,2.0)\n",
      "Adding conv layer 6 | Input size: 3 | Input channels: 256 | #filters: 288 | filter size: 3\n",
      "Adding spectral pool layer 6 | Input size: 3 | filter size: (3,3) | Freq Dropout Bounds: (0.34285714285714286,2.0)\n",
      "Adding conv layer 7 | Input size: 3 | Input channels: 288 | #filters: 288 | filter size: 3\n",
      "Adding spectral pool layer 7 | Input size: 3 | filter size: (3,3) | Freq Dropout Bounds: (0.3,2.0)\n",
      "Adding conv layer 8 | Input size: 3 | Input channels: 288 | #filters: 288 | filter size: 1\n",
      "Adding conv layer 9 | Input size: 3 | Input channels: 288 | #filters: 100 | filter size: 1\n",
      "Adding final softmax layer using global averaging\n",
      "(?, 100)\n",
      "number of batches for testing: 20\n",
      "Loading pre-trained model\n",
      "INFO:tensorflow:Restoring parameters from model/optimal_hyperparams/optimal_hyperparams_1513542042.6826596\n",
      "Test accuracy: 33.320\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "cnn.calc_test_accuracy(xtest, ytest, 'optimal_hyperparams/{0}'.format(full_model_name))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}