BME_Project_3_Model_Compression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 124
    },
    "colab_type": "code",
    "id": "4YdIxPa3_INq",
    "outputId": "19979dbf-d5e2-452f-b669-0af1278cec0c"
   },
   "outputs": [],
   "source": [
    "# execute once to install the correct packages and their versions\n",
    "# ! pip uninstall -y tensorflow\n",
    "# ! pip uninstall -y tf-nightly\n",
    "# ! pip install -q -U tensorflow-gpu=1.14.0\n",
    "\n",
    "# ! pip install -q tensorflow-model-optimization\n",
    "\n",
    "# ! pip uninstall -y keras==2.2.4\n",
    "# ! pip install -q keras==2.1.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 70
    },
    "colab_type": "code",
    "id": "P69mYiAb_Qan",
    "outputId": "47fc6341-cf3e-4f4c-d15b-481594ad4477"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2.4-tf\n",
      "1.14.0\n"
     ]
    }
   ],
   "source": [
    "# packages preloading\n",
    "## packages for dataset and image preprocessing\n",
    "from tensorflow.keras.datasets import cifar100\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.cluster import KMeans\n",
    "\n",
    "\n",
    "## packages for modeling\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from keras import backend as K\n",
    "from keras import regularizers\n",
    "from keras.objectives import categorical_crossentropy\n",
    "import math\n",
    "\n",
    "## packages for pruning\n",
    "import tensorflow_model_optimization as tfmot\n",
    "from tensorflow_model_optimization.sparsity import keras as sparsity\n",
    "\n",
    "## visualization & file saving\n",
    "from matplotlib import pyplot as plt\n",
    "import pickle \n",
    "import numpy as np\n",
    "import os, sys\n",
    "\n",
    "\n",
    "## version checking\n",
    "print (keras.__version__)\n",
    "print (tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 239
    },
    "colab_type": "code",
    "id": "nqGo9McEVKTz",
    "outputId": "7b6231fe-c63b-4494-e692-7aaf2b8f4e68"
   },
   "outputs": [],
   "source": [
    "# Identify GPU \n",
    "tf.config.experimental.list_physical_devices('GPU') \n",
    "# Assign GPU\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] ='0'\n",
    "#CONFIG MEMORY USAGE\n",
    "config = tf.ConfigProto() #device_count = {'GPU':1} \n",
    "config.gpu_options.allow_growth = True\n",
    "session = tf.Session(config=config)\n",
    "K.set_session(session)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zh7YhVcAZQF9"
   },
   "source": [
    "## Loading Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "U8Rc9C2-saqX"
   },
   "outputs": [],
   "source": [
    "# function for data augmentation\n",
    "def getDataGenerator(train_phase):\n",
    "    if train_phase == True:\n",
    "        datagen = ImageDataGenerator(\n",
    "        rotation_range=0.,\n",
    "        width_shift_range=0.05,\n",
    "        height_shift_range=0.05,\n",
    "        shear_range=0.05,\n",
    "        zoom_range=0.05,\n",
    "        channel_shift_range=0.,\n",
    "        fill_mode='nearest',\n",
    "        horizontal_flip=True,\n",
    "        vertical_flip=False,\n",
    "        rescale=None)\n",
    "    else: \n",
    "        datagen = ImageDataGenerator(\n",
    "        rescale=None\n",
    "        )\n",
    "\n",
    "    return datagen\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-aKVl68F9t8H"
   },
   "source": [
    "### - Cifar100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1JHYenFH9_sU"
   },
   "outputs": [],
   "source": [
    "# define DenseNet parms Cifar10\n",
    "nb_classes = 100\n",
    "img_dim = (32,32,3)\n",
    "\n",
    "batch_size = 64\n",
    "nb_epoch = 40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lVSlrB859w_a"
   },
   "outputs": [],
   "source": [
    "# datasets with data augmentation\n",
    "(x_train,y_train),(x_test,y_test) = cifar100.load_data()\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "y_train = keras.utils.to_categorical(y_train, nb_classes)\n",
    "y_test= keras.utils.to_categorical(y_test, nb_classes)\n",
    "\n",
    "# hold out 5000 samples from training for validation\n",
    "x_tr,x_val,y_tr,y_val = train_test_split(x_train,y_train,test_size=0.1, random_state=42)\n",
    "\n",
    "train_datagen = getDataGenerator(train_phase=True)\n",
    "train_datagen = train_datagen.flow(x_train,y_train,batch_size = batch_size)\n",
    "\n",
    "validation_datagen = getDataGenerator(train_phase=False)\n",
    "validation_datagen = validation_datagen.flow(x_val,y_val,batch_size = batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 88
    },
    "colab_type": "code",
    "id": "jEyOf-gt99lN",
    "outputId": "e5642b8b-cab3-48da-faa6-57c6b55087f6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 32, 32, 3) (50000, 100)\n",
      "(45000, 32, 32, 3) (45000, 100)\n",
      "(5000, 32, 32, 3) (5000, 100)\n",
      "(10000, 32, 32, 3) (10000, 100)\n"
     ]
    }
   ],
   "source": [
    "print(x_train.shape,y_train.shape)\n",
    "print(x_tr.shape,y_tr.shape)\n",
    "print(x_val.shape,y_val.shape)\n",
    "print(x_test.shape,y_test.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aFmcxrXacsRC"
   },
   "source": [
    "## Modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dwlKUw-q1Plb"
   },
   "source": [
    "### Model Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "X4AIWqbP4lI2"
   },
   "outputs": [],
   "source": [
    "# define chuncks of model \n",
    "def conv_layer(inputs,filters,kernel_size,strides,dropout_rate=None,weight_decay=1E-4):\n",
    "    \"\"\"single convolution 2D with pre-activation\"\"\"\n",
    "  \n",
    "    ## pre-activation BN and ReLU\n",
    "    bn_axis = 1 if K.image_data_format() == \"th\" else -1\n",
    "  \n",
    "    x = layers.BatchNormalization(axis = bn_axis, epsilon=1.1e-5)(inputs)\n",
    "    x = layers.Activation('relu')(x)\n",
    "  \n",
    "    ## conv2D layer with padding = 'same'\n",
    "    outputs = layers.Conv2D(filters = filters, kernel_size = kernel_size,strides = strides, \n",
    "                            kernel_initializer=\"he_uniform\", padding = 'same', use_bias=False,\n",
    "                            kernel_regularizer = regularizers.l2(weight_decay))(x)\n",
    "  \n",
    "    ## dropout layer after convolutional layers except the first conv layer (for dataset without data augmentation)\n",
    "    if dropout_rate is not None: \n",
    "        outputs = layers.Dropout(dropout_rate)(outputs)\n",
    "  \n",
    "    return outputs\n",
    "\n",
    "def bottleneck_layer(inputs,filters):\n",
    "    \"\"\"bottleneck layer with pre-activation\"\"\"\n",
    "    ## pre-activation BN and ReLU\n",
    "    bn_axis = 1 if K.image_data_format() == \"th\" else -1\n",
    "  \n",
    "    x = layers.BatchNormalization(axis = bn_axis, epsilon=1.1e-5)(inputs)\n",
    "    x = layers.Activation('relu')(x)\n",
    "  \n",
    "    ## conv2D layer with kernel size 1 x 1\n",
    "    ### number of feature maps of bottleneck layer = 4k \n",
    "    nb_filter = 4 * filters\n",
    "    outputs = layers.Conv2D(filters = nb_filter, kernel_size = (1,1),strides = (1,1), padding = 'same')(x)\n",
    "  \n",
    "    return outputs  \n",
    "\n",
    "def conv_block(inputs,filters):\n",
    "    \"\"\"conv block consisting of a bottleneck layer and a convolution layer with kernel size 3x3\"\"\"\n",
    "  \n",
    "    kernel_size = (3,3)\n",
    "    strides = (1,1)\n",
    "  \n",
    "    x = bottleneck_layer(inputs,filters)\n",
    "    outputs = conv_layer(x,filters,kernel_size,strides)\n",
    "    \n",
    "  \n",
    "    return outputs\n",
    "\n",
    "def dense_block(inputs,nb_filter,growth_rate, nb_conv_block):\n",
    "    \"\"\"a convolutional block with multiple conv blocks and concatenation for outputs of each conv block \n",
    "       key innovation for DenseNet\n",
    "    \"\"\"\n",
    "    concat_axis = 1 if K.image_data_format() == \"th\" else -1\n",
    "\n",
    "    # x_list is created for storage purpose\n",
    "    x_list = [inputs]\n",
    "  \n",
    "    for i in range(nb_conv_block):\n",
    "        outputs = conv_block(inputs, growth_rate)\n",
    "        x_list.append(outputs)\n",
    "        inputs = layers.concatenate([inputs,outputs],concat_axis)\n",
    "    \n",
    "        nb_filter +=growth_rate\n",
    "\n",
    "    return inputs,nb_filter,x_list\n",
    "\n",
    "def transition_block(inputs,nb_filter,theta,dropout_rate):\n",
    "    \"\"\"a block that consists of one bottleneck conv layer and a pooling layer\n",
    "       to furhter improve model compactness\n",
    "    \"\"\"\n",
    "    \n",
    "    #compression parameter to reduce number of channels/filters\n",
    "    ## theta = 1 for no compression\n",
    "    ## theta = 0.5 for DenseNet-BC\n",
    "  \n",
    "    x = conv_layer(inputs,math.floor(nb_filter * theta),(1,1),(1,1),dropout_rate)\n",
    "    x = layers.AveragePooling2D((2, 2), strides=(2, 2))(x)\n",
    "  \n",
    "    return x, math.floor(nb_filter * theta)\n",
    "\n",
    "# define the model \n",
    "def DenseNet(inputs,init_nb_filters,growth_rate,nb_layers,theta,dropout_rate,weight_decay):\n",
    "    \"\"\"the actual DenseNet model\"\"\"\n",
    "    ## The initial convolution layer comprises 2k convolutions of size 3 x 3 with stride 1\n",
    "    x = conv_layer(inputs,init_nb_filters,(3,3),(1,1))\n",
    "\n",
    "    ## Dense Block (1) + Transition Block (1)\n",
    "    x,nb_filters,x_list_ = dense_block(x,init_nb_filters,growth_rate,nb_layers)\n",
    "    x,nb_filters = transition_block(x, nb_filters,theta, dropout_rate)\n",
    "\n",
    "    ## Dense Block (2) + Transition Block (2)\n",
    "    x,nb_filters,x_list_ = dense_block(x,nb_filters,growth_rate,nb_layers)\n",
    "    x,nb_filters = transition_block(x, nb_filters,theta, dropout_rate)\n",
    "\n",
    "    ## Dense Block (3)\n",
    "    x,nb_filters,x_list_ = dense_block(x,nb_filters,growth_rate,nb_layers)\n",
    "\n",
    "    ## Global Average Pooling \n",
    "    x = layers.GlobalAveragePooling2D()(x)\n",
    "\n",
    "    ## FC classification layer\n",
    "    x = layers.Dense(nb_classes, activation='softmax', kernel_regularizer=regularizers.l2(weight_decay), bias_regularizer=regularizers.l2(weight_decay))(x)\n",
    "\n",
    "    return x\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "UbLrVU3o1eLt"
   },
   "source": [
    "### Model Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CZnC6DrEPMEV"
   },
   "outputs": [],
   "source": [
    "# hyperparameters\n",
    "## growth rate =k \n",
    "growth_rate = 12\n",
    "\n",
    "\n",
    "## initial number of output channels\n",
    "### DenseNet-B: 16\n",
    "### DenseNet-BC: 2 * growth_rate\n",
    "init_nb_filters = 24\n",
    "\n",
    "\n",
    "## same number of layers in each dense block\n",
    "depth = 100\n",
    "nb_layers = 16\n",
    "\n",
    "\n",
    "## dropout rate \n",
    "### DenseNet with data augmentation\n",
    "dropout_rate = None\n",
    "### DenseNet without data augmentation\n",
    "### dropout_rate = 0.2\n",
    "\n",
    "## weight decay \n",
    "weight_decay=1E-4\n",
    "\n",
    "## theta \n",
    "theta = 0.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-IGq8hMT1j3i"
   },
   "source": [
    "## Model Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "LDZAZYZraS9h"
   },
   "source": [
    "### DenseNet-BC-100-12 with data augmentation\n",
    "- dropout_rate = None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 90
    },
    "colab_type": "code",
    "id": "Hz5GEpjhaVLX",
    "outputId": "bfd893d2-d620-424f-fe51-cfd44b544818"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    }
   ],
   "source": [
    "# DenseNet-BC setup\n",
    "\n",
    "## input images 32 x 32\n",
    "inputs = keras.Input(img_dim)\n",
    "\n",
    "## call DenseNet-BC-100-12 with data augmentation\n",
    "outputs = DenseNet(inputs,init_nb_filters,growth_rate,nb_layers,theta,dropout_rate,weight_decay)\n",
    "\n",
    "## assign the actual model\n",
    "model = keras.Model(inputs,outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kYYFMK4-1nCh"
   },
   "source": [
    "## Model Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JU8iR5KEzxYd"
   },
   "outputs": [],
   "source": [
    "# training parameters\n",
    "## define learning rate decay rule\n",
    "def scheduler(epoch):\n",
    "    \"\"\"learning rate decay for 40 epoch\n",
    "    - 50% of epoch: 0.1 \n",
    "    50% -75% of epoch: 0.01 \n",
    "    70% - of epoch: 0.001\n",
    "    \"\"\"\n",
    "    if epoch < 20:\n",
    "        return 0.1\n",
    "    if epoch < 30:\n",
    "        return 0.01\n",
    "    return 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "SricOUEKsxGH",
    "outputId": "7095aed2-1816-4b3f-9750-1a8e5a009933"
   },
   "outputs": [],
   "source": [
    "# actual training \n",
    "## define optimizer:SGD \n",
    "optimizer = keras.optimizers.SGD(lr=.1, momentum=0.9, nesterov=True)\n",
    "model.compile(loss='categorical_crossentropy',optimizer=optimizer,metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## define callbacks\n",
    "model_checkpoint = keras.callbacks.ModelCheckpoint('densenet_100_12.h5', monitor=\"val_acc\", save_best_only=True,\n",
    "                                  save_weights_only=True, verbose=1)\n",
    "\n",
    "## set callback\n",
    "change_lr = keras.callbacks.LearningRateScheduler(scheduler)\n",
    "callbacks = [change_lr,model_checkpoint]\n",
    "\n",
    "## feed all settings into the model to train\n",
    "history = model.fit_generator(generator=train_datagen,\n",
    "                steps_per_epoch= x_tr.shape[0] // batch_size,\n",
    "                epochs=nb_epoch,b\n",
    "                callbacks=callbacks,\n",
    "                validation_data=validation_datagen,\n",
    "                validation_steps = x_val.shape[0] // batch_size,\n",
    "                verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fAO5zVud7DGs"
   },
   "outputs": [],
   "source": [
    "model.load_weights('./densenet_100_aug_40.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Compression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pruning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XOAB8K0MWhhq"
   },
   "source": [
    "### Pruning Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "id": "c1YK9O69WgzP",
    "outputId": "7a2db3d9-00c7-456c-a925-e0da834078f1"
   },
   "outputs": [],
   "source": [
    "def prune_after_training(model,percentile):\n",
    "\n",
    "    num_layers = len(model.layers)\n",
    "\n",
    "    for layer_idx in range(num_layers):\n",
    "        if len(model.layers[layer_idx].weights) !=0:\n",
    "            for weight_idx in range(len(model.layers[layer_idx].weights)):\n",
    "                if (len(model.layers[layer_idx].weights[weight_idx].shape) == 4) or (len(model.layers[layer_idx].weights[weight_idx].shape)==2):\n",
    "                    weight = model.layers[layer_idx].get_weights()[weight_idx]\n",
    "                    weight_abs = np.absolute(weight)\n",
    "                    value = np.percentile(weight_abs,percentile)\n",
    "                    # pruning\n",
    "                    ## create mask with percentage q of zeros\n",
    "                    mask = np.ones((weight.shape))\n",
    "                    mask[weight_abs <= value] = 0         \n",
    "                    ## get pruned weight matrix and pass it to weight\n",
    "                    pruned_weight = np.multiply(mask, weight)\n",
    "                    K.set_value(model.layers[layer_idx].weights[weight_idx],pruned_weight)\n",
    "    return model\n",
    "\n",
    "def get_total_sparsity(model):\n",
    "    \n",
    "    total = 0\n",
    "    zeros = 0\n",
    "    for i, w in enumerate(model.get_weights()):\n",
    "#         print(\n",
    "#             \"{} -- Total:{}, Zeros: {:.2f}%\".format(\n",
    "#                 model.weights[i].name, w.size, np.sum(w == 0) / w.size * 100\n",
    "#             )\n",
    "#         )\n",
    "        total +=w.size\n",
    "        zeros +=np.sum(w == 0)\n",
    "    total_sparsity = zeros/total * 100\n",
    "    return total_sparsity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Direct Pruning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "testing_acc_list = []\n",
    "sparsity_list = []\n",
    "\n",
    "for value in range(0,100,5):\n",
    "    model.load_weights('./densenet_100_aug_40.h5')\n",
    "    model_pruned_after_training = prune_after_training(model,value)\n",
    "    testing_acc = model_pruned_after_training.evaluate(x_test,y_test,batch_size)\n",
    "    testing_acc_list.append(testing_acc)\n",
    "    \n",
    "    sparsity = get_total_sparsity(model_pruned_after_training)\n",
    "    sparsity_list.append(sparsity)\n",
    "    \n",
    "    model_pruned_after_training.save_weights('./direct_pruned{}.h5'.format(value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "testing_accuracy_list = [testing_acc_list[i][1] for i in range(len(testing_acc_list))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sparsity_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "12h98HM0Wg6-"
   },
   "outputs": [],
   "source": [
    "plt.axhline(y=0.7122, color='black', linestyle='--',label = 'no pruning')\n",
    "plt.scatter(sparsity_list,testing_accuracy_list,s = 10, label = 'pruned after training')\n",
    "plt.xlim([0,90])\n",
    "plt.ylim([0,0.9])\n",
    "plt.legend()\n",
    "plt.xlabel('Total Parameters Pruned %')\n",
    "plt.ylabel('Testing Accuracy')\n",
    "plt.title('Direct Pruning')\n",
    "plt.savefig('Direct Pruning corrected')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "N_YVq67Q--T1"
   },
   "source": [
    "### Iterative Pruning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sparsity level\n",
    "## adjust it to run the next chunk\n",
    "## (45,90)\n",
    "value = 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.5/site-packages/tensorflow_model_optimization/python/core/sparsity/keras/pruning_schedule.py:240: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n"
     ]
    }
   ],
   "source": [
    "# DenseNet-BC setup\n",
    "## input images 32 x 32\n",
    "inputs = keras.Input(img_dim)\n",
    "## call DenseNet-BC-100-12 with data augmentation\n",
    "outputs = DenseNet(inputs,init_nb_filters,growth_rate,nb_layers,theta,dropout_rate,weight_decay)\n",
    "## assign the actual model\n",
    "model = keras.Model(inputs,outputs)\n",
    "\n",
    "## define optimizer:SGD \n",
    "optimizer = keras.optimizers.SGD(lr=.1, momentum=0.9, nesterov=True)\n",
    "model.compile(loss='categorical_crossentropy',optimizer=optimizer,metrics=['accuracy'])\n",
    "\n",
    "model.load_weights('./densenet_100_aug_40.h5')\n",
    "nb_epoch = 40\n",
    "end_step = np.ceil(1.0 * 45000 / batch_size).astype(np.int32) * nb_epoch\n",
    "pruning_schedule = tfmot.sparsity.keras.PolynomialDecay(\n",
    "                  initial_sparsity=0.0, final_sparsity= value/100,\n",
    "                  begin_step=2000, end_step=end_step,frequency=100)\n",
    "\n",
    "model_pruned = tfmot.sparsity.keras.prune_low_magnitude(model, pruning_schedule=pruning_schedule)\n",
    "\n",
    "optimizer = keras.optimizers.SGD(lr=.001, momentum=0.9, nesterov=True)\n",
    "model_pruned.compile(\n",
    "loss=tf.keras.losses.categorical_crossentropy,\n",
    "optimizer=optimizer,\n",
    "metrics=['accuracy'])\n",
    "# actual training \n",
    "# Add a pruning step callback to peg the pruning step to the optimizer's\n",
    "# step. Also add a callback to add pruning summaries to tensorboard\n",
    "\n",
    "#     model_checkpoint = keras.callbacks.ModelCheckpoint('./iter_pruning_{}.h5'.format(value), monitor=\"val_acc\", save_best_only=False,\n",
    "#                                   save_weights_only=True, verbose=1)\n",
    "callbacks = [sparsity.UpdatePruningStep()]\n",
    "## feed all settings into the model to train\n",
    "history = model_pruned.fit_generator(generator=train_datagen,\n",
    "                steps_per_epoch= x_tr.shape[0] // batch_size,\n",
    "                epochs=nb_epoch,\n",
    "                callbacks=callbacks,\n",
    "                validation_data=validation_datagen,\n",
    "                validation_steps = x_val.shape[0] // batch_size,\n",
    "                verbose=1)    \n",
    "\n",
    "model_pruned.save_weights('./iter_pruned_{}.h5'.format(value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 9s 850us/sample - loss: 1.4152 - acc: 0.7122\n"
     ]
    }
   ],
   "source": [
    "#model_pruned.load_weights('./iter_pruned_{}.h5'.format(value))\n",
    "# accuracy on test set using DenseNet-BC-100-12 pruned\n",
    "acc_iter = model_pruned.evaluate(x_test,y_test,batch_size)\n",
    "\n",
    "# get rid of extra parameters during pruning\n",
    "final_model = sparsity.strip_pruning(model_pruned)\n",
    "sparsity = get_total_sparsity(final_model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.4151958063125611, 0.7122] 0.0\n"
     ]
    }
   ],
   "source": [
    "print(acc_iter,sparsity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xt8FPX1//HXIYKAN1QQ/QpysSkQEAIEFBXFCwqIaAUUKir1Qr1Q9Ve1YusFsd96qd9qaa1Kq6LWK3hDvKKCBcVKqGghKNco8YIBBQWChOT8/pjJumw2ySZks5vwfj4e+8jOzGdmzs5u9uzMZ+aMuTsiIiKxGqU6ABERSU9KECIiEpcShIiIxKUEISIicSlBiIhIXEoQIiISlxKESBKZ2W/N7B+pjkOkJkzXQciuyMzygdZACbAZeAUY7+6bUhmXSDrRHoTsyk519z2BXkAOcH30RAvof0R2Wfrwyy7P3T8n2IPoZmZzzOx/zewdYAvQ0czyzezEsvZmNtHM/hk+b29mbmbnmdlnZrbOzH5Xw7bNzOxhM/vWzJaa2W/MrKCutoNIrN1SHYBIqplZW2AI8CzQHzgHGAx8AliCizka6AT8FHjfzJ5196XVbHsT0B7oCOwBvFyjFyRSS7QHIbuy581sAzAPeBv4Qzh+qrsvcfft7l6c4LJudvcid/8Q+BDoUYO2ZwJ/cPdv3b0AmFztVyRSi7QHIbuy0939jegRZgawpgbL+irq+RZgzxq0/Z+YddckDpFaoz0IkfJiT+3bDDSPGj4wSev9EmgTNdw2SesRSYgShEjVFgGjzKyxmeUAI5K0nqeB68xsXzM7GBifpPWIJEQJQqRqNwCHAt8CNwOPJ2k9k4ACYDXwBjAd+CFJ6xKpki6UE0lTZnYJMMrdj011LLJr0h6ESJows4PM7Cgza2RmnYCrgOdSHZfsupKaIMxskJl9YmYrzGxCnOntzOxNM/sovECpTbzliOwimgD3A98DbwEvAH9LaUSyS0vaISYzywCWAQMJjqsuAEa7e15Um2nATHd/2MyOB37h7uckJSAREamWZO5B9AVWuPsqd98GPAmcFtMmi+CXEsDsONNFRCRFknmh3MHseKFPAXB4TJsPgTOAPwM/A/Yys/3dfX10IzMbB4wD2GOPPXp37tw5aUGLiDRECxcuXOfuraozT6qvpL4a+KuZjQX+BXxOUH55B+4+BZgCkJOT47m5uXUZo4hIvWdmn1Z3nmQmiM/Z8UrQNuG4CHf/gmAPAjPbExju7huSGJOIiCQomX0QC4BMM+tgZk2AUcCM6AZm1jKq3v51wINJjEdERKohaQnC3bcTlAp4DVgKPO3uS8xskpkNC5sNAD4xs2UEd/f632TFIyIi1VPvrqRWH4SISPWZ2UJ3z6nOPLqSWkRE4lKCEBGRuJQgREQkLiUIERGJSwlCRETiUoIQEZG4lCBERCQuJQgREYlLCUJEROJSghARkbiUIEREJC4lCBERiUsJQkRE4lKCEBGRuJQgREQkrqQmCDMbZGafmNkKM5sQZ/ohZjbbzD4ws4/MbEgy4xERkcQlLUGYWQZwDzAYyAJGm1lWTLPrCe4015PglqR/S1Y8IiJSPcncg+gLrHD3Ve6+DXgSOC2mjQN7h8/3Ab5IYjwiIlINuyVx2QcDa6KGC4DDY9pMBF43s18BewAnJjEeERGphlR3Uo8Gprp7G2AI8KiZlYvJzMaZWa6Z5RYWFtZ5kCIiu6JkJojPgbZRw23CcdEuAJ4GcPf5QFOgZeyC3H2Ku+e4e06rVq2SFK6IiERLZoJYAGSaWQcza0LQCT0jps1nwAkAZtaFIEFoF0FEJA0kLUG4+3ZgPPAasJTgbKUlZjbJzIaFza4CLjKzD4EngLHu7smKSUREEpfMTmrc/WXg5ZhxN0Y9zwOOSmYMIiJSM6nupBYRkTSlBCEiInEpQYiISFxKECIiEpcShIiIxKUEISIicSlBiIhIXEoQIiISlxKEiIjEpQQhIiJxKUGIiEhcShAiIhKXEoSIiMSV1Gqu6WpW3lrmLi+kf2YrBma1rvexpNPrEZGGw+rb7Rf+5ydd/eEZb+3Ul+nlT3xAUXEJzRpnMHl0zxotq7a+2Hc2ltp6PSLSsJnZQnfPqdY89S1BNGrSzJselElm6z3Zt3kTAM4880wuvfRStmzZwpAhQ8rNM3bsWMaOHcu6devIGTCYr77bGpl24N5Nuf2GqznrrLNYs2YN55xzTrn5r7rqKk499VQ++eQTfvnLX/Ltlm0sX7uJUncamXHTjTdw7QUjWbRoEVdeeWW5+f/whz9w5JFH8u677/Lb3/42Mj5/3Wa++m4r+50wjiatO3J08y8oeOuxcvPff//9dOrUiRdffJH/+7//22Fa/rrNbD/mMnbbuxWbl/6LPVa+RfuWe+zQZvr06bRs2ZKpU6cyderUcst/+eWXad68Ob+64TZefO4Z9mneOLJtAebMmQPAnXfeycyZM3eYt1mzZrzyyisA3HLLLbz55ps7TN9///155plnALjuuuuYP3/+DtPbtGnDP//5TwCuvPJKFi1atMP0n/70p0yZMgWAcePGsWzZsh2mZ2dnc/fddwMwZswYCgoKdpjer18/br31VgCGDx/O+vXrd5h+wgkncMMNNwAwePBgioqKdpg+dOhQrr76agAGDBhArIQ/e/OfYMS430CzfaH5fpHpl1xySbU+e2z5Boq+jSzn+uuv58QTT6z2Z6/M3XffTXZ2Nm+88Qa///3vy02v7LMH8Oijj9K2bVueeuop7r333nLTE/3s/e1vf+Ppp58uN12fvVr47K1bx4gRI3j77bernSCS2gdhZoPM7BMzW2FmE+JMv8vMFoWPZWa2IZHllrqzcUtxjWLap3ljGpkB0MiMfZo3rvYyNm4ppjRMrKXu/PfzhMKuNJZmjTM47OAWNVpG090yAGiS0ahGrweCPZFpuQV89d1Wlq/dxLdbttVoORLHxy/Di1fA919C4cfBl3xNbPkmmH9nlyOSoKTtQZhZBrAMGAgUENyjenR4F7l47X8F9HT38ytb7u4HZXrHC/+yU4dSdvbwUG0e1kmXPogbX1jMI/M/jQyf268dk07rlpJYGpyXroYFf/9xuM9FcMqdqVtObfj4ZVj5Fhx6PHQu/+tV0k9aHWIys37ARHc/ORy+DsDdb62g/bvATe4+q7Ll7mwfRG1paF+E6g9Joo9fhmfOh+IiaNwMhj9Ysy/V2lrOzkqXOKRaapIgknkW08HAmqjhAuDweA3NrB3QAXirgunjgHEAhxxySFp86QzMap0WcdSWgVmtmTy6504lvbnLCykqLgGgqLiEucsLG9Q2qrHOQ4Iv0Z39xV1by9lZK98KkgMEf1e+VfNYtCeS1tLlNNdRwHR3L4k30d2nAFMAcnJy6levej2ys0mvf2YrpuUWRPYg+me2qvGyGtoeGp2H1M4XYG0tZ2ccejws+uePexCHHl+z5UTviSz6p/ZE0lAyE8TnQNuo4TbhuHhGAZclMRapA7WxFwI7HqqallugQ1Xpprb2ZGprT0R7IUmTzASxAMg0sw4EiWEU8PPYRmbWGdgXmB87Teqf2jj0pkNV9UBt7MnUxp6I9kKSKmmnubr7dmA88BqwFHja3ZeY2SQzGxbVdBTwpNe3CzIkafpntqJZ4+DU3Z09VCVprGxPpM9FNf9ij7cXIrWm3l0ol5OT47m5uakOQ5KswfVBSHLojKqEpdVprsmiBCEiO6itPogG3peRbqe5iqSU9kJ2EbXRH6K+jLhU7lsapLIzoR6Z/ymXP/EBs/LWpjokSWfqy4hLCUIapHhnQolU6NDjgz4M2LlrOxoYJQhpkHQmlFRLbZxR1QCpk1oaLPVBSJ1L445udVKLRGlo9bIkzTXAjm4dYhIRqQ0NsKNbCUJEpDY0wI5uHWISEakN6VKOvRYpQYiI1JZ0KMdei3SISURE4lKCEBGRuKpMEGZ2iZntUxfBiIhI+khkD6Id8B8ze9zMTkx2QCIikh6qTBDuPgHIBB4DLjaz5eFNf9pXNa+ZDTKzT8xshZlNqKDNmWaWZ2ZLzOzxasYvIiJJklAfhLuXAvnhoxQ4CHjBzG6taB4zywDuAQYDWcBoM8uKaZMJXAcc5e5dgSur/xJEkmtW3lpufGGxKsLKLieRPojLzOx94M/AQqC7u18E9ATOqmTWvsAKd1/l7tuAJ4HTYtpcBNzj7t8CuPvXNXgNIkmjsuGyK0tkD+J/gNHufqK7P+HuP0Bkr2JYJfMdDKyJGi4Ix0X7KfBTM3vHzN4zs0HxFmRm48ws18xyCwtVtlnqjsqGy64skQTxPBD52WRme5lZDoC7L97J9e9G0L8xABgN/N3MWsQ2cvcp7p7j7jmtWqlss9QdlQ2XXVkiV1JPAXpHDW8G7o8ZF8/nQNuo4TbhuGgFwL/dvRhYbWbLCBLGggTiEkm6gVmtmTy6p8qGyy4pkQTRKDycBASHlsyscQLzLQAyzawDQWIYBfw8ps3zBHsOD5lZS4JDTqsSilykjqhsuOyqEjnEtDq8WC7DzBqZ2WUEZzNVyt23A+OB14ClwNPuviQ8Rbas7+I1YL2Z5QGzgWvcfX2NXomIiNSqKu8oZ2atCU5XHQA4wRf5r9w9Jadz6I5yIiLVl5Q7yoWJYESNoxIRkXqpygRhZrsDY4GuQNOy8e4+LnlhiYhIqiXSB/EI0B4YCvwbOBTYmsSYREQkDSSSIH7q7tcBm9z9AWAQwVXSIiLSgCWSIIrDvxvMrAuwF3BA8kISEZF0kMh1EA+Y2b7ATQSnpTYHbkxqVCIiknKVJoiwIuu6sJjebOCQOolKRERSrtJDTO5eAvy2jmIREZE0kkgfxOtmdqWZHWRme5c9kh6ZiIikVCJ9EGPCv1dFjXN0uElEpEFL5ErqtlW1ERGRhieRK6ljK7AC4O66f7SISAOWyCGm/lHPmwLHE9x6VAlCRKQBS+QQ0yXRw+E1EUoOIiINXCJnMcX6HuhY24GIiEh6qTJBmNlzZvZs+Hie4OY/LyaycDMbZGafmNkKM5sQZ/pYMys0s0Xh48LqvwQREUmGRPog/hr1fDvwqbvnVzVTeBX2PcBAgntPLzCzGe6eF9P0KXcfn2C8IiJSRxJJEMuBr919K4CZNTOztu6+por5+gIr3H1VON+TwGlAbIIQafBm5a1l7vJC+me20v2tpd5IpA/iWaA0argUeCaB+Q4GopNIQTgu1nAz+8jMpptZ3GsuzGycmeWaWW5hYWECqxZJH7Py1nL5Ex/wyPxPufyJD5iVl5K79YpUWyIJYjd331Y24O4/ALvX0vpfBNq7e3dgFvBwvEbuPsXdc9w9p1WrVrW0apG6MXd5IUXFJQAUFZcwd7l+5Ej9kEiCWG9mQ8oGzGwo8E0C830ORO8RtAnHRbj7+jDhAPwD6J3AckXqlf6ZrWjWOAOAZo0z6J+pHzlSPyTSB3EJ8LiZ3RMOF/JjfabKLAAyzawDQWIYBexwVbaZHeTuX4aDwwjOkBJpUAZmtWby6J7qg5B6J5EL5ZYBOWbWIhzekMiC3X27mY0nuMlQBvCguy8xs0lArrvPAC43s2EEZ0d9A4yt2csQSW8Ds1orMUi9Y+5eeQOzW4D/K0sM4ZXUV7r7TXUQXzk5OTmem5ubilWLiNRbZrbQ3XOqM08ifRBDo/cawrvLnVrd4EREpH5JJEFkmFmTsgEzawo0qaS9iIg0AIl0Uj8JzDKzB8Ph81GxPhGRBi+RTuo/mNlHwInhqDvc/aXkhiUiIqmWyB4E7j4TmAlgZkeY2Z/d/YqkRiYi5ahkh9SlhBKEmR0GjAbOAr4gsVIbIlKLykp2FBWXMC23gMmjeypJSFJV2EltZh3N7Hdmthj4O8EFco3dvb+7311nEYoIoJIdUvcqO4tpBXAScIa7H+HudxFc0CYiKaCSHVLXKjvEdCZBeYw3zGwm8BRgdRKViJSjkh1S1xK5knov4GcEfRDHAg8Az7n7W8kPrzxdSS0iUn1JuZLa3b9390fcfTBwCEFBvZSU2RARkbqTyJXUEe6+zt3/5u7HJisgERFJD9VKECIisutQghARkbiUIEREJK4qE4SZfWtm38Q8VpvZNDNrX8W8g8zsEzNbYWYTKmk33MzczKrVwy4iIsmTSKmNe4Av+bGC62igPfAh8BBwXLyZzCwjnHcgUAAsMLMZ7p4X024v4Arg3zWIX0REkiSRQ0ynuvs97v5t+PgbcJK7PwbsV8l8fYEV7r7K3bcRlA0/LU67W4Dbga3VDV5ERJInkQRRZGZnlA2Ez38IB0srme9gYE3UcEE4LsLMegFtqyofbmbjzCzXzHILC1V/RkSkLiSSIMYAF4V9D+uBi4BzzKw5cGVNV2xmjYA/AVdV1dbdp7h7jrvntGql+jMiInUhkRsGrQAGVzD57Upm/RxoGzXcJhxXZi+gGzDHzAAOBGaY2TB3Vy0NEZEUqzJBmFlLgtuMto9u7+7jqph1AZBpZh0IEsMo4OdR828EWkatZw5wtZKDiEh6SOQspheA94B5QEmiC3b37WY2HngNyAAedPclZjYJyHX3GTUJWERE6kYiCWIPd6+ynyAed38ZeDlm3I0VtB1Qk3WIiEhyJNJJ/YqZnZT0SEREJK0kkiAuBl41s03hmUzfmtk3yQ5MRERSK5FDTC2rbiIiIg1NhQnCzDLdfTnQtYImHyUnJBERSQeV7UFMAC4gqKcUy4FjkhKRiIikhQoThLtfED493t2Lo6eZWeOkRiUiIimXSCd1vCqrqrwqItLAVdYHcQBwENDMzA4DLJy0N9C8DmITEZEUqqwP4hSCEhttCPohyhLE98ANSY5LRERSrLI+iIeAh8zsTHd/ug5jEhGRNJBIH8QBZrY3gJndZ2bvm9kJSY5LRERSLJEEMc7dvwvLbRxEcD+IO5IbloiIpFoiCcLDv0OAR9z9wwTnExGReiyRL/oPzexlYChB4b49+TFpiIhIA5VILaZfAL2BFe6+JbyB0AVVzCMiIvVclXsQ7l4CdAQuCUc1S2Q+ADMbZGafmNkKM5sQZ/rFZvZfM1tkZvPMLKs6wYuISPJU+UVvZn8FjgPGhKM2A/clMF8GwfUTg4EsYHScBPC4ux/m7tkEHd9/qkbsIiKSRInsCRzp7r8EtgK4+zdAkwTm60twWGqVu28DngROi27g7t9FDe6B+jZERNJGIn0QxWbWiPDL28z2B0oTmO9gYE3UcAFweGwjM7sM+DVB0jk+3oLMbBwwDuCQQw5JYNUiIrKzKtyDMLOy5HEP8AzQysxuBuYBt9dWAO5+j7sfClwLXF9BmynunuPuOa1ataqtVYuISCUq24N4H+jl7o+Y2ULgRIJ6TCPdfXECy/4caBs13CYcV5EngXsTWK6IiNSByhJEWXE+3H0JsKSay14AZJpZB4LEMAr4+Q4r+PGudRAUB1yOiCTVrLy1zF1eSP/MVgzMap3qcCSNVZYgWpnZryua6O6VnnHk7tvNbDzwGpABPOjuS8xsEpDr7jOA8WZ2IlAMfAucV+1XICIJm5W3lsuf+ICi4hKm5RYweXRPJQmpUGUJIgPYk6g9iepy95eBl2PG3Rj1/IqaLltEqm/u8kKKiksAKCouYe7yQiUIqVBlCeJLd59UZ5GISNL1z2zFtNwCiopLaNY4g/6ZOulDKpZQH4SINAwDs1ozeXRP9UFIQipLELrng0gDNDCrtRKDJKTC6yDCK6ZFRGQXpfs6iIhIXEoQIiISlxKEiIjEpQQhIiJxKUGIiEhcShAiIhKXEoSIiMSlBCEiInEpQYiISFxKECIiElci96ROe8XFxRQUFLB169ZUhyKS9po2bUqbNm1o3LhxqkORNJfUBGFmg4A/E9xb4h/uflvM9F8DFwLbgULgfHf/tLrrKSgoYK+99qJ9+/aYqQitSEXcnfXr11NQUECHDh1SHY6kuaQdYjKzDOAeYDCQBYw2s6yYZh8AOe7eHZgO3FGTdW3dupX9999fyUGkCmbG/vvvr71tSUgy+yD6AivcfZW7bwOeBE6LbuDus919Szj4HtCmpitTchBJjP5XJFHJTBAHA2uihgvCcRW5AHgl3gQzG2dmuWaWW1hYWIshiohIRdLiLCYzGwPkAH+MN93dp7h7jrvntGqlWyRWx4wZM7jtttuqbigiEiOZndSfA22jhtuE43ZgZicCvwOOdfcfkhhPvbZ9+3Z22636b9ewYcMYNmxYEiISkYYumQliAZBpZh0IEsMo4OfRDcysJ3A/MMjdv66tFQ8YMKDcuDPPPJNLL72ULVu2MGTIkHLTx44dy9ixY1m3bh0jRozYYdqcOXMqXV9+fj6DBw/m6KOP5t133+Xggw/mhRdeoFmzZixatIiLL76YLVu2cOihh/Lggw+y7777llt306ZNyc3N5bvvvuNPf/oTQ4cOZerUqTz77LNs2rSJkpISbr75Zu68805mzpwJwPjx48nJyWHs2LG0b9+e8847jxdffJHi4mKmTZtG586dmTp1Krm5ufz1r39l7Nix7L333uTm5vLVV19xxx13MGLECEpLSxk/fjxvvfUWbdu2pXHjxpx//vnltoOI7FqSdojJ3bcD44HXgKXA0+6+xMwmmVnZT9o/AnsC08xskZnNSFY8ybZ8+XIuu+wylixZQosWLXjmmWcAOPfcc7n99tv56KOPOOyww7j55pvjzp+fn8/777/PSy+9xMUXXxw5y+Q///kP06dP5+23364yhpYtW/Kf//yHSy65hDvvvDNumy+//JJ58+Yxc+ZMJkyYAMCzzz5Lfn4+eXl5PProo8yfP78mm0BEGpikXgfh7i8DL8eMuzHq+YnJWG9lv/ibN29e6fSWLVtWuccQT4cOHcjOzgagd+/e5Ofns3HjRjZs2MCxxx4LwHnnncfIkSPjzn/mmWfSqFEjMjMz6dixIx9//DEAAwcOZL/99ksohjPOOCOy/meffTZum9NPP51GjRqRlZXF2rVrAZg3bx4jR46kUaNGHHjggRx33HGJv3ARabDSopO6Idh9990jzzMyMti+fXu15o899bBseI899oiM22233SgtLY0Mx57LXhZDZeuPjtPdqxWjiOxalCCSaJ999mHfffdl7ty5ADz66KORvYlY06ZNo7S0lJUrV7Jq1So6depUrk27du3Iy8vjhx9+YMOGDbz55pu1EudRRx3FM888Q2lpKWvXrq3RHpSINDwNohZTOnv44YcjndQdO3bkoYceitvukEMOoW/fvnz33Xfcd999NG3atFybtm3bcuaZZ9KtWzc6dOhAz549ayXG4cOH8+abb5KVlUXbtm3p1asX++yzT60sW0TqL6tvhxlycnI8Nzd3h3FLly6lS5cuKYpo540dO5ahQ4em9KyhTZs2seeee7J+/Xr69u3LO++8w4EHHpiyeCS56vv/jFSfmS1095zqzKM9CAFg6NChbNiwgW3btnHDDTcoOYiIEkQ6mDp1aqpDUL+DiJSjTmoREYlLCUJEROJSghARkbiUIEREJC4liAZizpw5DB06tFrzXHPNNXTt2pVrrrmG559/nry8vCRFl7i7776bLVu2VN0wxo033sgbb7xRaRuVPhepHp3FVIdKSkrIyMhIdRgRU6ZM4ZtvviEjIyNyLUZWVuxdYStWkxLk7o6706hR/N8md999N2PGjKF58+blplW2/SZNmlTlulX6XKR6dtk9iFl5a7nxhcXMylu708vKz8+nc+fOnH322XTp0oURI0ZEfgW3b9+ea6+9ll69ejFt2jQGDBhA2YV+69ato3379kBwqusZZ5zBoEGDyMzM5De/+U1k+a+//jr9+vWjV69ejBw5kk2bNgHw6quv0rlzZ3r16lVhcb78/Hz69+9Pr1696NWrF++++y4QfFlu2rSJ3r17c/PNNzNjxgyuueYasrOzWblyJStXrmTQoEH07t2b/v37R4oHjh07losvvpjDDz98hxjLXsNpp53GgAEDyMzMjFSuzc/Pp1OnTpx77rl069aNNWvWxH1NkydP5osvvuC4446LFAzcc889ueqqq+jRowfz589n0qRJ9OnTh27dujFu3LhIPamxY8cyffr0yDa/6aab6NWrF4cddlgk9qlTpzJ+/PhI+8svv5wjjzySjh07RuYtLS3l0ksvpXPnzgwcOJAhQ4ZEponscsp+0dWXR+/evT1WXl5euXGVeX3JV975+le83bUzvfP1r/jrS76q1vyxVq9e7YDPmzfP3d1/8Ytf+B//+Ed3d2/Xrp3ffvvtkbbHHnusL1iwwN3dCwsLvV27du7u/tBDD3mHDh18w4YNXlRU5Icccoh/9tlnXlhY6P379/dNmza5u/ttt93mN998sxcVFXmbNm182bJlXlpa6iNHjvRTTjmlXGybN2/2oqIid3dftmyZR2+/PfbYI/L8vPPO82nTpkWGjz/+eF+2bJm7u7/33nt+3HHHRdqdcsopvn379nLreuihh/zAAw/0devW+ZYtW7xr166+YMECX716tZuZz58/P/K6472msu1VWFgYWSbgTz31VGR4/fr1kedjxozxGTNmlIu/Xbt2PnnyZHd3v+eee/yCCy6IxHfZZZdF2o8YMcJLSkp8yZIlfuihh7q7+7Rp03zw4MFeUlLiX375pbdo0WKH7dJQVPd/Ruo/INer+X27Sx5imru8kKLiEgCKikuYu7yQgVmtd2qZbdu25aijjgJgzJgxTJ48mauvvhqAs846K6FlnHDCCZEaSFlZWXz66ads2LCBvLy8yLK3bdtGv379+Pjjj+nQoQOZmZmRdU6ZMqXcMouLixk/fjyLFi0iIyODZcuWVRnHpk2bePfdd3coTf7DDz/e7G/kyJEVHuoZOHAg+++/PxCUH583bx6nn3467dq144gjjgDgvffei/ua4snIyGD48OGR4dmzZ3PHHXewZcsWvvnmG7p27cqpp55abj6VPhfZeUlNEGY2CPgzkAH8w91vi5l+DHA30B0Y5e51si/fP7MV03ILKCouoVnjDPpn7vx9risq1w0Vl+yuqFw3/Fiy290ZOHAgTzzxxA5tFy1alFBcd911F61bt+bDDz+ktLQ0bhHAWKWlpbRo0aLCdUS/nliJlC3ez+4cAAAWcklEQVSv6DXF07Rp00gy2rp1K5deeim5ubm0bduWiRMnltuGZVT6PLlm5a1l7vJC+me22ukfV5K+ktYHYWYZwD3AYCALGG1msT2gnwFjgceTFUc8A7NaM3l0T87t147Jo3vWygf8s88+i9yJ7fHHH+foo4+O2659+/YsXLgQIKFj20cccQTvvPMOK1asAGDz5s0sW7aMzp07k5+fz8qVKwEq/LLduHEjBx10EI0aNeLRRx+lpKQkbru99tqL77//HoC9996bDh06MG3aNCD48vzwww+rjBVg1qxZfPPNNxQVFfH8889H9hISeU2xccQqSwYtW7Zk06ZNSekbUOnzqs3KW8vlT3zAI/M/5fInPqiVfjxJT8nspO4LrHD3Ve6+DXgSOC26gbvnu/tHQGm8BSTTwKzWTDqtW639+unUqRP33HMPXbp04dtvv+WSSy6J2+7qq6/m3nvvpWfPnqxbt67K5bZq1YqpU6cyevRounfvHjm81LRpU6ZMmcIpp5xCr169OOCAA+LOf+mll/Lwww/To0cPPv744wp//Y8aNYo//vGP9OzZk5UrV/LYY4/xwAMP0KNHD7p27coLL7yQ0Hbo27cvw4cPp3v37gwfPpycnPLFIyt6TQDjxo1j0KBBcQ/ttGjRgosuuohu3bpx8skn06dPn4Riqo7hw4fTpk0bsrKyGDNmjEqfxxHvEK00TEkr921mI4BB7n5hOHwOcLi7j4/TdiowM5FDTOlY7js/P5+hQ4eyePHilMWQDqZOnUpubi5//etfUx3KTtkVSp/vzP9M2R5E2SHa2toLl+RqsOW+zWwcMA6CG+uIJJNKn1eu7BCt+iAavmTuQfQDJrr7yeHwdQDufmuctlOpx3sQIvWN/md2PTXZg0hmH8QCINPMOphZE2AUMCOJ6xMRkVqUtATh7tuB8cBrwFLgaXdfYmaTzGwYgJn1MbMCYCRwv5ktSVY8IiJSPUntg3D3l4GXY8bdGPV8AdAmmTGIiEjN7LK1mEREpHJKELXkyCOPBIJTXh9/vHav+/vDH/4Qd13pbMiQIWzYsCGp65g8eTJdunTh7LPPZs6cOZFChKk0depUvvjii2rPd9999/HII49U2iY3N5fLL7+8pqGJVF91izel+lEbxfqSafbs2XGL5lWmuLi40unRRfXqWryifOmiU6dOvmbNGnd3v+mmmyIFEhNV1XavSGXbJLoYY3Xmq2vp9D8jdYMaFOvbdfcgPn4ZXro6+FsL9txzTwAmTJjA3Llzyc7O5q677qKkpIRrrrmGPn360L17d+6//34guMFP//79GTZsWOQeDKeffjq9e/ema9eukcJ7EyZMoKioiOzsbM4+++wd1jVq1CheeumlSAxlJa8rWme0ZJcob9++PevWrSM/P58uXbpw0UUX0bVrV0466SSKiooAWLBgAd27dyc7O5trrrmGbt26lYtz06ZNnHDCCZHS3WVXdF988cWsWrWKwYMHc9ddd3Hfffdx1113kZ2dzdy5cyksLGT48OH06dOHPn368M477wAwceJEzjnnHI466ijOOeecHdY1Z84cjjnmGE455RQ6derExRdfHKmbFVt2fOHChRx77LH07t2bk08+mS+//JLp06eTm5vL2WefTXZ2NkVFReW25d///nf69OlDjx49GD58eGSbT5w4kTvvvBOAAQMGcO2119K3b19++tOfMnfu3Eh8ZTeFmjhxIueffz4DBgygY8eOTJ48OfI6brnlFjp16sTRRx/N6NGjI8sVqbbqZpRUP2plD2LpS+6/b+1+097B36UvVW/+OMp+5cfuQdx///1+yy23uLv71q1bvXfv3r5q1SqfPXu2N2/e3FetWhVpW1bKuqxU9rp163ZYduy6nn32WT/33HPd3f2HH37wNm3a+JYtWypcZ7RkligvW0ZhYaGvXr3aMzIy/IMPPnB395EjR/qjjz7q7u5du3b1d999193dr732Wu/atWu57VpcXOwbN26MrPvQQw/10tLSHdbhXn4PYvTo0T537lx3d//000+9c+fOkXa9evXyLVu2lFvX7Nmzfffdd/eVK1f69u3b/cQTT4yU+iaq7Pi2bdu8X79+/vXXX7u7+5NPPum/+MUvym2reNuy7D11d//d734XKUseHf+xxx7rv/71r93d/aWXXvITTjghEl/ZZ+umm27yfv36+datW72wsND3228/37Ztm7///vveo0cPLyoq8u+++85/8pOfxN2z0h7ErgeV+07QyregOPgVS3FRMNx5SFJW9frrr/PRRx9FCstt3LiR5cuX06RJE/r27UuHDh0ibSdPnsxzzz0HwJo1a1i+fHmkdHY8gwcP5oorruCHH37g1Vdf5ZhjjqFZs2YVrjN6XZC8EuVt27bdoU2HDh3Izs4GgvLb+fn5bNiwge+//z5S5vvnP/85M2fOLLd8d+e3v/0t//rXv2jUqBGff/45a9eurfLq5jfeeGOHW6h+9913kRstDRs2jGbNmsWdr2/fvnTs2BGA0aNHM2/ePEaMGLFD2fFPPvmExYsXM3DgQCC4091BBx1UYSzR23Lx4sVcf/31bNiwgU2bNnHyySfHnSe6XHl+fn7cNqeccgq77747u+++OwcccABr167lnXfe4bTTTqNp06Y0bdo0bin0dKKqsOlt10wQhx4Pi/4ZJIfGzYLhJHF3/vKXv5T7IpgzZ84OhfPmzJnDG2+8wfz582nevDkDBgyosJR1maZNmzJgwABee+01nnrqKUaNGlXpOmMlq0R5rNg2ZYeYEvHYY49RWFjIwoULady4Me3bt69yu0BQsvy9996LW968JuXKo8uOuztdu3aNVO+tSvT6xo4dy/PPP0+PHj2YOnVqhdViq1uuvLJ26Sq6ptO03ALVdEpDu2YfROchMPxB6HNR8LcW9x5iy1WffPLJ3HvvvRQXFwOwbNkyNm/eXG6+jRs3su+++9K8eXM+/vhj3nvvvci0xo0bR+aPddZZZ/HQQw8xd+5cBg0aVK11JqtEeSJatGjBXnvtxb///W8AnnzyybjtNm7cyAEHHEDjxo2ZPXs2n376adx2sdv9pJNO4i9/+UtkONH7Z7z//vusXr2a0tJSnnrqqbjbpFOnThQWFka2XXFxMUuWLIkbR6zvv/+egw46iOLiYh577LGEYqqOo446ihdffJGtW7eyadOmuHtl6UJVYdPfrpkgIEgKp9xZ64eWunfvTkZGBj169OCuu+7iwgsvJCsri169etGtWzd++ctfxv2lN2jQILZv306XLl2YMGFC5O5rEJTA7t69e6STOtpJJ53E22+/zYknnkiTJk0AEl5nskqUJ+qBBx7goosuIjs7m82bN8ctq3322WeTm5vLYYcdxiOPPELnzp3jLuvUU0/lueeei3RST548mdzcXLp3705WVhb33XdfQjH16dOH8ePH06VLFzp06MDPfvazcm2aNGnC9OnTufbaa+nRowfZ2dmRU2zL7tld1kkd65ZbbuHwww/nqKOOqvC17Iw+ffowbNgwunfvzuDBgznssMPStlx5/8xWNGsc7JXV1o27pJZVt9Mi1Y90P821vli9enXcTuG69P3330ee33rrrX755ZenMJqanaKcjsq26+bNm713796+cOHCcm3S5X/m9SVf+Q3P/3en7wsvVUOd1FKfvPTSS9x6661s376ddu3aMXXq1FSH1CCMGzeOvLw8tm7dynnnnUevXr1SHVKFBma1Vr9DGktaue9kUblvkZ2n/5ldT7qV+65T9S3RiaSK/lckUQ0iQTRt2pT169frgy9SBXdn/fr1cU//ra9m5a3lxhcWMytvbapDaXAaRB9EmzZtKCgooLBQp8mJVKVp06a0adMwquzX5rUUtXHRXm1d+JcusTSIBNG4ceNyVwmLSMMX71qKmnwZ1kaiqa1klaxYaiKph5jMbJCZfWJmK8xsQpzpu5vZU+H0f5tZ+2TGIyINS21dS1EbF+3V1oV/6RRL0hKEmWUA9wCDgSxgtJllxTS7APjW3X8C3AXcnqx4RKThGZjVmsmje3Juv3Y7dXipNhJNbSWrdIolaae5mlk/YKK7nxwOXwfg7rdGtXktbDPfzHYDvgJaeSVBxTvNVURkZ6XLcf9kxVKT01yT2QdxMLAmargAOLyiNu6+3cw2AvsDO9RzMLNxwLhw8AczW5yUiGuuJTExp4F0jAnSMy7FlBjFlLh0jKtTdWeoF53U7j4FmAJgZrnVzYLJppgSl45xKabEKKbEpWNcZlbtQy/J7KT+HIi+MUCbcFzcNuEhpn2A9UmMSUREEpTMBLEAyDSzDmbWBBgFzIhpMwM4L3w+Anirsv4HERGpO0k7xBT2KYwHXgMygAfdfYmZTSKoKjgDeAB41MxWAN8QJJGqTElWzDtBMSUuHeNSTIlRTIlLx7iqHVO9K9YnIiJ1o0HUYhIRkdqnBCEiInHVqwRRVemOOorhQTP7OvpaDDPbz8xmmdny8O++dRxTWzObbWZ5ZrbEzK5IdVxm1tTM3jezD8OYbg7HdwjLqqwIy6w0qauYomLLMLMPzGxmGsWUb2b/NbNFZacjpsHnqoWZTTezj81sqZn1S/FnqlO4fcoe35nZlWmwnf5f+BlfbGZPhJ/9lH6mzOyKMJ4lZnZlOK7a26neJIgES3fUhanAoJhxE4A33T0TeDMcrkvbgavcPQs4Args3DapjOsH4Hh37wFkA4PM7AiCcip3heVVviUot1LXrgCWRg2nQ0wAx7l7dtT586n+XP0ZeNXdOwM9CLZZymJy90/C7ZMN9Aa2AM+lMiYzOxi4HMhx924EJ+SMIoWfKTPrBlwE9CV434aa2U+oyXaq7j1KU/UA+gGvRQ1fB1yXoljaA4ujhj8BDgqfHwR8kuJt9QIwMF3iApoD/yG4kn4dsFu897SOYmkT/nMcD8wELNUxhevNB1rGjEvZ+0dwTdJqwhNZ0iGmmDhOAt5JdUz8WA1iP4KzQmcCJ6fyMwWMBB6IGr4B+E1NtlO92YMgfumOg1MUS6zW7v5l+PwrIGU32Q0r4vYE/k2K4woP5SwCvgZmASuBDe6+PWySivfwboJ/ltJweP80iAnAgdfNbGFYWgZS+/51AAqBh8LDcf8wsz1SHFO0UcAT4fOUxeTunwN3Ap8BXwIbgYWk9jO1GOhvZvubWXNgCMEFydXeTvUpQdQLHqTnlJw7bGZ7As8AV7r7d6mOy91LPDgc0IZgd7dzXa4/lpkNBb5294WpjKMCR7t7L4JDqJeZ2THRE1Pw/u0G9ALudfeewGZiDkmk6rMeHs8fBkyLnVbXMYXH8U8jSKj/A+xB+UPQdcrdlxIc4nodeBVYBJTEtEloO9WnBJFI6Y5UWWtmBwGEf7+u6wDMrDFBcnjM3Z9Nl7gA3H0DMJtgV7tFWFYF6v49PAoYZmb5wJMEh5n+nOKYgMgvUdz9a4Lj6n1J7ftXABS4+7/D4ekECSMdPlODgf+4e9k9RlMZ04nAancvdPdi4FmCz1lKP1Pu/oC793b3Ywj6QJZRg+1UnxJEIqU7UiW6ZMh5BH0AdcbMjOCq9KXu/qd0iMvMWplZi/B5M4I+kaUEiWJEKmJy9+vcvY27tyf4/Lzl7menMiYAM9vDzPYqe05wfH0xKXz/3P0rYI2ZlVUAPQHIS2VMUUbz4+ElSG1MnwFHmFnz8P+wbDul+jN1QPj3EOAM4HFqsp3qquOkljpfhhBkwpXA71IUwxMExxqLCX5lXUBwHPtNYDnwBrBfHcd0NMHu4kcEu5OLwm2VsriA7sAHYUyLgRvD8R2B94EVBIcIdk/R+zgAmJkOMYXr/zB8LCn7bKfB5yobyA3fw+eBfdMgpj0ICnruEzUu1THdDHwcfs4fBXZPg8/UXIJE9SFwQk23k0ptiIhIXPXpEJOIiNQhJQgREYlLCUJEROJSghARkbiUIEREJC4lCKmx8FL+ssqaX5nZ51HD5apXhtUkL05gubuZ2YYKxpeEy18cVslsVluvp6bMrJElqbqwmV1oZoXha15qZucnYz1VxFBQdk1L1LhmZvZ6+D78Mmr8A2bWva5jlORQgpAac/f1/mN1zfsIqldmh49tcWbZD6gyQVTh+3B9h4XDFyU6Y1gROBkaUYMKotWI57HwNR8H3GFmLWOWk7RbB1diMPAWQbXQsWEcvYASd/8oBfFIEihBSFKY2W/CX5eLzexX4ejbgLKa/reZ2d5m9paZ/cfMPgprJSXEgwt45gI/Cdf3YljobomZXRiO283MNpjZ3Wb2EdDXzG42swVhXPeFV79iZvPM7E9mlmvBfTVyzOw5C2rnT4x6XedZcJ+LRWb2NzNrFL6uvcJxj1TUroJ4/hiu7yMzu72K1/wVQdXXQ8zs92b2iJm9A0wN9zTujorzVTM7Omqdt1lwb475UVfZtjazZ8PX/L4F5djLroKfFW7L+wkq3sYqJqjSG52cJgE3JvgWSn1Ql1f36dFwH8BE4Orw+eEEV3A2A/YiKLFxGMGX+aKoeRoDe4fPDwCWh893I6iGGbuOyPhw3pnAReHwfuHf5gRXkO4btnfgjKhllLUzgqviB4fD84D/DZ9fRXCVfGugKfAF0ALoRnBFcVkZ5ynAz2PjraJdJJ5w+Uv48d7wLeK85guBu8PnPyGosNoC+D3BlbpNY9uFw68SXGFfts6y1/knYEL4/CngiPB5e8IS9sDfgN+Gz08L528RE1djgnpWHwBnEZRzuD7Vn0M9aveRil1TafiOBp5x9yIAM3se6E9QXTKaAbeZ2dEE5bfbhodPyvU/RNnLghLiAG8T3MAJ4P+Z2bDweRvgUIKSI9sIit+VOcHMriH44m9JUJr5lXBaWW2v/wL/9bAYnAXF/doQFGbrA+SGOx7N2LEEfZnK2kXH8034uv9uZi8RJLx4zjazAQQ3YbrQ3TeEy33B3bdWME+0Incve40LCd6Lsjg7hcsC2Dfs0zmGoFQL7v6CmX0fu0APCtONgkh11VcJCiHeTbCtHnL3lxKITdKYEoSk0rkEN6bp5e7bzayA4Iu7MmV9EBFmdiLBl9oR7l5kZvOillPkHvzktaA2/l/D9X1uZr+PWd8P4d/SqOdlw7sRJLQH3f2GmPXH/h9V1i4Sj7sXm1kOQSHDkcAlBIX6Yj3m7lfGGb856vl2djxkHP26ovuDSvjx/96Avh7TXxSVMBL1K+BBgsRTSLAH9iagBFHPqQ9CkmEu8LPwTJc9CQ5TzAW+JzjkVGYfgvszbDezgdT8pir7AN+EyaErwa/3eJoRfNmvs6B66vBqrucN4MyyTmILzuI6xMMbw0QlirjtYhcWxrC3u88E/h/BjZ5qKh/oaYH2BLfkTOT1XBYVT1ni/RfBITHM7FR2fM92YGb7E9xB7TGCw3tlN2JK+dllsvO0ByG1zt3fN7MnCEq0Q3DTmf8ChB3J/yX4dfkn4MVw+H2CKpM18RIwzszyCG6r+O94jdx9vZk9TNBH8WVF7Sri7v81s5uBN8LO6WKCs7I+Iyi3/pGZ5br7uRW0+yJmkfsAz5rZ7gQ/1n5dnXhivE1wz4GlBP0aiypvDgTJ4V4z+wXBd8HscNxNwBNmNgZ4J07c0SYCk9zdzewVgr2gMQT3j5d6TtVcRUQkLh1iEhGRuJQgREQkLiUIERGJSwlCRETiUoIQEZG4lCBERCQuJQgREYnr/wOhLF10ae+JxAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "acc_iter_list_iter = [0.7085,0.7057,0.6995,0.6917,0.6810,0.6675,0.6463,0.5813]\n",
    "sparsity_list_iter = [47,51.73,61.13,65.84,70.54,75.27,79.95,84.65]\n",
    "\n",
    "testing_accuracy_list = [0.7122,0.7132,0.7123,0.7136,0.7147,0.7087,0.698,0.685,0.6603,0.5976,0.4962,0.4495,0.2801,0.108,0.05,0.0173,0.0105,0.0099,0.01,0.01]\n",
    "sparsity_list = [0.01,4.71,9.41,14.11,18.81,23.51,28.22,32.92,37.62,42.33,47.03,51.73,56.43,61.13,65.80,70.54,75.24,79.94,84.64,89.35]\n",
    "\n",
    "plt.axhline(y=0.7122, color='black', linestyle='--',label = 'no pruning')\n",
    "plt.scatter(sparsity_list,testing_accuracy_list,s = 10, label = 'pruned after pretraining')\n",
    "plt.scatter(sparsity_list_iter,acc_iter_list_iter,s = 10, label = 'iterative pruning after pretraining')\n",
    "plt.xlim([0,90])\n",
    "plt.ylim([0,0.9])\n",
    "plt.legend()\n",
    "plt.xlabel('Total Parameters Pruned %')\n",
    "plt.ylabel('Testing Accuracy')\n",
    "plt.title('Pruning')\n",
    "plt.savefig('Iterative Pruning')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Quantization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def quantization(model,bits = 8):\n",
    "    cluster_centers = []\n",
    "    num_layers = len(model.layers)\n",
    "\n",
    "    for layer_idx in range(num_layers):\n",
    "        if len(model.layers[layer_idx].weights) !=0:\n",
    "            for weight_idx in range(len(model.layers[layer_idx].weights)):\n",
    "                # find conv and fc layer\n",
    "                if (len(model.layers[layer_idx].weights[weight_idx].shape) == 4) or (len(model.layers[layer_idx].weights[weight_idx].shape) == 2):\n",
    "                    #extract weight matrix\n",
    "                    weight = model.layers[layer_idx].get_weights()[weight_idx]\n",
    "                    #create mask matrix for weight matrix\n",
    "                    mask = np.ones((weight.shape))\n",
    "                    mask[weight ==0] = 0 \n",
    "                    masked_weight = (weight * mask).flatten().reshape((-1,1)) #2D\n",
    "                    \n",
    "                    # shrink the weight \n",
    "                    weight_shrinked = masked_weight[masked_weight!=0].reshape((-1,1))\n",
    "                    \n",
    "                    # Apply K-means algorithm to find centroids\n",
    "                    kmeans = KMeans(2**bits).fit(weight_shrinked)\n",
    "                    cluster_centers_layer = kmeans.cluster_centers_.reshape(kmeans.cluster_centers_.shape[0])            \n",
    "                    predicted_idx = kmeans.predict(masked_weight)\n",
    "\n",
    "                    #assign centered weight value using index still 2D array\n",
    "                    masked_weight = cluster_centers_layer[predicted_idx]\n",
    "\n",
    "                    quantized_mask_weight = masked_weight.flatten().reshape(weight.shape) * mask\n",
    "                    cluster_centers.append(cluster_centers_layer)\n",
    "                    K.set_value(model.layers[layer_idx].weights[weight_idx],quantized_mask_weight)\n",
    "    \n",
    "    return model,cluster_centers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make sure change the value to run the next chunk of code\n",
    "value = 80"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## input images 32 x 32\n",
    "inputs = keras.Input(img_dim)\n",
    "## call DenseNet-BC-100-12 with data augmentation\n",
    "outputs = DenseNet(inputs,init_nb_filters,growth_rate,nb_layers,theta,dropout_rate,weight_decay)\n",
    "## assign the actual model\n",
    "model = keras.Model(inputs,outputs)\n",
    "\n",
    "model.load_weights('./densenet_100_aug_40.h5')\n",
    "nb_epoch = 40\n",
    "end_step = np.ceil(1.0 * 45000 / batch_size).astype(np.int32) * nb_epoch\n",
    "pruning_schedule = tfmot.sparsity.keras.PolynomialDecay(\n",
    "                  initial_sparsity=0.0, final_sparsity= value/100,\n",
    "                  begin_step=2000, end_step=end_step,frequency=100)\n",
    "\n",
    "model_pruned = tfmot.sparsity.keras.prune_low_magnitude(model, pruning_schedule=pruning_schedule)\n",
    "\n",
    "optimizer = keras.optimizers.SGD(lr=.001, momentum=0.9, nesterov=True)\n",
    "model_pruned.compile(\n",
    "loss=tf.keras.losses.categorical_crossentropy,\n",
    "optimizer=optimizer,\n",
    "metrics=['accuracy'])\n",
    "   \n",
    "model_pruned.load_weights('./iter_pruned_{}.h5'.format(value))\n",
    "\n",
    "# get rid of extra parameters during pruning\n",
    "final_model = sparsity.strip_pruning(model_pruned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "quantized_model,centers = quantization(final_model,bits = 6)\n",
    "# actual training \n",
    "## define optimizer:SGD \n",
    "optimizer = keras.optimizers.SGD(lr=.1, momentum=0.9, nesterov=True)\n",
    "quantized_model.compile(loss='categorical_crossentropy',optimizer=optimizer,metrics=['accuracy'])\n",
    "acc_quan = quantized_model.evaluate(x_test,y_test,batch_size)\n",
    "sparsity = get_total_sparsity(quantized_model)\n",
    "print(acc_quan,sparsity)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Huffman Coding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def huffman_coding_per_layer(weight, centers):\n",
    "    \"\"\"\n",
    "    Huffman coding for each layer\n",
    "    :param weight: weight parameter of the current layer.\n",
    "    :param centers: KMeans centroids in the quantization codebook of the current weight layer.\n",
    "    :return: \n",
    "            'encodings': Encoding map mapping each weight parameter to its Huffman coding.\n",
    "            'frequency': Frequency map mapping each weight parameter to the total number of its appearance.\n",
    "\n",
    "    Generate Huffman Coding and Frequency Map according to incoming weights and centers (KMeans centriods).\n",
    "    \"\"\"\n",
    "    weight = weight.flatten()\n",
    "    weight = weight[weight!=0]\n",
    "    frequency = {}\n",
    "    \n",
    "    for w in weight:\n",
    "        if w not in frequency:\n",
    "            frequency[w] = 0\n",
    "        frequency[w] +=1\n",
    "    \n",
    "    encodings = {}\n",
    "    \n",
    "    nb_unique_w = len(frequency)\n",
    "    freq_val = [(v,[k]) for k,v in frequency.items()]\n",
    "    heapq.heapify(freq_val)\n",
    "    \n",
    "    while len(freq_val) > 1:\n",
    "        tempa = heapq.heappop(freq_val)\n",
    "        tempb = heapq.heappop(freq_val)\n",
    "        \n",
    "        for element in tempa[1]:\n",
    "            if element not in encodings:\n",
    "                encodings[element] = '0'\n",
    "            else:\n",
    "                encodings[element] = '0' + encodings[element]\n",
    "        for element in tempb[1]:\n",
    "            if element not in encodings:\n",
    "                encodings[element] = '1'\n",
    "            else:\n",
    "                encodings[element] = '1' + encodings[element]\n",
    "        \n",
    "        temp = (tempa[0]+tempb[0],tempa[1]+tempb[1])\n",
    "        heapq.heappush(freq_val,temp)  \n",
    "    \n",
    "    return encodings, frequency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make sure change the value to run the next chunk of code\n",
    "value = 80"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## input images 32 x 32\n",
    "inputs = keras.Input(img_dim)\n",
    "## call DenseNet-BC-100-12 with data augmentation\n",
    "outputs = DenseNet(inputs,init_nb_filters,growth_rate,nb_layers,theta,dropout_rate,weight_decay)\n",
    "## assign the actual model\n",
    "model = keras.Model(inputs,outputs)\n",
    "\n",
    "model.load_weights('./densenet_100_aug_40.h5')\n",
    "nb_epoch = 40\n",
    "end_step = np.ceil(1.0 * 45000 / batch_size).astype(np.int32) * nb_epoch\n",
    "pruning_schedule = tfmot.sparsity.keras.PolynomialDecay(\n",
    "                  initial_sparsity=0.0, final_sparsity= value/100,\n",
    "                  begin_step=2000, end_step=end_step,frequency=100)\n",
    "\n",
    "model_pruned = tfmot.sparsity.keras.prune_low_magnitude(model, pruning_schedule=pruning_schedule)\n",
    "\n",
    "optimizer = keras.optimizers.SGD(lr=.001, momentum=0.9, nesterov=True)\n",
    "model_pruned.compile(\n",
    "loss=tf.keras.losses.categorical_crossentropy,\n",
    "optimizer=optimizer,\n",
    "metrics=['accuracy'])\n",
    "   \n",
    "model_pruned.load_weights('./iter_pruned_{}.h5'.format(value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# illustration of huffman coding for the first convolutional layer\n",
    "final_model = sparsity.strip_pruning(model_pruned)\n",
    "quantized_model,centers = quantization(final_model,bits = 6)\n",
    "weight = quantized_model.layers[3].get_weights()[0]\n",
    "encodings,frequency = huffman_coding_per_layer(weight, centers[0])"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "BME Project 3.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}