FINAL_VGG.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "FINAL_VGG.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "evdW9t5NA3X7",
        "colab_type": "text"
      },
      "source": [
        "# COMP 551 Project 3 - Eric Liu, Ajay Patel, Aaron Sossin\n",
        "VGG Net Model\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lMDgMqgZ_GEi",
        "colab_type": "text"
      },
      "source": [
        "## Install libraries and datasets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rhnl2znqFLFi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install tensorflow\n",
        "\n",
        "from google.colab import files\n",
        "files.upload()\n",
        "!pip install -q kaggle\n",
        "!mkdir -p ~/.kaggle\n",
        "!cp kaggle.json ~/.kaggle/\n",
        "!kaggle competitions download -c modified-mnist\n",
        "!unzip test_max_x.zip\n",
        "!unzip train_max_x.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4QjO_Foy_Kxb",
        "colab_type": "text"
      },
      "source": [
        "## Train model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3L4btqo4FQ0Q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras import utils, Sequential, layers, callbacks\n",
        "from tensorflow.keras.optimizers import Adam, SGD\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "train_images = pd.read_pickle('train_max_x')\n",
        "train_labels = np.array(pd.read_csv('train_max_y.csv'))[:,1]\n",
        "\n",
        "x_train, x_test, y_train, y_test = train_test_split(train_images, train_labels, test_size=0.15, stratify=train_labels)\n",
        "x_train = x_train.reshape(x_train.shape[0], 128, 128, 1)\n",
        "x_test = x_test.reshape(x_test.shape[0], 128, 128, 1)\n",
        "x_train = x_train.astype('float64')\n",
        "x_test = x_test.astype('float64')\n",
        "\n",
        "# normalize\n",
        "x_train /= 255.0\n",
        "x_test /= 255.0\n",
        "\n",
        "y_train = utils.to_categorical(y_train, 10)\n",
        "y_test = utils.to_categorical(y_test, 10)\n",
        "\n",
        "# HYPERPARAMS\n",
        "batch_size = 32\n",
        "epochs = 25\n",
        "input_shape = (128, 128, 1)\n",
        "num_classes = 10\n",
        "learning_rate = 0.001\n",
        "\n",
        "model = Sequential()\n",
        "model.add(layers.Conv2D(32, kernel_size=(3,3), activation='relu', input_shape=input_shape))\n",
        "model.add(layers.BatchNormalization())\n",
        "model.add(layers.Conv2D(32, kernel_size=(3,3), activation = 'relu'))\n",
        "model.add(layers.BatchNormalization())\n",
        "model.add(layers.MaxPooling2D(pool_size=(2,2)))\n",
        "\n",
        "model.add(layers.Conv2D(64, kernel_size=(3,3), activation='relu'))\n",
        "model.add(layers.BatchNormalization())\n",
        "model.add(layers.Conv2D(64, kernel_size=(3,3), activation = 'relu'))\n",
        "model.add(layers.BatchNormalization())\n",
        "model.add(layers.MaxPooling2D(pool_size=(2,2)))\n",
        "\n",
        "model.add(layers.Conv2D(128, kernel_size=(3,3), activation='relu'))\n",
        "model.add(layers.BatchNormalization())\n",
        "model.add(layers.Conv2D(128, kernel_size=(3,3), activation = 'relu'))\n",
        "model.add(layers.BatchNormalization())\n",
        "model.add(layers.MaxPooling2D(pool_size=(2,2)))\n",
        "\n",
        "model.add(layers.Conv2D(256, kernel_size=(3,3), activation='relu'))\n",
        "model.add(layers.BatchNormalization())\n",
        "model.add(layers.Conv2D(256, kernel_size=(3,3), activation = 'relu'))\n",
        "model.add(layers.BatchNormalization())\n",
        "model.add(layers.MaxPooling2D(pool_size=(2,2)))\n",
        "\n",
        "model.add(layers.Flatten())\n",
        "model.add(layers.Dense(10, activation='softmax'))\n",
        "\n",
        "optimizer = Adam(learning_rate=learning_rate)\n",
        "model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])\n",
        "model.summary()\n",
        "\n",
        "lr_reducer = callbacks.ReduceLROnPlateau(monitor='val_acc',patience=5,verbose=1,factor=0.5)\n",
        "early_stop = callbacks.EarlyStopping(monitor='val_acc',patience=10,verbose=1)\n",
        "callbacks = [lr_reducer, early_stop]\n",
        "\n",
        "history = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(x_test, y_test), callbacks=callbacks) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dw9CRiHSC3rB",
        "colab_type": "text"
      },
      "source": [
        "## TRAINING RESULTS (10 epochs unless specified otherwise, results on last epoch)\n",
        "> Always training on 85% of data, validating on 15% of data.\n",
        "\n",
        "CALLBACK CONFIGURATION:\n",
        "- ReduceLROnPlateau(monitor='val_acc',patience=3,verbose=1,factor=0.5)\n",
        "- EarlyStopping(monitor='val_acc',patience=5,verbose=1)\n",
        "\n",
        "### Adam Optimizer\n",
        "- No callbacks, batch size 32: training accuracy of 0.9676 and validation accuracy of 0.9365. Max validation accuracy of 0.9365 on epoch 10\n",
        "- No callbacks, batch size 64: training accuracy of 0.9735 and validation accuracy of 0.9348. Max validation accuracy of 0.9348 on epoch 10\n",
        "- No callbacks, batch size 128: training accuracy of 0.9771 and validation accuracy of 0.9185. Max validation accuracy of 0.9377 on epoch 9\n",
        "- LRReducer and EarlyStop callbacks, batch size 32: training accuracy of 0.9680 and validation accuracy of 0.9208. Max validation accuracy of 0.9347 on epoch 8.\n",
        "- LRReducer and EarlyStop callbacks, batch size 64: training accuracy of 0.9672 and validation accuracy of 0.9351. Max validation accuracy of 0.9376 on epoch 7.\n",
        "- LRReducer and EarlyStop callbacks, batch size 128: training accuracy of 0.9776 and validation accuracy of 0.9140. Max validation accuracy of 0.9372 on epoch 9.\n",
        "\n",
        "25 epochs\n",
        "- LRReducer and EarlyStop callbacks, batch size 32: training accuracy of 0.9998 and validation accuracy of 0.9563. **Early stopped on epoch 23 (stagnation)**. Max validation accuracy of 0.9571 on epoch 18, but difference is minimal. **SUBMIT THIS TO KAGGLE TONIGHT**\n",
        "\n",
        "100 epochs\n",
        "- LRReducer and EarlyStop callbacks, batch size 128: training accuracy of 0.9993 and validation accuracy of 0.9421. **Early stopped on epoch 31**. Max validation accuracy of 0.9483 on epoch 21.\n",
        "- LRReducer and EarlyStop callbacks, batch size 32: training accuracy of 0.9993 and validation accuracy of 0.9421. **Early stopped on epoch 31 (stagnation)**. Max validation accuracy of 0.9483 on epoch 21.\n",
        "\n",
        "### SGD Optimizer\n",
        "> Takes many more epochs to reach a good training and validation accuracy. Reaches a very high training accuracy much faster (0.98 at epoch 12), and peaks at near perfect training accuracy, but validation accuracy is quite low (overfitting). Efforts can be explored made to reduce overfitting (regularization).\n",
        "\n",
        "25 Epochs\n",
        "- LRReducer and EarlyStop callbacks, batch size 32: training accuracy of 0.9996 and validation accuracy of 0.8676. Max validation accuracy of 0.8776 on epoch 23.\n",
        "- Example of overfitting: at epoch 10, had training accuracy of 0.9325 yet validation accuracy of 0.3791\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PAnVlp5u_Mlr",
        "colab_type": "text"
      },
      "source": [
        "## Test model and output predictions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uyzjuRwlO25O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from google.colab import files\n",
        "\n",
        "test_images = pd.read_pickle('test_max_x')\n",
        "\n",
        "test = test_images.reshape(test_images.shape[0], 128, 128, 1)\n",
        "test = test.astype('float64')\n",
        "test /= 255\n",
        "\n",
        "predictions = model.predict(test, batch_size=batch_size)\n",
        "preds = []\n",
        "for pred in predictions:\n",
        "    preds.append(np.argmax(pred))\n",
        "print(preds)\n",
        "\n",
        "# write to csv\n",
        "import csv\n",
        "with open('predictions.csv', 'w') as file:\n",
        "    writer = csv.writer(file)\n",
        "    writer.writerow(['Id', 'Label'])\n",
        "    for i, p in enumerate(preds):\n",
        "        writer.writerow([str(i), str(p)])\n",
        "    \n",
        "files.download('predictions.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}