lego_classifier_gpu.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "lego_classifier.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Ladvien/lego_id_training_data/blob/master/lego_classifier_gpu.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tlixwBWhNNhb",
        "colab_type": "code",
        "outputId": "51bf751f-d323-448a-be89-1e4c6b36269a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 163
        }
      },
      "source": [
        "!git clone https://github.com/Ladvien/lego_id_training_data.git\n",
        "!mkdir ./data\n",
        "!mkdir ./data/output\n",
        "!ls"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'lego_id_training_data'...\n",
            "remote: Enumerating objects: 5766, done.\u001b[K\n",
            "remote: Counting objects: 100% (5766/5766), done.\u001b[K\n",
            "remote: Compressing objects: 100% (5763/5763), done.\u001b[K\n",
            "remote: Total 5766 (delta 1), reused 5766 (delta 1), pack-reused 0\n",
            "Receiving objects: 100% (5766/5766), 82.15 MiB | 11.01 MiB/s, done.\n",
            "Resolving deltas: 100% (1/1), done.\n",
            "data  lego_id_training_data  sample_data\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nlV9HYK0LT4C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#!/usr/bin/env python3\n",
        "# -*- coding: utf-8 -*-\n",
        "\"\"\"\n",
        "Created on Fri Aug 16 06:13:38 2019\n",
        "\n",
        "@author: ladvien and ezracc\n",
        "\"\"\"\n",
        "# Import Tensorflow\n",
        "import tensorflow as tf\n",
        "\n",
        "\n",
        "# Import needed tools.\n",
        "import os\n",
        "import matplotlib.pyplot as plt\n",
        "import json\n",
        "import numpy as np\n",
        "from scipy import stats\n",
        "\n",
        "# Import Keras\n",
        "import tensorflow.keras\n",
        "from tensorflow.keras.layers import Dense,Flatten, Dropout, Lambda\n",
        "from tensorflow.keras.layers import SeparableConv2D, BatchNormalization, MaxPooling2D, Conv2D, Activation\n",
        "from tensorflow.compat.v1.keras.preprocessing.image import ImageDataGenerator\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard, CSVLogger, ReduceLROnPlateau\n",
        "from tensorflow.keras.preprocessing import image\n",
        "\n",
        "# Tensorboard\n",
        "from tensorboard import program\n",
        "import webbrowser\n",
        "import time"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FbfxzR3Wx-CG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# https://medium.com/tensorflow/tf-keras-on-tpus-on-colab-674367932aa0\n",
        "# %load_ext watermark\n",
        "# %watermark -p tensorflow,numpy -m"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ihgBru0GyRUC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#################################\n",
        "# TODO: Make experiment folder\n",
        "#################################\n",
        "# 1. Generate a unique id\n",
        "# 2. Save weighs to folder\n",
        "# 3. Save tensorboard logs and open tensorboard to this folder\n",
        "\n",
        "#################################\n",
        "# Training Parameters\n",
        "#################################\n",
        "\n",
        "continue_training       = False\n",
        "\n",
        "input_shape             = (300, 300, 3) # This is the shape of the image width, length, colors\n",
        "image_size              = (input_shape[0], input_shape[1]) # DOH! image_size is (height, width)\n",
        "train_test_ratio        = 0.2\n",
        "zoom_range              = 0.2\n",
        "shear_range             = 0.2\n",
        "\n",
        "# Hyperparameters\n",
        "batch_size              = 16\n",
        "epochs                  = 180\n",
        "steps_per_epoch         = 100\n",
        "validation_steps        = 100 \n",
        "optimizer               = 'adadelta' \n",
        "learning_rate           = 1.0\n",
        "val_save_step_num       = 1\n",
        "\n",
        "path_to_graphs          = './data/output/logs/'\n",
        "model_save_dir          = './data/output/'\n",
        "train_dir               = './lego_id_training_data/train/'\n",
        "val_dir                 = './lego_id_training_data/test/'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ozmbq2OIF_Gy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#################################\n",
        "# Helper functions\n",
        "#################################\n",
        "\n",
        "def make_dir(dir_path):\n",
        "    if not os.path.exists(dir_path):\n",
        "        os.mkdir(dir_path)\n",
        "    \n",
        "\n",
        "def show_final_history(history):\n",
        "    fig, ax = plt.subplots(1, 2, figsize=(15,5))\n",
        "    ax[0].set_title('loss')\n",
        "    ax[0].plot(history.epoch, history.history['loss'], label='Train loss')\n",
        "    ax[0].plot(history.epoch, history.history['val_loss'], label='Validation loss')\n",
        "    ax[1].set_title('acc')\n",
        "    ax[1].plot(history.epoch, history.history['acc'], label='Train acc')\n",
        "    ax[1].plot(history.epoch, history.history['val_acc'], label='Validation acc')\n",
        "    ax[0].legend()\n",
        "    ax[1].legend()\n",
        "\n",
        "#################################\n",
        "# Create needed dirs\n",
        "#################################\n",
        "make_dir(model_save_dir)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vKZGj5ccacuZ",
        "colab_type": "code",
        "outputId": "c241443d-4d83-4334-b272-b0d2493049cd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "#################################\n",
        "# Data generators\n",
        "#################################\n",
        "\n",
        "# These Keras generators will pull files from disk\n",
        "# and prepare them for training and validation.\n",
        "augs_gen = ImageDataGenerator (\n",
        "    shear_range = shear_range,  \n",
        "    zoom_range = shear_range,        \n",
        "    horizontal_flip = True,\n",
        "    validation_split = train_test_ratio\n",
        ")  \n",
        "\n",
        "train_gen = augs_gen.flow_from_directory (\n",
        "    train_dir,\n",
        "    target_size = image_size, # THIS IS HEIGHT, WIDTH\n",
        "    batch_size = batch_size,\n",
        "    class_mode = 'sparse',\n",
        "    shuffle = True\n",
        ")\n",
        "\n",
        "test_gen = augs_gen.flow_from_directory (\n",
        "    val_dir,\n",
        "    target_size = image_size,\n",
        "    batch_size = batch_size,\n",
        "    class_mode = 'sparse',\n",
        "    shuffle = False\n",
        ")"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 4453 images belonging to 10 classes.\n",
            "Found 2715 images belonging to 10 classes.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o_PxhbV9ajrE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#################################\n",
        "# Save Class IDs\n",
        "#################################\n",
        "classes_json = train_gen.class_indices\n",
        "num_classes = len(train_gen.class_indices)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9GPqlfBfav61",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#################################\n",
        "# Model Building\n",
        "#################################\n",
        "\n",
        "with open(model_save_dir + 'classes.json', 'w') as fp:\n",
        "    json.dump(classes_json, fp, indent = 4)\n",
        "\n",
        "def ConvBlock(model, num_layers, filters):\n",
        "    for i in range(num_layers):\n",
        "        model.add(tensorflow.keras.layers.Conv2D(filters,(3,3), activation = 'selu'))\n",
        "        model.add(tensorflow.keras.layers.SeparableConv2D(filters, (3, 3), activation = 'selu'))\n",
        "        model.add(tensorflow.keras.layers.BatchNormalization())\n",
        "        model.add(tensorflow.keras.layers.MaxPooling2D((2, 2), strides=(2, 2)))\n",
        "    \n",
        "def FCN():\n",
        "    model = tensorflow.keras.models.Sequential()\n",
        "    model.add(tensorflow.keras.layers.Lambda(lambda x: x, input_shape = input_shape))\n",
        "    ConvBlock(model, 1, 32)\n",
        "    ConvBlock(model, 1, 64)\n",
        "    ConvBlock(model, 1, 128)\n",
        "    ConvBlock(model, 1, 256)\n",
        "    model.add(tensorflow.keras.layers.Flatten())\n",
        "    model.add(tensorflow.keras.layers.Dense(1024, activation = 'selu'))\n",
        "    model.add(tensorflow.keras.layers.Dropout(0.2))\n",
        "    model.add(tensorflow.keras.layers.Dense(num_classes, activation = 'softmax'))\n",
        "    return model\n",
        "\n",
        "# model = tf.keras.models.Sequential()\n",
        "# model.add(tf.keras.layers.BatchNormalization(input_shape=input_shape))\n",
        "# model.add(tf.keras.layers.Conv2D(64, (5, 5), padding='same', activation='elu'))\n",
        "# model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2), strides=(2,2)))\n",
        "# model.add(tf.keras.layers.Dropout(0.25))\n",
        "\n",
        "# model.add(tf.keras.layers.BatchNormalization(input_shape=input_shape))\n",
        "# model.add(tf.keras.layers.Conv2D(128, (5, 5), padding='same', activation='elu'))\n",
        "# model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))\n",
        "# model.add(tf.keras.layers.Dropout(0.25))\n",
        "\n",
        "# model.add(tf.keras.layers.BatchNormalization(input_shape=input_shape))\n",
        "# model.add(tf.keras.layers.Conv2D(256, (5, 5), padding='same', activation='elu'))\n",
        "# model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2), strides=(2,2)))\n",
        "# model.add(tf.keras.layers.Dropout(0.25))\n",
        "\n",
        "# model.add(tf.keras.layers.Flatten())\n",
        "# model.add(tf.keras.layers.Dense(256))\n",
        "# model.add(tf.keras.layers.Activation('elu'))\n",
        "# model.add(tf.keras.layers.Dropout(0.5))\n",
        "# model.add(tf.keras.layers.Dense(10))\n",
        "# model.add(tf.keras.layers.Activation('softmax'))\n",
        "# model.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wFvb5fpWaywG",
        "colab_type": "code",
        "outputId": "98f77bc5-037f-42b7-d892-6a4d5a636210",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "#################################\n",
        "# Create model\n",
        "#################################\n",
        "\n",
        "def get_optimizer(optimizer, learning_rate = 0.001):\n",
        "    if optimizer == 'adam':\n",
        "        return tensorflow.keras.optimizers.Adam(lr = learning_rate, beta_1 = 0.9, beta_2 = 0.999, epsilon = None, decay = 0., amsgrad = False)\n",
        "    elif optimizer == 'sgd':\n",
        "        return tensorflow.keras.optimizers.SGD(lr = learning_rate, momentum = 0.99) \n",
        "    elif optimizer == 'adadelta':\n",
        "        return tensorflow.keras.optimizers.Adadelta(lr=learning_rate, rho=0.95, epsilon=None, decay=0.0)\n",
        "\n",
        "selected_optimizer = get_optimizer(optimizer, learning_rate)\n",
        "\n",
        "model = FCN()\n",
        "model.summary()\n",
        "\n",
        "model.compile(\n",
        "    loss = 'sparse_categorical_crossentropy',\n",
        "    optimizer = selected_optimizer,\n",
        "    metrics = ['accuracy']\n",
        ")"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "W0828 14:26:00.165407 140010922551168 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/backend.py:4075: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "lambda (Lambda)              (None, 300, 300, 3)       0         \n",
            "_________________________________________________________________\n",
            "conv2d_3 (Conv2D)            (None, 298, 298, 32)      896       \n",
            "_________________________________________________________________\n",
            "separable_conv2d (SeparableC (None, 296, 296, 32)      1344      \n",
            "_________________________________________________________________\n",
            "batch_normalization_3 (Batch (None, 296, 296, 32)      128       \n",
            "_________________________________________________________________\n",
            "max_pooling2d_3 (MaxPooling2 (None, 148, 148, 32)      0         \n",
            "_________________________________________________________________\n",
            "conv2d_4 (Conv2D)            (None, 146, 146, 64)      18496     \n",
            "_________________________________________________________________\n",
            "separable_conv2d_1 (Separabl (None, 144, 144, 64)      4736      \n",
            "_________________________________________________________________\n",
            "batch_normalization_4 (Batch (None, 144, 144, 64)      256       \n",
            "_________________________________________________________________\n",
            "max_pooling2d_4 (MaxPooling2 (None, 72, 72, 64)        0         \n",
            "_________________________________________________________________\n",
            "conv2d_5 (Conv2D)            (None, 70, 70, 128)       73856     \n",
            "_________________________________________________________________\n",
            "separable_conv2d_2 (Separabl (None, 68, 68, 128)       17664     \n",
            "_________________________________________________________________\n",
            "batch_normalization_5 (Batch (None, 68, 68, 128)       512       \n",
            "_________________________________________________________________\n",
            "max_pooling2d_5 (MaxPooling2 (None, 34, 34, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv2d_6 (Conv2D)            (None, 32, 32, 256)       295168    \n",
            "_________________________________________________________________\n",
            "separable_conv2d_3 (Separabl (None, 30, 30, 256)       68096     \n",
            "_________________________________________________________________\n",
            "batch_normalization_6 (Batch (None, 30, 30, 256)       1024      \n",
            "_________________________________________________________________\n",
            "max_pooling2d_6 (MaxPooling2 (None, 15, 15, 256)       0         \n",
            "_________________________________________________________________\n",
            "flatten_1 (Flatten)          (None, 57600)             0         \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 1024)              58983424  \n",
            "_________________________________________________________________\n",
            "dropout_4 (Dropout)          (None, 1024)              0         \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 10)                10250     \n",
            "=================================================================\n",
            "Total params: 59,475,850\n",
            "Trainable params: 59,474,890\n",
            "Non-trainable params: 960\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CLEg2m-pa1vE",
        "colab_type": "code",
        "outputId": "e1dc0bdf-22b3-4cf3-e8b0-d1a3b896e4a6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "#################################\n",
        "# Keras Callbacks\n",
        "#################################\n",
        "best_model_weights = model_save_dir + 'base.model'\n",
        "\n",
        "checkpoint = ModelCheckpoint(\n",
        "    best_model_weights,\n",
        "    monitor = 'val_loss',\n",
        "    verbose = 1,\n",
        "    save_best_only = True,\n",
        "    mode = 'min',\n",
        "    save_weights_only = False,\n",
        "    period = val_save_step_num\n",
        ")\n",
        "\n",
        "earlystop = EarlyStopping(\n",
        "    monitor='val_loss',\n",
        "    min_delta=0.001,\n",
        "    patience=10,\n",
        "    verbose=1,\n",
        "    mode='auto'\n",
        ")\n",
        "\n",
        "tensorboard = TensorBoard(\n",
        "    log_dir = model_save_dir + '/logs',\n",
        "    histogram_freq=0,\n",
        "    batch_size=16,\n",
        "    write_graph=True,\n",
        "    write_grads=True,\n",
        "    write_images=False,\n",
        ")\n",
        "\n",
        "csvlogger = CSVLogger(\n",
        "    filename = model_save_dir + 'training.csv',\n",
        "    separator = ',',\n",
        "    append = False\n",
        ")\n",
        "\n",
        "reduce = ReduceLROnPlateau(\n",
        "    monitor='val_loss',\n",
        "    factor=0.5,\n",
        "    patience=40,\n",
        "    verbose=1, \n",
        "    mode='auto',\n",
        "    cooldown=1 \n",
        ")\n",
        "\n",
        "callbacks = [checkpoint, csvlogger, tensorboard]"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "W0828 14:26:03.091193 140010922551168 callbacks.py:875] `period` argument is deprecated. Please use `save_freq` to specify the frequency in number of samples seen.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vQRRa6daa5Em",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# tb = program.TensorBoard()\n",
        "# tb.configure(argv=[None, '--logdir', '/home/ladvien/lego_sorter/data/output/logs/'])\n",
        "# url = tb.launch()\n",
        "# # Wait for load\n",
        "# time.sleep(1)\n",
        "# webbrowser.open('http://localhost:6006')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OP0kVi10c9xy",
        "colab_type": "code",
        "outputId": "5a44fcd4-6be4-4176-ee8a-0b41a7bea88a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "#################################\n",
        "# Execute Training\n",
        "#################################\n",
        "\n",
        "if continue_training:\n",
        "    model.load_weights(best_model_weights)\n",
        "    model_score = model.evaluate_generator(test_gen, steps = validation_steps)\n",
        "\n",
        "    print('Model Test Loss:', model_score[0])\n",
        "    print('Model Test Accuracy:', model_score[1])\n",
        "\n",
        "\n",
        "history = model.fit_generator(\n",
        "    train_gen, \n",
        "    steps_per_epoch  = steps_per_epoch, \n",
        "    validation_data  = test_gen,\n",
        "    validation_steps = validation_steps,\n",
        "    epochs = epochs, \n",
        "    verbose = 1,\n",
        "    callbacks = callbacks\n",
        ")\n",
        "\n",
        "# tpu_model = tf.contrib.tpu.keras_to_tpu_model(\n",
        "#     model,\n",
        "#     strategy=tf.contrib.tpu.TPUDistributionStrategy(\n",
        "#         tf.contrib.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])\n",
        "#     )\n",
        "# )\n",
        "# tpu_model.compile(\n",
        "#     optimizer=tf.train.AdamOptimizer(learning_rate=1e-3, ),\n",
        "#     loss=tf.keras.losses.sparse_categorical_crossentropy,\n",
        "#     metrics=['sparse_categorical_accuracy']\n",
        "# )\n",
        "\n",
        "# show_final_history(history)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/180\n",
            " 99/100 [============================>.] - ETA: 1s - loss: 15.5442 - acc: 0.3427\n",
            "Epoch 00001: val_loss improved from inf to 7.03804, saving model to ./data/output/base.model\n",
            "100/100 [==============================] - 144s 1s/step - loss: 15.4074 - acc: 0.3449 - val_loss: 7.0380 - val_acc: 0.3338\n",
            "Epoch 2/180\n",
            " 26/100 [======>.......................] - ETA: 1:10 - loss: 2.4384 - acc: 0.5168"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kr1Nfm-8lGUc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 378
        },
        "outputId": "7f2e2e51-7449-42d8-fd60-3243b902199a"
      },
      "source": [
        "# tpu_model.fit_generator(\n",
        "#     generator = train_gen\n",
        "# )"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-46-92036924f873>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m tpu_model.fit_generator(\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mgenerator\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_gen\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m )\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   1431\u001b[0m         \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1432\u001b[0m         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1433\u001b[0;31m         steps_name='steps_per_epoch')\n\u001b[0m\u001b[1;32m   1434\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1435\u001b[0m   def evaluate_generator(self,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training_generator.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[0;34m(model, data, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch, mode, batch_size, steps_name, **kwargs)\u001b[0m\n\u001b[1;32m    176\u001b[0m       \u001b[0msamples\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnum_samples_or_steps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    177\u001b[0m       \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0;31m# Handle ProgBar as part of Callbacks once hooks are ready.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 178\u001b[0;31m       mode=mode)\n\u001b[0m\u001b[1;32m    179\u001b[0m   \u001b[0;31m# TODO(omalleyt): Handle ProgBar as part of Callbacks once hooks are ready.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    180\u001b[0m   \u001b[0mprogbar\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtraining_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_progbar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcount_mode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/callbacks.py\u001b[0m in \u001b[0;36mconfigure_callbacks\u001b[0;34m(callbacks, model, do_validation, batch_size, epochs, steps_per_epoch, samples, verbose, count_mode, mode)\u001b[0m\n\u001b[1;32m    118\u001b[0m   \u001b[0mcallback_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop_training\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    119\u001b[0m   \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 120\u001b[0;31m   \u001b[0;32mif\u001b[0m \u001b[0mcallback_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ckpt_saved_epoch\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    121\u001b[0m     \u001b[0;31m# The attribute `_ckpt_saved_epoch` is supposed to be None at the start of\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m     \u001b[0;31m# training (it should be made None at the end of successful multi-worker\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'KerasTPUModel' object has no attribute '_ckpt_saved_epoch'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wWAmxJ9_xQxz",
        "colab_type": "code",
        "outputId": "22490a34-f319-412c-f80d-d50cf87e0449",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 244
        }
      },
      "source": [
        "\n",
        "#################################\n",
        "# Test Image\n",
        "#################################\n",
        "\n",
        "# 1. Get each class and label.\n",
        "# 2. Generate nubmber of n predictions for each class.\n",
        "# 3. Take the mode of the predictions of each class.\n",
        "# 4. Compare the prediction mode against actual class.\n",
        "\n",
        "number_of_tests = 100\n",
        "number_correct = 0\n",
        "\n",
        "for key, item in classes_json.items():    \n",
        "    class_name = key\n",
        "    predictions = []\n",
        "    for i in range(0, number_of_tests):\n",
        "        img = image.load_img(f'./generated_images/test/{class_name}/{class_name}_{str(i)}.png', target_size = input_shape)\n",
        "        x = image.img_to_array(img)\n",
        "        x = np.expand_dims(x, axis=0)\n",
        "        classes = model.predict_classes(x, batch_size=1)\n",
        "        predictions.append(classes[0])\n",
        "    \n",
        "    \n",
        "    if int(stats.mode(np.asarray(predictions))[0]) == item:\n",
        "        print(f'{key} is correct')\n",
        "        number_correct += 1\n",
        "\n",
        "print(f'Correct classes: {number_correct}')\n",
        "#################################\n",
        "# Evaluate Training\n",
        "#################################\n",
        "model.load_weights(best_model_weights)\n",
        "model_score = model.evaluate_generator(test_gen, steps = validation_steps)\n",
        "\n",
        "print('Model Test Loss:', model_score[0])\n",
        "print('Model Test Accuracy:', model_score[1])\n",
        "    \n",
        "#################################\n",
        "# Save Model\n",
        "#################################\n",
        "model_json = model.to_json()\n",
        "with open(model_save_dir + 'model.json', 'w') as json_file:\n",
        "    json_file.write(model_json)\n",
        "    \n",
        "model.save(model_save_dir + 'model.h5')\n",
        "print('Weights Saved')\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-18-df9232117fc8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;31m# )\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m tpu_model = tf.contrib.tpu.keras_to_tpu_model(\n\u001b[0m\u001b[1;32m     21\u001b[0m     \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m     strategy=tf.contrib.tpu.TPUDistributionStrategy(\n",
            "\u001b[0;31mNameError\u001b[0m: name 'tf' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QNSq_4kWa4Do",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sSzRXwKsxXjL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hVbDyq67wZC6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Use a tf optimizer rather than a Keras one for now\n",
        "opt = tf.train.AdamOptimizer(learning_rate)\n",
        "model.compile(\n",
        "      optimizer=opt,\n",
        "      loss='categorical_crossentropy',\n",
        "      metrics=['acc'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8LciWldNw0gB",
        "colab_type": "code",
        "outputId": "b3de2921-5573-4300-e36e-80678126c9a0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 244
        }
      },
      "source": [
        "# TPU training.\n",
        "\n",
        "tpu_model = tensorflow.tpu (\n",
        "    model,\n",
        "    strategy=tf.contrib.tpu.TPUDistributionStrategy(\n",
        "        tf.contrib.cluster_resolver.TPUClusterResolver(TPU_ADDRESS)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-39-e5a30a4676d8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m tpu_model = tensorflow.tpu (\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     strategy=tf.contrib.tpu.TPUDistributionStrategy(\n\u001b[1;32m      5\u001b[0m         tf.contrib.cluster_resolver.TPUClusterResolver(TPU_ADDRESS)))\n",
            "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wSlqakTeNA1Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}