src/notebooks/cnn_model_development/CrackNetDevelopmentNotebookTensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CrackNet - Development Notebook (Tensorflow)\n",
    "\n",
    "Implementation in Tensorflow of the CNN described in Deep Learning-based Crack Detection Using Convolutional Neural Network and Naıve Bayes Data Fusion. [1]\n",
    "\n",
    "<img src=\"original_paper_cnn.png\">\n",
    "\n",
    "## References\n",
    "\n",
    "- [1] Chen, Fu-Chen & Jahanshahi, Mohammad. (2017). NB-CNN: Deep Learning-based Crack Detection Using Convolutional Neural Network and Naïve Bayes Data Fusion. IEEE Transactions on Industrial Electronics. PP. 1-1. 10.1109/TIE.2017.2764844.\n",
    "- [2] The code based on Convolution Model Application exercise from [Coursera's Deep Learning Specialization Course 4 - Convolutional Networks](http://www.coursera.org/learn/convolutional-neural-networks)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "WORKSPACE_BASE_PATH=\"/tf/notebooks/\" # Parent directory containing src, checkpoints, models, etc.\n",
    "CODE_BASE_PATH=\"/tf/notebooks/src/\" # Path were components are stored.\n",
    "DATA_BASE_PATH=\"/tf/notebooks/data/\" # Directory with data in case it is not inside WORKSPACE BASE path.\n",
    "sys.path.append(CODE_BASE_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from tensorflow.python.framework import ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_IMG_DIM = 64\n",
    "N_CLASSES = 2\n",
    "BATCH_SIZE = 32"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Dataset preparation\n",
    "\n",
    "For simplicity, Keras dataset generators will be used instead of tf.data pipeline."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dataset(dataset_path,training_path_prefix,test_path_prefix,batch_size,target_size):\n",
    "    train_datagen = ImageDataGenerator( rescale=1./255,\n",
    "                                        shear_range=0.2,\n",
    "                                        zoom_range=0.2,\n",
    "                                        horizontal_flip=True )\n",
    "\n",
    "    test_datagen = ImageDataGenerator(rescale=1./255)\n",
    "    training_set_generator = train_datagen.flow_from_directory(\n",
    "            dataset_path+training_path_prefix,\n",
    "            target_size,\n",
    "            batch_size=batch_size,\n",
    "            class_mode='categorical',\n",
    "            shuffle=True,\n",
    "            seed=42\n",
    "        )\n",
    "    test_set_generator = test_datagen.flow_from_directory(\n",
    "            dataset_path+test_path_prefix,\n",
    "            target_size,\n",
    "            class_mode='categorical',\n",
    "            shuffle=True,\n",
    "            seed=42\n",
    "        )\n",
    "    return training_set_generator,test_set_generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 32000 images belonging to 2 classes.\n",
      "Found 8000 images belonging to 2 classes.\n",
      "Training step size:  1000\n",
      "Validation step size:  250\n"
     ]
    }
   ],
   "source": [
    "training_set_generator,test_set_generator = load_dataset(\n",
    "    dataset_path=DATA_BASE_PATH+\"datasets/cracks_splitted8020/\",\n",
    "    training_path_prefix=\"train_set\",\n",
    "    test_path_prefix=\"test_set\",\n",
    "    batch_size = BATCH_SIZE,\n",
    "    target_size=(INPUT_IMG_DIM,INPUT_IMG_DIM) )  \n",
    "\n",
    "step_size_train=training_set_generator.n//training_set_generator.batch_size\n",
    "step_size_validation=test_set_generator.n//test_set_generator.batch_size\n",
    "\n",
    "print(\"Training step size: \", step_size_train)\n",
    "print(\"Validation step size: \", step_size_validation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Model Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_placeholders(height, width, n_channels, n_classes):\n",
    "    X = tf.placeholder(tf.float32, [None, height, width, n_channels])\n",
    "    Y = tf.placeholder(tf.float32, [None, n_classes])    \n",
    "    return X, Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize_parameters():\n",
    "    tf.set_random_seed(1)\n",
    "    \n",
    "    # [filter_height, filter_width, in_channels, out_channels]\n",
    "    W1 = tf.get_variable(\"W1\", [11, 11,  3, 32], initializer=tf.contrib.layers.xavier_initializer(seed=0))\n",
    "    W2 = tf.get_variable(\"W2\", [11, 11, 32, 48], initializer=tf.contrib.layers.xavier_initializer(seed=0))\n",
    "    W3 = tf.get_variable(\"W3\", [ 7,  7, 48, 64], initializer=tf.contrib.layers.xavier_initializer(seed=0))\n",
    "    #W4 = tf.get_variable(\"W4\", [ 5,  5, 64, 80], initializer=tf.contrib.layers.xavier_initializer(seed=0))\n",
    "    \n",
    "    parameters = {\n",
    "        \"W1\": W1,\n",
    "        \"W2\": W2,\n",
    "        \"W3\": W3\n",
    "        #\"W4\": W4\n",
    "    }    \n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_propagation(X, parameters):\n",
    "    W1 = parameters['W1']\n",
    "    W2 = parameters['W2']\n",
    "    W3 = parameters['W3']\n",
    "    #W4 = parameters['W4']\n",
    "    \n",
    "    # Conv2D 1\n",
    "    Z1 = tf.nn.conv2d(X, W1, strides=[1, 1, 1, 1], padding='SAME')\n",
    "    A1 = tf.nn.relu(Z1)\n",
    "    P1 = tf.nn.max_pool(A1, ksize = [1, 7, 7, 1], strides = [1, 2, 2, 1], padding='SAME')\n",
    "    \n",
    "    # Conv2D 2\n",
    "    Z2 = tf.nn.conv2d(P1, W2, strides=[1, 1, 1, 1], padding='SAME')\n",
    "    A2 = tf.nn.relu(Z2)\n",
    "    P2 = tf.nn.max_pool(A2, ksize = [1, 5, 5, 1], strides = [1, 2, 2, 1], padding='SAME')\n",
    "    \n",
    "    # Conv2D 3\n",
    "    Z3 = tf.nn.conv2d(P2, W3, strides=[1, 1, 1, 1], padding='SAME')\n",
    "    A3 = tf.nn.relu(Z3)\n",
    "    P3 = tf.nn.max_pool(A3, ksize = [1, 3, 3, 1], strides = [1, 2, 2, 1], padding='SAME')\n",
    "    \n",
    "    # Flatten\n",
    "    P = tf.contrib.layers.flatten(P3)\n",
    "    \n",
    "    # FC1\n",
    "    Z4 = tf.contrib.layers.fully_connected(P, 5120, activation_fn=None)\n",
    "    \n",
    "    # FC2\n",
    "    Z5 = tf.contrib.layers.fully_connected(Z4, 2, activation_fn=None)\n",
    "    \n",
    "    return Z5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_cost(Z5, Y):    \n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=Z5, labels=Y))    \n",
    "    return cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time, sys\n",
    "from IPython.display import clear_output\n",
    "\n",
    "def update_epoch_progress(epoch, batch, total_batches, last_train_accuracy=None, last_train_cost=None):\n",
    "    bar_length = 20\n",
    "    progress = batch/total_batches\n",
    "    if progress < 0:\n",
    "        progress = 0\n",
    "    if progress >= 1:\n",
    "        progress = 1\n",
    "    block = int(round(bar_length * progress))\n",
    "    clear_output(wait = True)\n",
    "    \n",
    "\n",
    "    stats =  \"Train Accuracy: \" + str(last_train_accuracy) +\", Train Cost: \" + str(last_train_cost)\n",
    "    \n",
    "    text = \"Epoch {0}: [{1}] Batch {2}/{3} {4}\".format( \n",
    "        epoch, \n",
    "        \"#\" * block + \"-\" * (bar_length - block), \n",
    "        batch, \n",
    "        total_batches, \n",
    "        stats\n",
    "    )\n",
    "    print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model( training_set_generator, test_set_generator, learning_rate=0.002,\n",
    "          num_epochs=100, step_size_train = None, minibatch_size = BATCH_SIZE ):\n",
    "    \"\"\"\n",
    "    Implements a three-layer ConvNet in Tensorflow:\n",
    "    CONV2D -> RELU -> MAXPOOL -> CONV2D -> RELU -> MAXPOOL -> FLATTEN -> FULLYCONNECTED\n",
    "    \n",
    "    Arguments:\n",
    "    X_train -- training set, of shape (None, 64, 64, 3)\n",
    "    Y_train -- test set, of shape (None, n_y = 6)\n",
    "    X_test -- training set, of shape (None, 64, 64, 3)\n",
    "    Y_test -- test set, of shape (None, n_y = 6)\n",
    "    learning_rate -- learning rate of the optimization\n",
    "    num_epochs -- number of epochs of the optimization loop\n",
    "    minibatch_size -- size of a minibatch\n",
    "    print_cost -- True to print the cost every 100 epochs\n",
    "    \n",
    "    Returns:\n",
    "    train_accuracy -- real number, accuracy on the train set (X_train)\n",
    "    test_accuracy -- real number, testing accuracy on the test set (X_test)\n",
    "    parameters -- parameters learnt by the model. They can then be used to predict.\n",
    "    \"\"\"\n",
    "    \n",
    "    ops.reset_default_graph()                         # to be able to rerun the model without overwriting tf variables\n",
    "    tf.set_random_seed(1)                             # to keep results consistent (tensorflow seed)\n",
    "    seed = 3                                          # to keep results consistent (numpy seed)\n",
    "    height,width, n_channels = *training_set_generator.target_size, 3\n",
    "    n_classes = N_CLASSES\n",
    "    costs = []                                        # To keep track of the cost\n",
    "    \n",
    "    last_train_accuracy = None\n",
    "    last_train_cost = None\n",
    "    \n",
    "    if step_size_train == None:\n",
    "        num_minibatches = len(training_set_generator)\n",
    "    else:\n",
    "        num_minibatches = step_size_train\n",
    "            \n",
    "    # Initialization\n",
    "    X, Y = create_placeholders(height, width, n_channels, n_classes)\n",
    "    parameters = initialize_parameters()\n",
    "    \n",
    "    # Forward propagation\n",
    "    Z5 = forward_propagation(X, parameters)\n",
    "    \n",
    "    # Cost function\n",
    "    cost = compute_cost(Z5, Y)\n",
    "    \n",
    "    # Backpropagation\n",
    "    optimizer = tf.train.AdamOptimizer().minimize(cost)    \n",
    "    init = tf.global_variables_initializer()\n",
    "     \n",
    "    # Start the session to compute the tensorflow graph\n",
    "    with tf.Session() as sess:\n",
    "        \n",
    "        # Run the initialization\n",
    "        sess.run(init)\n",
    "        \n",
    "        # Do the training loop\n",
    "        for epoch in range(num_epochs):\n",
    "\n",
    "            train_accuracy = 0.\n",
    "            minibatch_cost = 0.            \n",
    "\n",
    "            for i in range(num_minibatches):\n",
    "                #update_epoch_progress(epoch,i,num_minibatches, last_train_accuracy, last_train_cost)\n",
    "                train_X, train_Y = training_set_generator.next()\n",
    "                \n",
    "                _ , temp_cost = sess.run([optimizer, cost], feed_dict={X:train_X, Y:train_Y})\n",
    "                \n",
    "                minibatch_cost += temp_cost / num_minibatches\n",
    "                \n",
    "                # Calculate the correct predictions\n",
    "                predict_op = tf.argmax(Z5, 1)\n",
    "                correct_prediction = tf.equal(predict_op, tf.argmax(Y, 1))\n",
    "                accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
    "                \n",
    "                temp_train_accuracy = accuracy.eval({X: train_X, Y: train_Y})\n",
    "                \n",
    "                train_accuracy += temp_train_accuracy / num_minibatches\n",
    "                \n",
    "                print(temp_train_accuracy,train_accuracy)\n",
    "                \n",
    "            last_train_accuracy = train_accuracy\n",
    "            last_train_cost = minibatch_cost            \n",
    "            costs.append(minibatch_cost)\n",
    "            \n",
    "        return costs, parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5 0.0005\n",
      "0.34375 0.00084375\n",
      "0.625 0.00146875\n",
      "0.5 0.00196875\n",
      "0.625 0.00259375\n",
      "0.53125 0.003125\n",
      "0.5 0.003625\n",
      "0.46875 0.00409375\n",
      "0.46875 0.004562500000000001\n",
      "0.71875 0.00528125\n",
      "0.75 0.00603125\n",
      "0.53125 0.006562500000000001\n",
      "0.59375 0.00715625\n",
      "0.90625 0.0080625\n",
      "0.8125 0.008875000000000001\n",
      "0.53125 0.009406250000000001\n",
      "0.46875 0.009875000000000002\n",
      "0.71875 0.010593750000000002\n",
      "0.59375 0.011187500000000003\n",
      "0.5625 0.011750000000000003\n",
      "0.40625 0.012156250000000004\n",
      "0.5625 0.012718750000000004\n",
      "0.5625 0.013281250000000005\n",
      "0.46875 0.013750000000000005\n",
      "0.4375 0.014187500000000006\n",
      "0.84375 0.015031250000000006\n",
      "0.875 0.015906250000000007\n",
      "0.75 0.016656250000000008\n",
      "0.875 0.01753125000000001\n",
      "0.84375 0.01837500000000001\n",
      "1.0 0.01937500000000001\n",
      "0.78125 0.02015625000000001\n",
      "0.78125 0.02093750000000001\n",
      "0.84375 0.021781250000000012\n",
      "0.65625 0.022437500000000013\n",
      "0.6875 0.023125000000000014\n",
      "0.6875 0.023812500000000014\n",
      "0.5 0.024312500000000015\n",
      "0.65625 0.024968750000000015\n",
      "0.53125 0.025500000000000016\n",
      "0.4375 0.025937500000000016\n",
      "0.75 0.026687500000000017\n",
      "0.625 0.027312500000000017\n",
      "0.5 0.027812500000000018\n",
      "0.90625 0.02871875000000002\n",
      "0.71875 0.02943750000000002\n",
      "0.75 0.03018750000000002\n",
      "0.9375 0.03112500000000002\n",
      "0.78125 0.03190625000000002\n",
      "0.875 0.03278125000000002\n",
      "0.8125 0.03359375000000002\n",
      "0.84375 0.03443750000000002\n",
      "0.78125 0.035218750000000014\n",
      "0.8125 0.036031250000000015\n",
      "0.84375 0.03687500000000001\n",
      "0.875 0.03775000000000001\n",
      "0.90625 0.03865625000000001\n",
      "0.96875 0.03962500000000001\n",
      "0.96875 0.040593750000000005\n",
      "0.96875 0.0415625\n",
      "0.96875 0.04253125\n",
      "0.96875 0.0435\n",
      "0.8125 0.0443125\n",
      "0.9375 0.04525\n",
      "0.84375 0.046093749999999996\n",
      "0.71875 0.04681249999999999\n",
      "0.78125 0.04759374999999999\n",
      "0.65625 0.04824999999999999\n",
      "0.8125 0.04906249999999999\n",
      "0.90625 0.049968749999999985\n",
      "0.96875 0.05093749999999998\n",
      "1.0 0.051937499999999984\n",
      "1.0 0.052937499999999985\n",
      "0.96875 0.05390624999999998\n",
      "0.875 0.05478124999999998\n",
      "0.84375 0.05562499999999998\n",
      "0.96875 0.05659374999999998\n",
      "0.96875 0.057562499999999975\n",
      "1.0 0.058562499999999976\n",
      "0.9375 0.059499999999999977\n",
      "0.96875 0.060468749999999974\n",
      "0.875 0.061343749999999975\n",
      "0.90625 0.06224999999999997\n",
      "0.90625 0.06315624999999997\n",
      "0.96875 0.06412499999999997\n",
      "0.96875 0.06509374999999998\n",
      "0.9375 0.06603124999999997\n",
      "1.0 0.06703124999999997\n",
      "1.0 0.06803124999999997\n",
      "1.0 0.06903124999999997\n",
      "0.9375 0.06996874999999997\n",
      "0.9375 0.07090624999999996\n",
      "0.96875 0.07187499999999997\n",
      "0.96875 0.07284374999999997\n",
      "0.90625 0.07374999999999997\n",
      "0.9375 0.07468749999999996\n",
      "0.96875 0.07565624999999997\n",
      "1.0 0.07665624999999997\n",
      "0.96875 0.07762499999999997\n",
      "0.90625 0.07853124999999997\n",
      "1.0 0.07953124999999997\n",
      "1.0 0.08053124999999997\n",
      "0.96875 0.08149999999999998\n",
      "0.96875 0.08246874999999998\n",
      "0.9375 0.08340624999999997\n",
      "0.84375 0.08424999999999998\n",
      "0.96875 0.08521874999999998\n",
      "0.96875 0.08618749999999999\n",
      "1.0 0.08718749999999999\n",
      "1.0 0.08818749999999999\n",
      "0.96875 0.08915624999999999\n",
      "0.96875 0.090125\n",
      "0.96875 0.09109375\n",
      "0.96875 0.0920625\n",
      "1.0 0.0930625\n",
      "0.9375 0.094\n",
      "0.96875 0.09496875\n",
      "1.0 0.09596875\n",
      "0.9375 0.09690625\n",
      "0.96875 0.097875\n",
      "1.0 0.098875\n",
      "0.96875 0.09984375000000001\n",
      "1.0 0.10084375000000001\n",
      "0.9375 0.10178125\n",
      "0.9375 0.10271875\n",
      "1.0 0.10371875\n",
      "1.0 0.10471875\n",
      "1.0 0.10571875\n",
      "0.9375 0.10665625\n",
      "0.9375 0.10759374999999999\n",
      "0.96875 0.10856249999999999\n",
      "1.0 0.1095625\n",
      "1.0 0.1105625\n",
      "0.9375 0.11149999999999999\n",
      "0.9375 0.11243749999999998\n",
      "1.0 0.11343749999999998\n",
      "1.0 0.11443749999999998\n",
      "0.96875 0.11540624999999999\n",
      "1.0 0.11640624999999999\n",
      "0.96875 0.117375\n",
      "0.96875 0.11834375\n",
      "0.96875 0.1193125\n",
      "1.0 0.1203125\n",
      "0.96875 0.12128125000000001\n",
      "0.9375 0.12221875\n",
      "1.0 0.12321875\n",
      "0.96875 0.1241875\n",
      "0.96875 0.12515625\n",
      "1.0 0.12615625\n",
      "1.0 0.12715625\n",
      "0.96875 0.128125\n",
      "1.0 0.129125\n",
      "1.0 0.130125\n",
      "0.96875 0.13109374999999998\n",
      "1.0 0.13209374999999998\n",
      "0.9375 0.13303125\n",
      "1.0 0.13403125\n",
      "0.96875 0.13499999999999998\n",
      "0.96875 0.13596874999999997\n",
      "0.96875 0.13693749999999996\n",
      "1.0 0.13793749999999996\n",
      "1.0 0.13893749999999996\n",
      "1.0 0.13993749999999996\n",
      "0.96875 0.14090624999999996\n",
      "1.0 0.14190624999999996\n",
      "1.0 0.14290624999999996\n",
      "0.96875 0.14387499999999995\n",
      "1.0 0.14487499999999995\n",
      "0.9375 0.14581249999999996\n",
      "1.0 0.14681249999999996\n",
      "0.96875 0.14778124999999995\n",
      "1.0 0.14878124999999995\n",
      "1.0 0.14978124999999995\n",
      "1.0 0.15078124999999995\n",
      "0.96875 0.15174999999999994\n",
      "0.875 0.15262499999999993\n",
      "0.9375 0.15356249999999994\n",
      "0.90625 0.15446874999999993\n",
      "1.0 0.15546874999999993\n",
      "1.0 0.15646874999999993\n",
      "1.0 0.15746874999999994\n",
      "1.0 0.15846874999999994\n",
      "1.0 0.15946874999999994\n",
      "1.0 0.16046874999999994\n",
      "0.96875 0.16143749999999993\n",
      "1.0 0.16243749999999993\n",
      "1.0 0.16343749999999993\n",
      "1.0 0.16443749999999993\n",
      "1.0 0.16543749999999993\n",
      "0.96875 0.16640624999999992\n",
      "1.0 0.16740624999999992\n",
      "1.0 0.16840624999999992\n",
      "1.0 0.16940624999999992\n",
      "0.96875 0.17037499999999992\n",
      "0.9375 0.17131249999999992\n",
      "0.9375 0.17224999999999993\n",
      "0.9375 0.17318749999999994\n",
      "1.0 0.17418749999999994\n",
      "1.0 0.17518749999999994\n",
      "1.0 0.17618749999999994\n",
      "1.0 0.17718749999999994\n",
      "0.96875 0.17815624999999993\n",
      "0.96875 0.17912499999999992\n",
      "1.0 0.18012499999999992\n",
      "0.9375 0.18106249999999993\n",
      "0.96875 0.18203124999999992\n",
      "1.0 0.18303124999999992\n",
      "0.96875 0.1839999999999999\n",
      "0.96875 0.1849687499999999\n",
      "1.0 0.1859687499999999\n",
      "1.0 0.1869687499999999\n",
      "1.0 0.1879687499999999\n",
      "0.96875 0.1889374999999999\n",
      "1.0 0.1899374999999999\n",
      "0.96875 0.1909062499999999\n",
      "1.0 0.1919062499999999\n",
      "1.0 0.1929062499999999\n",
      "1.0 0.1939062499999999\n",
      "1.0 0.1949062499999999\n",
      "0.9375 0.1958437499999999\n",
      "1.0 0.1968437499999999\n",
      "1.0 0.1978437499999999\n",
      "0.90625 0.1987499999999999\n",
      "1.0 0.1997499999999999\n",
      "1.0 0.2007499999999999\n",
      "1.0 0.2017499999999999\n",
      "1.0 0.2027499999999999\n",
      "0.96875 0.2037187499999999\n",
      "0.96875 0.20468749999999988\n",
      "1.0 0.20568749999999988\n",
      "0.96875 0.20665624999999987\n",
      "0.9375 0.20759374999999988\n",
      "0.9375 0.2085312499999999\n",
      "1.0 0.2095312499999999\n",
      "1.0 0.2105312499999999\n",
      "1.0 0.2115312499999999\n",
      "1.0 0.2125312499999999\n",
      "0.96875 0.21349999999999988\n",
      "0.96875 0.21446874999999987\n",
      "0.96875 0.21543749999999987\n",
      "0.9375 0.21637499999999987\n",
      "1.0 0.21737499999999987\n",
      "0.96875 0.21834374999999986\n",
      "1.0 0.21934374999999987\n",
      "0.96875 0.22031249999999986\n",
      "1.0 0.22131249999999986\n",
      "1.0 0.22231249999999986\n",
      "0.96875 0.22328124999999985\n",
      "0.9375 0.22421874999999986\n",
      "1.0 0.22521874999999986\n",
      "1.0 0.22621874999999986\n",
      "0.96875 0.22718749999999985\n",
      "0.96875 0.22815624999999984\n",
      "0.9375 0.22909374999999985\n",
      "0.96875 0.23006249999999984\n",
      "0.96875 0.23103124999999983\n",
      "1.0 0.23203124999999983\n",
      "0.9375 0.23296874999999984\n",
      "0.96875 0.23393749999999983\n",
      "0.96875 0.23490624999999982\n",
      "0.9375 0.23584374999999982\n",
      "0.96875 0.23681249999999981\n",
      "0.96875 0.2377812499999998\n",
      "0.9375 0.2387187499999998\n",
      "0.96875 0.2396874999999998\n",
      "1.0 0.2406874999999998\n",
      "1.0 0.2416874999999998\n",
      "1.0 0.2426874999999998\n",
      "0.96875 0.2436562499999998\n",
      "1.0 0.2446562499999998\n",
      "0.96875 0.2456249999999998\n",
      "1.0 0.2466249999999998\n",
      "0.96875 0.24759374999999978\n",
      "0.96875 0.24856249999999977\n",
      "1.0 0.24956249999999977\n",
      "1.0 0.25056249999999974\n",
      "0.96875 0.25153124999999976\n",
      "1.0 0.25253124999999976\n",
      "0.9375 0.25346874999999974\n",
      "0.96875 0.25443749999999976\n",
      "0.9375 0.25537499999999974\n",
      "0.84375 0.2562187499999997\n",
      "1.0 0.2572187499999997\n",
      "0.96875 0.25818749999999974\n",
      "0.96875 0.25915624999999975\n",
      "0.9375 0.26009374999999973\n",
      "1.0 0.26109374999999974\n",
      "0.96875 0.26206249999999975\n",
      "1.0 0.26306249999999975\n",
      "0.96875 0.2640312499999998\n",
      "0.96875 0.2649999999999998\n",
      "0.96875 0.2659687499999998\n",
      "0.9375 0.2669062499999998\n",
      "0.9375 0.26784374999999977\n",
      "0.9375 0.26878124999999975\n",
      "0.90625 0.26968749999999975\n",
      "0.90625 0.27059374999999974\n",
      "0.96875 0.27156249999999976\n",
      "0.9375 0.27249999999999974\n",
      "1.0 0.27349999999999974\n",
      "1.0 0.27449999999999974\n",
      "1.0 0.27549999999999975\n",
      "0.96875 0.27646874999999976\n",
      "1.0 0.27746874999999976\n",
      "1.0 0.27846874999999977\n",
      "0.96875 0.2794374999999998\n",
      "0.96875 0.2804062499999998\n",
      "1.0 0.2814062499999998\n",
      "0.96875 0.2823749999999998\n",
      "0.96875 0.28334374999999984\n",
      "0.96875 0.28431249999999986\n",
      "1.0 0.28531249999999986\n",
      "0.96875 0.2862812499999999\n",
      "0.96875 0.2872499999999999\n",
      "1.0 0.2882499999999999\n",
      "0.96875 0.2892187499999999\n",
      "1.0 0.2902187499999999\n",
      "0.96875 0.29118749999999993\n",
      "0.96875 0.29215624999999995\n",
      "0.96875 0.29312499999999997\n",
      "1.0 0.29412499999999997\n",
      "1.0 0.29512499999999997\n",
      "1.0 0.29612499999999997\n",
      "0.90625 0.29703124999999997\n",
      "1.0 0.29803124999999997\n",
      "1.0 0.29903124999999997\n",
      "1.0 0.30003124999999997\n",
      "1.0 0.30103125\n",
      "1.0 0.30203125\n",
      "0.96875 0.303\n",
      "1.0 0.304\n",
      "1.0 0.305\n",
      "0.96875 0.30596875\n",
      "0.96875 0.30693750000000003\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.96875 0.30790625000000005\n",
      "0.96875 0.30887500000000007\n",
      "1.0 0.30987500000000007\n",
      "0.96875 0.3108437500000001\n",
      "0.9375 0.31178125000000007\n",
      "1.0 0.31278125000000007\n",
      "1.0 0.31378125000000007\n",
      "0.9375 0.31471875000000005\n",
      "0.96875 0.31568750000000007\n",
      "0.9375 0.31662500000000005\n",
      "0.9375 0.3175625\n",
      "1.0 0.3185625\n",
      "0.90625 0.31946875\n",
      "1.0 0.32046875\n",
      "1.0 0.32146875\n",
      "0.96875 0.32243750000000004\n",
      "1.0 0.32343750000000004\n",
      "0.96875 0.32440625000000006\n",
      "0.96875 0.3253750000000001\n",
      "1.0 0.3263750000000001\n",
      "1.0 0.3273750000000001\n",
      "0.96875 0.3283437500000001\n",
      "1.0 0.3293437500000001\n",
      "1.0 0.3303437500000001\n",
      "0.9375 0.3312812500000001\n",
      "1.0 0.3322812500000001\n",
      "0.96875 0.3332500000000001\n",
      "0.9375 0.3341875000000001\n",
      "0.96875 0.3351562500000001\n",
      "0.96875 0.3361250000000001\n",
      "1.0 0.3371250000000001\n",
      "0.96875 0.33809375000000014\n",
      "1.0 0.33909375000000014\n",
      "1.0 0.34009375000000014\n",
      "0.96875 0.34106250000000016\n",
      "1.0 0.34206250000000016\n",
      "1.0 0.34306250000000016\n",
      "0.96875 0.3440312500000002\n",
      "1.0 0.3450312500000002\n",
      "1.0 0.3460312500000002\n",
      "1.0 0.3470312500000002\n",
      "0.96875 0.3480000000000002\n",
      "1.0 0.3490000000000002\n",
      "0.96875 0.3499687500000002\n",
      "0.9375 0.3509062500000002\n",
      "0.96875 0.3518750000000002\n",
      "1.0 0.3528750000000002\n",
      "0.96875 0.35384375000000023\n",
      "1.0 0.35484375000000024\n",
      "1.0 0.35584375000000024\n",
      "1.0 0.35684375000000024\n",
      "1.0 0.35784375000000024\n",
      "0.96875 0.35881250000000026\n",
      "0.96875 0.3597812500000003\n",
      "1.0 0.3607812500000003\n",
      "0.96875 0.3617500000000003\n",
      "0.9375 0.3626875000000003\n",
      "1.0 0.3636875000000003\n",
      "0.90625 0.36459375000000027\n",
      "0.96875 0.3655625000000003\n",
      "0.96875 0.3665312500000003\n",
      "0.90625 0.3674375000000003\n",
      "0.96875 0.3684062500000003\n",
      "1.0 0.3694062500000003\n",
      "1.0 0.3704062500000003\n",
      "1.0 0.3714062500000003\n",
      "0.96875 0.37237500000000034\n",
      "1.0 0.37337500000000035\n",
      "0.96875 0.37434375000000036\n",
      "0.96875 0.3753125000000004\n",
      "0.96875 0.3762812500000004\n",
      "1.0 0.3772812500000004\n",
      "0.96875 0.3782500000000004\n",
      "0.96875 0.37921875000000044\n",
      "1.0 0.38021875000000044\n",
      "1.0 0.38121875000000044\n",
      "1.0 0.38221875000000044\n",
      "1.0 0.38321875000000044\n",
      "1.0 0.38421875000000044\n",
      "0.9375 0.3851562500000004\n",
      "0.96875 0.38612500000000044\n",
      "1.0 0.38712500000000044\n",
      "0.96875 0.38809375000000046\n",
      "1.0 0.38909375000000046\n",
      "1.0 0.39009375000000046\n",
      "0.96875 0.3910625000000005\n",
      "0.96875 0.3920312500000005\n",
      "1.0 0.3930312500000005\n",
      "0.9375 0.3939687500000005\n",
      "0.96875 0.3949375000000005\n",
      "0.96875 0.3959062500000005\n",
      "1.0 0.3969062500000005\n",
      "1.0 0.3979062500000005\n",
      "1.0 0.3989062500000005\n",
      "0.96875 0.39987500000000054\n",
      "0.96875 0.40084375000000055\n",
      "0.96875 0.40181250000000057\n",
      "1.0 0.4028125000000006\n",
      "0.96875 0.4037812500000006\n",
      "0.9375 0.40471875000000057\n",
      "0.9375 0.40565625000000055\n",
      "1.0 0.40665625000000055\n",
      "0.9375 0.40759375000000053\n",
      "0.96875 0.40856250000000055\n",
      "0.96875 0.40953125000000057\n",
      "0.96875 0.4105000000000006\n",
      "0.96875 0.4114687500000006\n",
      "1.0 0.4124687500000006\n",
      "1.0 0.4134687500000006\n",
      "0.96875 0.4144375000000006\n",
      "1.0 0.4154375000000006\n",
      "1.0 0.4164375000000006\n",
      "0.96875 0.41740625000000064\n",
      "1.0 0.41840625000000065\n",
      "0.90625 0.41931250000000064\n",
      "0.90625 0.42021875000000064\n",
      "0.96875 0.42118750000000066\n",
      "0.90625 0.42209375000000066\n",
      "0.96875 0.4230625000000007\n",
      "1.0 0.4240625000000007\n",
      "0.96875 0.4250312500000007\n",
      "0.875 0.4259062500000007\n",
      "0.96875 0.4268750000000007\n",
      "1.0 0.4278750000000007\n",
      "0.96875 0.42884375000000075\n",
      "0.96875 0.42981250000000076\n",
      "0.9375 0.43075000000000074\n",
      "0.9375 0.4316875000000007\n",
      "0.96875 0.43265625000000074\n",
      "0.96875 0.43362500000000076\n",
      "0.96875 0.4345937500000008\n",
      "0.96875 0.4355625000000008\n",
      "1.0 0.4365625000000008\n",
      "1.0 0.4375625000000008\n",
      "0.9375 0.4385000000000008\n",
      "1.0 0.4395000000000008\n",
      "1.0 0.4405000000000008\n",
      "1.0 0.4415000000000008\n",
      "1.0 0.4425000000000008\n",
      "0.96875 0.4434687500000008\n",
      "1.0 0.4444687500000008\n",
      "0.96875 0.4454375000000008\n",
      "0.96875 0.44640625000000084\n",
      "1.0 0.44740625000000084\n",
      "0.9375 0.4483437500000008\n",
      "1.0 0.4493437500000008\n",
      "0.9375 0.4502812500000008\n",
      "0.9375 0.4512187500000008\n",
      "1.0 0.4522187500000008\n",
      "0.9375 0.45315625000000076\n",
      "0.96875 0.4541250000000008\n",
      "0.96875 0.4550937500000008\n",
      "0.9375 0.4560312500000008\n",
      "0.96875 0.4570000000000008\n",
      "0.9375 0.4579375000000008\n",
      "0.9375 0.45887500000000075\n",
      "0.96875 0.4598437500000008\n",
      "1.0 0.4608437500000008\n",
      "0.96875 0.4618125000000008\n",
      "0.96875 0.4627812500000008\n",
      "0.96875 0.46375000000000083\n",
      "1.0 0.46475000000000083\n",
      "1.0 0.46575000000000083\n",
      "0.96875 0.46671875000000085\n",
      "0.9375 0.46765625000000083\n",
      "0.96875 0.46862500000000085\n",
      "1.0 0.46962500000000085\n",
      "0.96875 0.47059375000000087\n",
      "1.0 0.47159375000000087\n",
      "0.96875 0.4725625000000009\n",
      "1.0 0.4735625000000009\n",
      "0.96875 0.4745312500000009\n",
      "1.0 0.4755312500000009\n",
      "0.96875 0.4765000000000009\n",
      "1.0 0.4775000000000009\n",
      "0.96875 0.47846875000000094\n",
      "1.0 0.47946875000000094\n",
      "1.0 0.48046875000000094\n",
      "0.96875 0.48143750000000096\n",
      "0.96875 0.482406250000001\n",
      "0.96875 0.483375000000001\n",
      "1.0 0.484375000000001\n",
      "0.96875 0.485343750000001\n",
      "0.96875 0.48631250000000104\n",
      "0.9375 0.487250000000001\n",
      "1.0 0.488250000000001\n",
      "1.0 0.489250000000001\n",
      "1.0 0.490250000000001\n",
      "0.96875 0.49121875000000104\n",
      "1.0 0.49221875000000104\n",
      "0.96875 0.49318750000000106\n",
      "0.96875 0.4941562500000011\n",
      "0.9375 0.49509375000000105\n",
      "1.0 0.49609375000000105\n",
      "1.0 0.49709375000000106\n",
      "1.0 0.49809375000000106\n",
      "1.0 0.49909375000000106\n",
      "0.9375 0.500031250000001\n",
      "1.0 0.501031250000001\n",
      "0.96875 0.502000000000001\n",
      "1.0 0.503000000000001\n",
      "0.90625 0.503906250000001\n",
      "0.96875 0.504875000000001\n",
      "0.96875 0.5058437500000009\n",
      "1.0 0.5068437500000009\n",
      "0.96875 0.5078125000000009\n",
      "0.9375 0.5087500000000009\n",
      "1.0 0.5097500000000009\n",
      "1.0 0.5107500000000009\n",
      "0.96875 0.5117187500000009\n",
      "0.96875 0.5126875000000009\n",
      "0.96875 0.5136562500000008\n",
      "0.9375 0.5145937500000008\n",
      "1.0 0.5155937500000008\n",
      "1.0 0.5165937500000009\n",
      "1.0 0.5175937500000009\n",
      "0.96875 0.5185625000000008\n",
      "0.96875 0.5195312500000008\n",
      "0.96875 0.5205000000000007\n",
      "1.0 0.5215000000000007\n",
      "0.9375 0.5224375000000008\n",
      "0.96875 0.5234062500000007\n",
      "1.0 0.5244062500000007\n",
      "1.0 0.5254062500000007\n",
      "1.0 0.5264062500000007\n",
      "1.0 0.5274062500000007\n",
      "0.96875 0.5283750000000007\n",
      "1.0 0.5293750000000007\n",
      "1.0 0.5303750000000007\n",
      "1.0 0.5313750000000007\n",
      "1.0 0.5323750000000007\n",
      "1.0 0.5333750000000007\n",
      "0.96875 0.5343437500000007\n",
      "0.96875 0.5353125000000006\n",
      "0.9375 0.5362500000000007\n",
      "1.0 0.5372500000000007\n",
      "0.9375 0.5381875000000007\n",
      "0.9375 0.5391250000000007\n",
      "1.0 0.5401250000000007\n",
      "0.96875 0.5410937500000007\n",
      "0.90625 0.5420000000000007\n",
      "1.0 0.5430000000000007\n",
      "1.0 0.5440000000000007\n",
      "1.0 0.5450000000000007\n",
      "0.96875 0.5459687500000007\n",
      "1.0 0.5469687500000007\n",
      "1.0 0.5479687500000007\n",
      "1.0 0.5489687500000007\n",
      "1.0 0.5499687500000007\n",
      "0.9375 0.5509062500000007\n",
      "0.96875 0.5518750000000007\n",
      "0.9375 0.5528125000000007\n",
      "0.9375 0.5537500000000007\n",
      "0.9375 0.5546875000000008\n",
      "1.0 0.5556875000000008\n",
      "1.0 0.5566875000000008\n",
      "1.0 0.5576875000000008\n",
      "1.0 0.5586875000000008\n",
      "1.0 0.5596875000000008\n",
      "1.0 0.5606875000000008\n",
      "0.96875 0.5616562500000007\n",
      "1.0 0.5626562500000007\n",
      "0.96875 0.5636250000000007\n",
      "0.9375 0.5645625000000007\n",
      "0.9375 0.5655000000000008\n",
      "0.96875 0.5664687500000007\n",
      "1.0 0.5674687500000007\n",
      "1.0 0.5684687500000007\n",
      "0.90625 0.5693750000000007\n",
      "0.96875 0.5703437500000007\n",
      "1.0 0.5713437500000007\n",
      "1.0 0.5723437500000007\n",
      "0.96875 0.5733125000000007\n",
      "1.0 0.5743125000000007\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-38-96367a0cd669>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m             \u001b[0mnum_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m             \u001b[0mstep_size_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstep_size_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m             \u001b[0mminibatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m )\n",
      "\u001b[0;32m<ipython-input-37-8b715ad53703>\u001b[0m in \u001b[0;36mmodel\u001b[0;34m(training_set_generator, test_set_generator, learning_rate, num_epochs, step_size_train, minibatch_size)\u001b[0m\n\u001b[1;32m     66\u001b[0m                 \u001b[0mtrain_X\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_Y\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtraining_set_generator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m                 \u001b[0m_\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mtemp_cost\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcost\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mtrain_X\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mtrain_Y\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m                 \u001b[0mminibatch_cost\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mtemp_cost\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mnum_minibatches\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    927\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    928\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 929\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    930\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    931\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1150\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1151\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1152\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1153\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1154\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1326\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1327\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1328\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1329\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1330\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1332\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1333\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1334\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1335\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1336\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1317\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1318\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1319\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1320\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1321\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1405\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1406\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1407\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1408\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1409\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "costs, parameters = model( \n",
    "            training_set_generator, \n",
    "            test_set_generator, \n",
    "            num_epochs=5, \n",
    "            step_size_train = step_size_train,\n",
    "            minibatch_size=BATCH_SIZE\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_report(costs):\n",
    "    # plot the cost\n",
    "    plt.plot(np.squeeze(costs))\n",
    "    plt.ylabel('cost')\n",
    "    plt.xlabel('iterations (per tens)')\n",
    "    plt.title(\"Learning rate =\" + str(learning_rate))\n",
    "    plt.show()\n",
    "\n",
    "    # Calculate the correct predictions\n",
    "    predict_op = tf.argmax(Z5, 1)\n",
    "    correct_prediction = tf.equal(predict_op, tf.argmax(Y, 1))\n",
    "        \n",
    "    # Calculate accuracy on the test set\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
    "    print(accuracy)\n",
    "    train_accuracy = accuracy.eval({X: X_train, Y: Y_train})\n",
    "    test_accuracy = accuracy.eval({X: X_test, Y: Y_test})\n",
    "    print(\"Train Accuracy:\", train_accuracy)\n",
    "    print(\"Test Accuracy:\", test_accuracy)\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_batches = 1000\n",
    "\n",
    "for i in range(total_batches):\n",
    "    time.sleep(0.1) #Replace this with a real computation\n",
    "    update_epoch_progress(0, i,total_batches)\n",
    "\n",
    "update_progress(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "number_of_elements = 1000\n",
    "\n",
    "for i in range(number_of_elements):\n",
    "    time.sleep(0.1) #Replace this with a real computation\n",
    "    update_progress(i / number_of_elements)\n",
    "\n",
    "update_progress(1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}