keras_team.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    },
    "colab": {
      "name": "keras-team.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mashyko/deep-learning/blob/master/keras_team.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tkRgYPM--fEY",
        "colab_type": "text"
      },
      "source": [
        "https://github.com/keras-team/keras/tree/master/examples\n",
        "MNISTdata classification "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tjx_XCbnCJKf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "b3a1b509-1472-43b6-a85f-56ce67d119ef"
      },
      "source": [
        "'''Trains a simple deep NN on the MNIST dataset.\n",
        "Gets to 98.40% test accuracy after 20 epochs\n",
        "(there is *a lot* of margin for parameter tuning).\n",
        "2 seconds per epoch on a K520 GPU.\n",
        "'''\n",
        "\n",
        "from __future__ import print_function\n",
        "\n",
        "import keras\n",
        "from keras.datasets import mnist\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Dropout\n",
        "from keras.optimizers import RMSprop\n",
        "\n",
        "batch_size = 128\n",
        "num_classes = 10\n",
        "epochs = 20\n",
        "\n",
        "# the data, split between train and test sets\n",
        "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
        "\n",
        "x_train = x_train.reshape(60000, 784)\n",
        "x_test = x_test.reshape(10000, 784)\n",
        "x_train = x_train.astype('float32')\n",
        "x_test = x_test.astype('float32')\n",
        "x_train /= 255\n",
        "x_test /= 255\n",
        "print(x_train.shape[0], 'train samples')\n",
        "print(x_test.shape[0], 'test samples')\n",
        "\n",
        "# convert class vectors to binary class matrices\n",
        "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
        "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
        "\n",
        "model = Sequential()\n",
        "model.add(Dense(512, activation='relu', input_shape=(784,)))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Dense(512, activation='relu'))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Dense(num_classes, activation='softmax'))\n",
        "\n",
        "model.summary()\n",
        "\n",
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer=RMSprop(),\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "history = model.fit(x_train, y_train,\n",
        "                    batch_size=batch_size,\n",
        "                    epochs=epochs,\n",
        "                    verbose=1,\n",
        "                    validation_data=(x_test, y_test))\n",
        "score = model.evaluate(x_test, y_test, verbose=0)\n",
        "print('Test loss:', score[0])\n",
        "print('Test accuracy:', score[1])"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 0s 0us/step\n",
            "60000 train samples\n",
            "10000 test samples\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:66: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:541: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4432: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:148: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3733: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "dense_1 (Dense)              (None, 512)               401920    \n",
            "_________________________________________________________________\n",
            "dropout_1 (Dropout)          (None, 512)               0         \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 512)               262656    \n",
            "_________________________________________________________________\n",
            "dropout_2 (Dropout)          (None, 512)               0         \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 10)                5130      \n",
            "=================================================================\n",
            "Total params: 669,706\n",
            "Trainable params: 669,706\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:793: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3576: The name tf.log is deprecated. Please use tf.math.log instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1033: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1020: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3005: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
            "\n",
            "Train on 60000 samples, validate on 10000 samples\n",
            "Epoch 1/20\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:190: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:197: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:207: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:216: The name tf.is_variable_initialized is deprecated. Please use tf.compat.v1.is_variable_initialized instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:223: The name tf.variables_initializer is deprecated. Please use tf.compat.v1.variables_initializer instead.\n",
            "\n",
            "60000/60000 [==============================] - 6s 107us/step - loss: 0.2451 - acc: 0.9238 - val_loss: 0.1132 - val_acc: 0.9647\n",
            "Epoch 2/20\n",
            "60000/60000 [==============================] - 2s 33us/step - loss: 0.1019 - acc: 0.9687 - val_loss: 0.0800 - val_acc: 0.9747\n",
            "Epoch 3/20\n",
            "60000/60000 [==============================] - 2s 35us/step - loss: 0.0752 - acc: 0.9774 - val_loss: 0.0865 - val_acc: 0.9767\n",
            "Epoch 4/20\n",
            "60000/60000 [==============================] - 2s 35us/step - loss: 0.0605 - acc: 0.9817 - val_loss: 0.0811 - val_acc: 0.9781\n",
            "Epoch 5/20\n",
            "60000/60000 [==============================] - 2s 34us/step - loss: 0.0498 - acc: 0.9851 - val_loss: 0.0828 - val_acc: 0.9789\n",
            "Epoch 6/20\n",
            "60000/60000 [==============================] - 2s 33us/step - loss: 0.0420 - acc: 0.9872 - val_loss: 0.0775 - val_acc: 0.9798\n",
            "Epoch 7/20\n",
            "60000/60000 [==============================] - 2s 32us/step - loss: 0.0382 - acc: 0.9890 - val_loss: 0.0837 - val_acc: 0.9817\n",
            "Epoch 8/20\n",
            "60000/60000 [==============================] - 2s 35us/step - loss: 0.0359 - acc: 0.9897 - val_loss: 0.0810 - val_acc: 0.9816\n",
            "Epoch 9/20\n",
            "60000/60000 [==============================] - 2s 33us/step - loss: 0.0319 - acc: 0.9908 - val_loss: 0.0934 - val_acc: 0.9816\n",
            "Epoch 10/20\n",
            "60000/60000 [==============================] - 2s 34us/step - loss: 0.0290 - acc: 0.9918 - val_loss: 0.0885 - val_acc: 0.9822\n",
            "Epoch 11/20\n",
            "60000/60000 [==============================] - 2s 33us/step - loss: 0.0269 - acc: 0.9924 - val_loss: 0.0858 - val_acc: 0.9835\n",
            "Epoch 12/20\n",
            "60000/60000 [==============================] - 2s 33us/step - loss: 0.0251 - acc: 0.9925 - val_loss: 0.1195 - val_acc: 0.9798\n",
            "Epoch 13/20\n",
            "60000/60000 [==============================] - 2s 34us/step - loss: 0.0241 - acc: 0.9934 - val_loss: 0.0906 - val_acc: 0.9825\n",
            "Epoch 14/20\n",
            "60000/60000 [==============================] - 2s 35us/step - loss: 0.0225 - acc: 0.9940 - val_loss: 0.1014 - val_acc: 0.9821\n",
            "Epoch 15/20\n",
            "60000/60000 [==============================] - 2s 33us/step - loss: 0.0226 - acc: 0.9941 - val_loss: 0.0989 - val_acc: 0.9840\n",
            "Epoch 16/20\n",
            "60000/60000 [==============================] - 2s 34us/step - loss: 0.0200 - acc: 0.9946 - val_loss: 0.1132 - val_acc: 0.9823\n",
            "Epoch 17/20\n",
            "60000/60000 [==============================] - 2s 32us/step - loss: 0.0216 - acc: 0.9945 - val_loss: 0.1112 - val_acc: 0.9829\n",
            "Epoch 18/20\n",
            "60000/60000 [==============================] - 2s 33us/step - loss: 0.0190 - acc: 0.9950 - val_loss: 0.0947 - val_acc: 0.9837\n",
            "Epoch 19/20\n",
            "60000/60000 [==============================] - 2s 35us/step - loss: 0.0167 - acc: 0.9954 - val_loss: 0.1299 - val_acc: 0.9811\n",
            "Epoch 20/20\n",
            "60000/60000 [==============================] - 2s 33us/step - loss: 0.0196 - acc: 0.9950 - val_loss: 0.1048 - val_acc: 0.9838\n",
            "Test loss: 0.10476716020676934\n",
            "Test accuracy: 0.9838\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "KjAbPpM9-fEf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 581
        },
        "outputId": "33dfbc54-9d5a-4a99-ebeb-051bbdba3e21"
      },
      "source": [
        "'''Trains a simple convnet on the MNIST dataset.\n",
        "Gets to 99.25% test accuracy after 12 epochs\n",
        "(there is still a lot of margin for parameter tuning).\n",
        "16 seconds per epoch on a GRID K520 GPU.\n",
        "'''\n",
        "\n",
        "# mnist_cnn.ipynb\n",
        "\n",
        "from __future__ import print_function\n",
        "import keras\n",
        "from keras.datasets import mnist\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Dropout, Flatten\n",
        "from keras.layers import Conv2D, MaxPooling2D\n",
        "from keras import backend as K\n",
        "\n",
        "batch_size = 128\n",
        "num_classes = 10\n",
        "epochs = 12\n",
        "\n",
        "# input image dimensions\n",
        "img_rows, img_cols = 28, 28\n",
        "\n",
        "# the data, split between train and test sets\n",
        "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
        "\n",
        "if K.image_data_format() == 'channels_first':\n",
        "    x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)\n",
        "    x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)\n",
        "    input_shape = (1, img_rows, img_cols)\n",
        "else:\n",
        "    x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
        "    x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n",
        "    input_shape = (img_rows, img_cols, 1)\n",
        "\n",
        "x_train = x_train.astype('float32')\n",
        "x_test = x_test.astype('float32')\n",
        "x_train /= 255\n",
        "x_test /= 255\n",
        "print('x_train shape:', x_train.shape)\n",
        "print(x_train.shape[0], 'train samples')\n",
        "print(x_test.shape[0], 'test samples')\n",
        "\n",
        "# convert class vectors to binary class matrices\n",
        "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
        "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
        "\n",
        "model = Sequential()\n",
        "model.add(Conv2D(32, kernel_size=(3, 3),\n",
        "                 activation='relu',\n",
        "                 input_shape=input_shape))\n",
        "model.add(Conv2D(64, (3, 3), activation='relu'))\n",
        "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
        "model.add(Dropout(0.25))\n",
        "model.add(Flatten())\n",
        "model.add(Dense(128, activation='relu'))\n",
        "model.add(Dropout(0.5))\n",
        "model.add(Dense(num_classes, activation='softmax'))\n",
        "\n",
        "model.compile(loss=keras.losses.categorical_crossentropy,\n",
        "              optimizer=keras.optimizers.Adadelta(),\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "model.fit(x_train, y_train,\n",
        "          batch_size=batch_size,\n",
        "          epochs=epochs,\n",
        "          verbose=1,\n",
        "          validation_data=(x_test, y_test))\n",
        "score = model.evaluate(x_test, y_test, verbose=0)\n",
        "print('Test loss:', score[0])\n",
        "print('Test accuracy:', score[1])"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "x_train shape: (60000, 28, 28, 1)\n",
            "60000 train samples\n",
            "10000 test samples\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4267: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
            "\n",
            "Train on 60000 samples, validate on 10000 samples\n",
            "Epoch 1/12\n",
            "60000/60000 [==============================] - 9s 150us/step - loss: 0.2609 - acc: 0.9188 - val_loss: 0.0570 - val_acc: 0.9819\n",
            "Epoch 2/12\n",
            "60000/60000 [==============================] - 6s 102us/step - loss: 0.0863 - acc: 0.9739 - val_loss: 0.0392 - val_acc: 0.9869\n",
            "Epoch 3/12\n",
            "60000/60000 [==============================] - 6s 102us/step - loss: 0.0639 - acc: 0.9808 - val_loss: 0.0334 - val_acc: 0.9890\n",
            "Epoch 4/12\n",
            "60000/60000 [==============================] - 6s 102us/step - loss: 0.0535 - acc: 0.9846 - val_loss: 0.0326 - val_acc: 0.9893\n",
            "Epoch 5/12\n",
            "60000/60000 [==============================] - 6s 105us/step - loss: 0.0443 - acc: 0.9870 - val_loss: 0.0281 - val_acc: 0.9902\n",
            "Epoch 6/12\n",
            "60000/60000 [==============================] - 6s 102us/step - loss: 0.0408 - acc: 0.9874 - val_loss: 0.0313 - val_acc: 0.9897\n",
            "Epoch 7/12\n",
            "60000/60000 [==============================] - 6s 101us/step - loss: 0.0363 - acc: 0.9885 - val_loss: 0.0247 - val_acc: 0.9924\n",
            "Epoch 8/12\n",
            "60000/60000 [==============================] - 6s 101us/step - loss: 0.0330 - acc: 0.9901 - val_loss: 0.0317 - val_acc: 0.9902\n",
            "Epoch 9/12\n",
            "60000/60000 [==============================] - 6s 102us/step - loss: 0.0323 - acc: 0.9899 - val_loss: 0.0252 - val_acc: 0.9923\n",
            "Epoch 10/12\n",
            "60000/60000 [==============================] - 6s 102us/step - loss: 0.0293 - acc: 0.9912 - val_loss: 0.0293 - val_acc: 0.9915\n",
            "Epoch 11/12\n",
            "60000/60000 [==============================] - 6s 102us/step - loss: 0.0251 - acc: 0.9921 - val_loss: 0.0321 - val_acc: 0.9903\n",
            "Epoch 12/12\n",
            "60000/60000 [==============================] - 6s 102us/step - loss: 0.0256 - acc: 0.9919 - val_loss: 0.0290 - val_acc: 0.9922\n",
            "Test loss: 0.02902445701093152\n",
            "Test accuracy: 0.9922\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kkCn2SVn-fEi",
        "colab_type": "text"
      },
      "source": [
        "mnist_hierarchical_rnn.py"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7WVQESyB-fEj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "cca3ce17-e7fc-4ccf-89e7-e46c45c24d21"
      },
      "source": [
        "\"\"\"Example of using Hierarchical RNN (HRNN) to classify MNIST digits.\n",
        "HRNNs can learn across multiple levels\n",
        "of temporal hierarchy over a complex sequence.\n",
        "Usually, the first recurrent layer of an HRNN\n",
        "encodes a sentence (e.g. of word vectors)\n",
        "into a  sentence vector.\n",
        "The second recurrent layer then encodes a sequence of\n",
        "such vectors (encoded by the first layer) into a document vector.\n",
        "This document vector is considered to preserve both\n",
        "the word-level and sentence-level structure of the context.\n",
        "# References\n",
        "- [A Hierarchical Neural Autoencoder for Paragraphs and Documents](https://arxiv.org/abs/1506.01057)\n",
        "    Encodes paragraphs and documents with HRNN.\n",
        "    Results have shown that HRNN outperforms standard\n",
        "    RNNs and may play some role in more sophisticated generation tasks like\n",
        "    summarization or question answering.\n",
        "- [Hierarchical recurrent neural network for skeleton based action recognition](http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7298714)\n",
        "    Achieved state-of-the-art results on\n",
        "    skeleton based action recognition with 3 levels\n",
        "    of bidirectional HRNN combined with fully connected layers.\n",
        "In the below MNIST example the first LSTM layer first encodes every\n",
        "column of pixels of shape (28, 1) to a column vector of shape (128,).\n",
        "The second LSTM layer encodes then these 28 column vectors of shape (28, 128)\n",
        "to a image vector representing the whole image.\n",
        "A final Dense layer is added for prediction.\n",
        "After 5 epochs: train acc: 0.9858, val acc: 0.9864\n",
        "\"\"\"\n",
        "from __future__ import print_function\n",
        "\n",
        "import keras\n",
        "from keras.datasets import mnist\n",
        "from keras.models import Model\n",
        "from keras.layers import Input, Dense, TimeDistributed\n",
        "from keras.layers import LSTM\n",
        "\n",
        "# Training parameters.\n",
        "batch_size = 32\n",
        "num_classes = 10\n",
        "epochs = 5\n",
        "\n",
        "# Embedding dimensions.\n",
        "row_hidden = 128\n",
        "col_hidden = 128\n",
        "\n",
        "# The data, split between train and test sets.\n",
        "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
        "\n",
        "# Reshapes data to 4D for Hierarchical RNN.\n",
        "x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)\n",
        "x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)\n",
        "x_train = x_train.astype('float32')\n",
        "x_test = x_test.astype('float32')\n",
        "x_train /= 255\n",
        "x_test /= 255\n",
        "print('x_train shape:', x_train.shape)\n",
        "print(x_train.shape[0], 'train samples')\n",
        "print(x_test.shape[0], 'test samples')\n",
        "\n",
        "# Converts class vectors to binary class matrices.\n",
        "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
        "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
        "\n",
        "row, col, pixel = x_train.shape[1:]\n",
        "\n",
        "# 4D input.\n",
        "x = Input(shape=(row, col, pixel))\n",
        "\n",
        "# Encodes a row of pixels using TimeDistributed Wrapper.\n",
        "encoded_rows = TimeDistributed(LSTM(row_hidden))(x)\n",
        "\n",
        "# Encodes columns of encoded rows.\n",
        "encoded_columns = LSTM(col_hidden)(encoded_rows)\n",
        "\n",
        "# Final predictions and model.\n",
        "prediction = Dense(num_classes, activation='softmax')(encoded_columns)\n",
        "model = Model(x, prediction)\n",
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer='rmsprop',\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "# Training.\n",
        "model.fit(x_train, y_train,\n",
        "          batch_size=batch_size,\n",
        "          epochs=epochs,\n",
        "          verbose=1,\n",
        "          validation_data=(x_test, y_test))\n",
        "\n",
        "# Evaluation.\n",
        "scores = model.evaluate(x_test, y_test, verbose=0)\n",
        "print('Test loss:', scores[0])\n",
        "print('Test accuracy:', scores[1])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "x_train shape: (60000, 28, 28, 1)\n",
            "60000 train samples\n",
            "10000 test samples\n",
            "Train on 60000 samples, validate on 10000 samples\n",
            "Epoch 1/5\n",
            "59968/60000 [============================>.] - ETA: 0s - loss: 0.4620 - acc: 0.8446"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qBYoG984-fEm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}