test3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras import utils\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "num_classes = 10\n",
    "img_rows, img_cols = 28, 28"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist = keras.datasets.mnist\n",
    "(x_train, y_train),(x_test, y_test) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
    "x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "y_train = utils.to_categorical(y_train, num_classes)\n",
    "y_test = utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def squash(x, axis=-1):\n",
    "    squared_norm = keras.backend.sum(keras.backend.square(x), axis, keepdims=True) + keras.backend.epsilon()\n",
    "    scale = keras.backend.sqrt(squared_norm)/(1+squared_norm)\n",
    "    return scale*x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class digitCaps(keras.layers.Layer):\n",
    "    def __init__(self,in_type=1152, in_shape=8, out_type=10, out_shape=16, routings=3, activation='squash',**kwargs):\n",
    "        self.in_type = in_type\n",
    "        self.in_shape = in_shape       \n",
    "        self.out_type = out_type\n",
    "        self.out_shape = out_shape\n",
    "        self.routings = routings\n",
    "        if activation == 'squash':\n",
    "            self.activation = squash\n",
    "        else:\n",
    "            self.activation = keras.layers.Activation(activation)\n",
    "        super(digitCaps,self).__init__(**kwargs)\n",
    "        \n",
    "    def build(self,input_shape):\n",
    "        self.kernel1 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel2 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel3 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel4 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel5 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel6 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel7 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel8 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel9 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel10 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        super(digitCaps,self).build(input_shape)\n",
    "\n",
    "    def call(self,inputs):\n",
    "        type1 = tf.squeeze(tf.matmul(inputs, self.kernel1),axis=2)\n",
    "        type2 = tf.squeeze(tf.matmul(inputs, self.kernel2),axis=2)\n",
    "        type3 = tf.squeeze(tf.matmul(inputs, self.kernel3),axis=2)\n",
    "        type4 = tf.squeeze(tf.matmul(inputs, self.kernel4),axis=2)\n",
    "        type5 = tf.squeeze(tf.matmul(inputs, self.kernel5),axis=2)\n",
    "        type6 = tf.squeeze(tf.matmul(inputs, self.kernel6),axis=2)\n",
    "        type7 = tf.squeeze(tf.matmul(inputs, self.kernel7),axis=2)\n",
    "        type8 = tf.squeeze(tf.matmul(inputs, self.kernel8),axis=2)\n",
    "        type9 = tf.squeeze(tf.matmul(inputs, self.kernel9),axis=2)\n",
    "        type10 = tf.squeeze(tf.matmul(inputs, self.kernel10),axis=2)\n",
    "        result = tf.stack([type1,type2,type3,type4,type5,type6,type7,type8,type9,type10],1)\n",
    "        B = tf.zeros_like(tf.transpose(tf.transpose(result)[1]))\n",
    "        for i in range(self.routings):\n",
    "            C = tf.nn.softmax(B,1)\n",
    "            O = tf.einsum('ijk,ijkl->ijl', C, result)\n",
    "            V = squash(O,-1)\n",
    "            B = B + tf.einsum('ijk,ijlk->ijl', V, result)\n",
    "        return V"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class selfCaps(keras.layers.Layer):\n",
    "    def __init__(self,in_type=1152, in_shape=8, out_type=10, out_shape=16, activation='squash',**kwargs):\n",
    "        self.in_type = in_type\n",
    "        self.in_shape = in_shape       \n",
    "        self.out_type = out_type\n",
    "        self.out_shape = out_shape\n",
    "        if activation == 'squash':\n",
    "            self.activation = squash\n",
    "        else:\n",
    "            self.activation = keras.layers.Activation(activation)\n",
    "        super(selfCaps,self).__init__(**kwargs)\n",
    "        \n",
    "    def build(self,input_shape):\n",
    "        self.kernel1 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel2 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel3 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel4 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel5 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel6 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel7 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel8 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel9 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.kernel10 = self.add_weight(shape=(self.in_type,self.in_shape,self.out_shape),initializer=\"uniform\", trainable=True)\n",
    "        self.O = self.add_weight(shape=(self.out_type,self.out_shape,self.out_shape),initializer=\"ones\", trainable=True)\n",
    "        super(selfCaps,self).build(input_shape)\n",
    "\n",
    "    def call(self,inputs):\n",
    "        type1 = tf.squeeze(tf.matmul(inputs, self.kernel1),axis=2)\n",
    "        type2 = tf.squeeze(tf.matmul(inputs, self.kernel2),axis=2)\n",
    "        type3 = tf.squeeze(tf.matmul(inputs, self.kernel3),axis=2)\n",
    "        type4 = tf.squeeze(tf.matmul(inputs, self.kernel4),axis=2)\n",
    "        type5 = tf.squeeze(tf.matmul(inputs, self.kernel5),axis=2)\n",
    "        type6 = tf.squeeze(tf.matmul(inputs, self.kernel6),axis=2)\n",
    "        type7 = tf.squeeze(tf.matmul(inputs, self.kernel7),axis=2)\n",
    "        type8 = tf.squeeze(tf.matmul(inputs, self.kernel8),axis=2)\n",
    "        type9 = tf.squeeze(tf.matmul(inputs, self.kernel9),axis=2)\n",
    "        type10 = tf.squeeze(tf.matmul(inputs, self.kernel10),axis=2)\n",
    "        result = tf.stack([type1,type2,type3,type4,type5,type6,type7,type8,type9,type10],1)\n",
    "        V = tf.reduce_sum(result, axis=2, keepdims=True)\n",
    "        V = squash(V,-1)\n",
    "        V = tf.squeeze(tf.matmul(V, self.O),axis=2)\n",
    "        return V"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class mask(keras.layers.Layer):\n",
    "    def __init__(self, **kwargs):\n",
    "        super(mask,self).__init__(**kwargs)\n",
    "        \n",
    "    def build(self,input_shape):\n",
    "        super(mask,self).build(input_shape)\n",
    "\n",
    "    def call(self,inputs):\n",
    "        info = keras.backend.sqrt(keras.backend.sum(keras.backend.square(inputs), 2))\n",
    "        inf = tf.argmax(info,axis=1)[1]\n",
    "        out = inputs[:,inf,:]\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = keras.layers.Lambda(lambda x: keras.backend.sqrt(keras.backend.sum(keras.backend.square(x), 2)), output_shape=(10,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 28, 28, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d (Conv2D)                 (None, 24, 24, 256)  6656        input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 18, 18, 256)  3211520     conv2d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 12, 12, 256)  3211520     conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 6, 6, 256)    3211520     conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "up_sampling2d (UpSampling2D)    (None, 12, 12, 256)  0           conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add (Add)                       (None, 12, 12, 256)  0           up_sampling2d[0][0]              \n",
      "                                                                 conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "up_sampling2d_1 (UpSampling2D)  (None, 24, 24, 256)  0           add[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "reshape (Reshape)               (None, 1152, 1, 8)   0           conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "reshape_1 (Reshape)             (None, 1152, 1, 32)  0           add[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 24, 24, 256)  0           up_sampling2d_1[0][0]            \n",
      "                                                                 conv2d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "digit_caps (digitCaps)          (None, 10, 16)       1474560     reshape[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "digit_caps_1 (digitCaps)        (None, 10, 16)       5898240     reshape_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "reshape_2 (Reshape)             (None, 1152, 1, 128) 0           add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_add_2 (TensorFlowOp [(None, 10, 16)]     0           digit_caps[0][0]                 \n",
      "                                                                 digit_caps_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "digit_caps_2 (digitCaps)        (None, 10, 16)       23592960    reshape_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_add_3 (TensorFlowOp [(None, 10, 16)]     0           tf_op_layer_add_2[0][0]          \n",
      "                                                                 digit_caps_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_stack (TensorFlowOp [(3, None, 10, 16)]  0           digit_caps[0][0]                 \n",
      "                                                                 digit_caps_1[0][0]               \n",
      "                                                                 digit_caps_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "reshape_3 (Reshape)             (None, 10, 1, 16)    0           tf_op_layer_add_3[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_transpose (TensorFl [(None, 10, 3, 16)]  0           tf_op_layer_stack[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "attention (Attention)           (None, 10, 1, 16)    0           reshape_3[0][0]                  \n",
      "                                                                 tf_op_layer_transpose[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "reshape_4 (Reshape)             (None, 10, 16)       0           attention[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, 10)           0           reshape_4[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 40,606,976\n",
      "Trainable params: 40,606,976\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "inputs = tf.keras.layers.Input(shape=(28,28,1))\n",
    "C1 = tf.keras.layers.Conv2D(filters=256, kernel_size=(5,5), strides=(1,1), activation=\"relu\")(inputs)#24\n",
    "C2 = tf.keras.layers.Conv2D(filters=256, kernel_size=(7,7), strides=(1,1), activation=\"relu\")(C1)#18\n",
    "C3 = tf.keras.layers.Conv2D(filters=256, kernel_size=(7,7), strides=(1,1), activation=\"relu\")(C2)#12\n",
    "C4 = tf.keras.layers.Conv2D(filters=256, kernel_size=(7,7), strides=(1,1), activation=\"relu\")(C3)#6\n",
    "U3 = tf.keras.layers.UpSampling2D(size=(2,2))(C4)\n",
    "P3 = tf.keras.layers.Add()([U3,C3])\n",
    "U1 = tf.keras.layers.UpSampling2D(size=(2,2))(P3)\n",
    "P1 = tf.keras.layers.Add()([U1,C1])\n",
    "R4 = tf.keras.layers.Reshape(target_shape=(1152,1,8))(C4)\n",
    "R3 = tf.keras.layers.Reshape(target_shape=(1152,1,32))(P3)\n",
    "R2 = tf.keras.layers.Reshape(target_shape=(1152,1,128))(P1)\n",
    "F4 = digitCaps(in_type=1152, in_shape=8, out_type=10, out_shape=16)(R4)\n",
    "F3 = digitCaps(in_type=1152, in_shape=32, out_type=10, out_shape=16)(R3)\n",
    "F2 = digitCaps(in_type=1152, in_shape=128, out_type=10, out_shape=16)(R2)\n",
    "query =tf.stack(F4+F3+F2)\n",
    "query = tf.keras.layers.Reshape(target_shape=(10,1,16))(query)\n",
    "key = tf.stack([F4,F3,F2])\n",
    "key = tf.transpose(key,[1,2,0,3])\n",
    "o = tf.keras.layers.Attention()([query,key])\n",
    "o = tf.keras.layers.Reshape(target_shape=(10,16))(o)\n",
    "output = keras.layers.Lambda(lambda x: keras.backend.sqrt(keras.backend.sum(keras.backend.square(x), 2)), output_shape=(10,))(o)\n",
    "model1 =  tf.keras.models.Model(inputs=inputs, outputs = output)\n",
    "model1.summary()\n",
    "model1.compile(optimizer=tf.keras.optimizers.Adam(0.00001),\n",
    "            loss=lambda y_true,y_pred:K.mean(K.sum(y_true * K.square(K.maximum(0., 0.9 - y_pred)) + 0.5 * (1 - y_true) * K.square(K.maximum(0., y_pred - 0.1)),1)),\n",
    "            metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_val = x_train[:12000]\n",
    "y_val = y_train[:12000]\n",
    "x_train = x_train[12000:]\n",
    "y_train = y_train[12000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 10000 samples\n",
      "Epoch 1/20\n",
      "  128/48000 [..............................] - ETA: 2:38:40"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-12-6fac5890561d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mmodel1his\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m128\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m20\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mx_test\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mc:\\users\\56236\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[0;32m    817\u001b[0m         \u001b[0mmax_queue_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmax_queue_size\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    818\u001b[0m         \u001b[0mworkers\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mworkers\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 819\u001b[1;33m         use_multiprocessing=use_multiprocessing)\n\u001b[0m\u001b[0;32m    820\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    821\u001b[0m   def evaluate(self,\n",
      "\u001b[1;32mc:\\users\\56236\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training_v2.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, model, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[0;32m    340\u001b[0m                 \u001b[0mmode\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mModeKeys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    341\u001b[0m                 \u001b[0mtraining_context\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtraining_context\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 342\u001b[1;33m                 total_epochs=epochs)\n\u001b[0m\u001b[0;32m    343\u001b[0m             \u001b[0mcbks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmake_logs\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepoch_logs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtraining_result\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mModeKeys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    344\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\56236\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training_v2.py\u001b[0m in \u001b[0;36mrun_one_epoch\u001b[1;34m(model, iterator, execution_function, dataset_size, batch_size, strategy, steps_per_epoch, num_samples, mode, training_context, total_epochs)\u001b[0m\n\u001b[0;32m    126\u001b[0m         step=step, mode=mode, size=current_batch_size) as batch_logs:\n\u001b[0;32m    127\u001b[0m       \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 128\u001b[1;33m         \u001b[0mbatch_outs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mexecution_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    129\u001b[0m       \u001b[1;32mexcept\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mStopIteration\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOutOfRangeError\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    130\u001b[0m         \u001b[1;31m# TODO(kaftan): File bug about tf function and errors.OutOfRangeError?\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\56236\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training_v2_utils.py\u001b[0m in \u001b[0;36mexecution_function\u001b[1;34m(input_fn)\u001b[0m\n\u001b[0;32m     96\u001b[0m     \u001b[1;31m# `numpy` translates Tensors to values in Eager mode.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     97\u001b[0m     return nest.map_structure(_non_none_constant_value,\n\u001b[1;32m---> 98\u001b[1;33m                               distributed_function(input_fn))\n\u001b[0m\u001b[0;32m     99\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    100\u001b[0m   \u001b[1;32mreturn\u001b[0m \u001b[0mexecution_function\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\56236\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    566\u001b[0m         \u001b[0mxla_context\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mExit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    567\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 568\u001b[1;33m       \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    569\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    570\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\56236\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    630\u001b[0m         \u001b[1;31m# Lifting succeeded, so variables are initialized and we can run the\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    631\u001b[0m         \u001b[1;31m# stateless function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 632\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    633\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    634\u001b[0m       \u001b[0mcanon_args\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcanon_kwds\u001b[0m \u001b[1;33m=\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\56236\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   2361\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2362\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2363\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2364\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2365\u001b[0m   \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\56236\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[1;34m(self, args, kwargs)\u001b[0m\n\u001b[0;32m   1609\u001b[0m          if isinstance(t, (ops.Tensor,\n\u001b[0;32m   1610\u001b[0m                            resource_variable_ops.BaseResourceVariable))),\n\u001b[1;32m-> 1611\u001b[1;33m         self.captured_inputs)\n\u001b[0m\u001b[0;32m   1612\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1613\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\56236\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1690\u001b[0m       \u001b[1;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1691\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[1;32m-> 1692\u001b[1;33m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[0;32m   1693\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[0;32m   1694\u001b[0m         \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\56236\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    543\u001b[0m               \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    544\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"executor_type\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexecutor_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"config_proto\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 545\u001b[1;33m               ctx=ctx)\n\u001b[0m\u001b[0;32m    546\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    547\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[1;32mc:\\users\\56236\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\tensorflow_core\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     59\u001b[0m     tensors = pywrap_tensorflow.TFE_Py_Execute(ctx._handle, device_name,\n\u001b[0;32m     60\u001b[0m                                                \u001b[0mop_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 61\u001b[1;33m                                                num_outputs)\n\u001b[0m\u001b[0;32m     62\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     63\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model1his = model1.fit(x_train, y_train, batch_size=128, epochs=20, validation_data=[x_test, y_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = tf.keras.layers.Input(shape=(28,28,1))\n",
    "C1 = keras.layers.Conv2D(filters=256, kernel_size=(9,9), strides=(1,1), activation=\"relu\")(inputs)\n",
    "C2 = keras.layers.Conv2D(filters=256, kernel_size=(9,9), strides=(2,2), activation=\"relu\")(C1)\n",
    "R1 = keras.layers.Reshape(target_shape=(1152,1,8))(C2)\n",
    "P1 = digitCaps(in_type=1152, in_shape=8, out_type=10, out_shape=16)(R1)\n",
    "out_caps = keras.layers.Lambda(lambda x: keras.backend.sqrt(keras.backend.sum(keras.backend.square(x), 2)), output_shape=(10,))(P1)\n",
    "capsule =  tf.keras.models.Model(inputs=inputs, outputs = out_caps)\n",
    "capsule.summary()\n",
    "capsule.compile(optimizer=tf.keras.optimizers.Adam(0.00001),\n",
    "            loss=[lambda y_true,y_pred:K.mean(K.sum(y_true * K.square(K.maximum(0., 0.9 - y_pred)) + 0.5 * (1 - y_true) * K.square(K.maximum(0., y_pred - 0.1)),1))],\n",
    "            metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model1his = capsule.fit(x_train, y_train, batch_size=128, epochs=20, validation_data=[x_test, y_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "loss = [0.1943,0.0429,0.0283,0.0223,0.0186,0.0165,0.0145,0.0132,0.0120,0.0112,0.0100,0.0090, 0.0088,0.0079,0.0073,0.0070,0.0064,0.0059,0.0055,0.0053]\n",
    "val_loss = [0.0515,0.0298,0.0223,0.0184,0.0156,0.0159,0.0134,0.0127,0.0115, 0.0109,0.0098,0.0091,0.0098,0.0092,0.0086,0.0083,0.0094,0.0078,0.0073,0.0081]\n",
    "epochs = range(1, len(loss) + 1)\n",
    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend()\n",
    "plt.savefig(\"result2.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "acc = [0.8196,0.9551,0.9719,0.9780,0.9814,0.9839,0.9852,0.9868,0.9876,0.9895,0.9902,0.9912,0.9911,0.9921,0.9928,0.9932,0.9937,0.9946,0.9948,0.9949]\n",
    "val_acc = [0.9474,0.9704,0.9788,0.9813,0.9852,0.9845,0.9874,0.9872,0.9877,0.9889,0.9901,0.9907,0.9899,0.9915,0.9914,0.9915,0.9905,0.9922,0.9927,0.9928]\n",
    "plt.plot(epochs, acc, 'bo', label='Training acc')\n",
    "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n",
    "plt.title('Training and validation accuracy')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend()\n",
    "plt.savefig(\"result1.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "acc = model1his.history['accuracy']\n",
    "val_acc = model1his.history['val_accuracy']\n",
    "plt.plot(epochs, acc, 'bo', label='Training acc')\n",
    "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n",
    "plt.title('Training and validation accuracy')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend()\n",
    "plt.savefig(\"result3.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "loss = model1his.history['loss']\n",
    "val_loss = model1his.history['val_loss']\n",
    "epochs = range(1, len(loss) + 1)\n",
    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend()\n",
    "plt.savefig(\"result4.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model1.evaluate(x_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "capsule.evaluate(x_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}