Lambda Layer_Custom ReLU.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fbdf7711-008d-4e4b-b4ee-d069d1f2d714",
   "metadata": {},
   "source": [
    "# *Lambda Layer: Custom ReLU Function*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2d34b45a-2eee-45a2-9def-7d2332211843",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import sklearn\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras.layers import Dense, Lambda, Flatten\n",
    "from tensorflow.keras.activations import softmax, relu\n",
    "from tensorflow.keras import backend as K\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f95f85e-094b-47cd-a9ae-0e53b11600fd",
   "metadata": {},
   "source": [
    "### *Fashion MNIST Dataset*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "74ff904c-c0bb-4a61-a522-2992d69c9dbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist = keras.datasets.mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2d47da0d-38cd-4be9-ab81-33572b365fb9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
      "11493376/11490434 [==============================] - 32s 3us/step\n"
     ]
    }
   ],
   "source": [
    "(train_x, train_y), (test_x, test_y) = mnist.load_data()\n",
    "train_x, test_x = train_x/255.0, test_x/255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "201a456a-bd05-43d1-a36c-d430ebef5462",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8be661e2-5ce8-498e-abf0-8f7d8cf6d7b5",
   "metadata": {},
   "source": [
    "### *Creating a Validation set*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0631e7f6-7887-458b-940e-78eb33cf9fe2",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x, val_x, train_y, val_y = train_test_split(train_x, train_y, test_size=0.2, random_state= 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2ae0054f-a2fb-4338-a697-a70d32bb9bc4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(48000, 28, 28)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "053d15bd-97e7-4686-8e70-8498190c6d7c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(12000, 28, 28)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_x.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1afad91a-389f-4345-b550-cd602134292b",
   "metadata": {},
   "source": [
    "## *Model Creation: Standard ReLU*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1e55bf96-8dfa-47c2-be4b-26980595a0be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_1 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_relu = keras.Sequential([\n",
    "    Flatten(input_shape = (28,28)),\n",
    "    Dense(128, activation = relu),\n",
    "    Dense(10, activation= softmax)\n",
    "])\n",
    "\n",
    "model_relu.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "98e19702-4640-478d-8d9b-ef5a8f6c3540",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "1500/1500 [==============================] - 7s 4ms/step - loss: 0.2790 - accuracy: 0.9208 - val_loss: 0.1741 - val_accuracy: 0.9482\n",
      "Epoch 2/5\n",
      "1500/1500 [==============================] - 5s 3ms/step - loss: 0.1243 - accuracy: 0.9635 - val_loss: 0.1085 - val_accuracy: 0.9678\n",
      "Epoch 3/5\n",
      "1500/1500 [==============================] - 5s 3ms/step - loss: 0.0851 - accuracy: 0.9748 - val_loss: 0.1052 - val_accuracy: 0.9678\n",
      "Epoch 4/5\n",
      "1500/1500 [==============================] - 4s 3ms/step - loss: 0.0645 - accuracy: 0.9809 - val_loss: 0.0877 - val_accuracy: 0.9740\n",
      "Epoch 5/5\n",
      "1500/1500 [==============================] - 4s 3ms/step - loss: 0.0514 - accuracy: 0.9845 - val_loss: 0.0824 - val_accuracy: 0.9747\n"
     ]
    }
   ],
   "source": [
    "model_relu.compile(optimizer='adam',loss= keras.losses.sparse_categorical_crossentropy, metrics=['accuracy'])\n",
    "\n",
    "history_relu = model_relu.fit(train_x, train_y,\n",
    "                              epochs= 5,\n",
    "                              validation_data=(val_x, val_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "025e7013-1238-4f65-b423-540bf1264d15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "313/313 [==============================] - 1s 3ms/step - loss: 0.0796 - accuracy: 0.9753\n",
      "Accuracy:  97.53000140190125\n",
      "Loss:  0.07962781935930252\n"
     ]
    }
   ],
   "source": [
    "loss_relu, accuracy_relu = model_relu.evaluate(test_x, test_y)\n",
    "print(\"Accuracy: \", accuracy_relu*100)\n",
    "print(\"Loss: \", loss_relu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af746bf4-7d82-4d50-a5eb-bcef7134adad",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "7566345b-9c43-4d4e-a7e1-77d85f9283ac",
   "metadata": {},
   "source": [
    "## *Model: Custom ReLU using **Lambda Layers***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f21daa91-97c6-4f86-8bbd-09d6a8b90d31",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_2 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "lambda (Lambda)              (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_lam_1 = keras.Sequential([\n",
    "    Flatten(input_shape = (28,28)),\n",
    "    Dense(128),\n",
    "    Lambda(lambda x: tf.abs(x)),\n",
    "    Dense(10, activation= softmax)\n",
    "])\n",
    "\n",
    "model_lam_1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6521c1f1-e113-46c7-9649-8566a0ac1a5b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "1500/1500 [==============================] - 5s 4ms/step - loss: 0.2467 - accuracy: 0.9293 - val_loss: 0.1257 - val_accuracy: 0.9643\n",
      "Epoch 2/5\n",
      "1500/1500 [==============================] - 4s 3ms/step - loss: 0.0995 - accuracy: 0.9700 - val_loss: 0.0910 - val_accuracy: 0.9729\n",
      "Epoch 3/5\n",
      "1500/1500 [==============================] - 4s 3ms/step - loss: 0.0669 - accuracy: 0.9798 - val_loss: 0.0957 - val_accuracy: 0.9714\n",
      "Epoch 4/5\n",
      "1500/1500 [==============================] - 5s 3ms/step - loss: 0.0500 - accuracy: 0.9849 - val_loss: 0.0891 - val_accuracy: 0.9742\n",
      "Epoch 5/5\n",
      "1500/1500 [==============================] - 5s 3ms/step - loss: 0.0371 - accuracy: 0.9886 - val_loss: 0.0804 - val_accuracy: 0.9781\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1d8b97c8220>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_lam_1.compile(optimizer='adam',loss= keras.losses.sparse_categorical_crossentropy, metrics=['accuracy'])\n",
    "\n",
    "model_lam_1.fit(train_x, train_y,\n",
    "                epochs= 5,\n",
    "                validation_data=(val_x, val_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "00856f50-60b0-4205-b059-a3ab4b979a08",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "313/313 [==============================] - 1s 4ms/step - loss: 0.0863 - accuracy: 0.9760\n",
      "Accuracy:  97.60000109672546\n",
      "Loss:  0.08629355579614639\n"
     ]
    }
   ],
   "source": [
    "loss_lam_1, accuracy_lam_1 = model_lam_1.evaluate(test_x, test_y)\n",
    "print(\"Accuracy: \", accuracy_lam_1*100)\n",
    "print(\"Loss: \", loss_lam_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "693292bf-7314-4644-a017-7204c5a5c49f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "de8e85a6-5598-49a7-b93d-1eeb7bd297b3",
   "metadata": {},
   "source": [
    "## *Lambda Layers: Model 2*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b72e9c10-1c33-4b63-844f-f8618603ac54",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_2 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "lambda (Lambda)              (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "def my_custom_relu(x):\n",
    "    return K.maximum(-0.1, x)\n",
    "\n",
    "model_lam_2 = keras.Sequential([\n",
    "    Flatten(input_shape = (28,28)),\n",
    "    Dense(128),\n",
    "    Lambda(my_custom_relu),\n",
    "    Dense(10, activation= softmax)\n",
    "])\n",
    "\n",
    "model_lam_1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "596ab97a-bdc4-44ca-9701-0a268ace76e2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "1500/1500 [==============================] - 6s 4ms/step - loss: 0.2795 - accuracy: 0.9202 - val_loss: 0.1657 - val_accuracy: 0.9538\n",
      "Epoch 2/5\n",
      "1500/1500 [==============================] - 5s 3ms/step - loss: 0.1249 - accuracy: 0.9644 - val_loss: 0.1235 - val_accuracy: 0.9635\n",
      "Epoch 3/5\n",
      "1500/1500 [==============================] - 5s 3ms/step - loss: 0.0851 - accuracy: 0.9753 - val_loss: 0.0979 - val_accuracy: 0.9710\n",
      "Epoch 4/5\n",
      "1500/1500 [==============================] - 5s 3ms/step - loss: 0.0633 - accuracy: 0.9805 - val_loss: 0.0914 - val_accuracy: 0.9722\n",
      "Epoch 5/5\n",
      "1500/1500 [==============================] - 5s 3ms/step - loss: 0.0480 - accuracy: 0.9857 - val_loss: 0.0833 - val_accuracy: 0.9752\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1d8b5fa8ca0>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_lam_2.compile(optimizer='adam',loss= keras.losses.sparse_categorical_crossentropy, metrics=['accuracy'])\n",
    "\n",
    "model_lam_2.fit(train_x, train_y,\n",
    "                epochs= 5,\n",
    "                validation_data=(val_x, val_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "8708c878-2d40-4818-a8ad-9cb074f08fd7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "313/313 [==============================] - 1s 3ms/step - loss: 0.0790 - accuracy: 0.9744\n",
      "Accuracy:  97.4399983882904\n",
      "Loss:  0.07900013029575348\n"
     ]
    }
   ],
   "source": [
    "loss_lam_2, accuracy_lam_2 = model_lam_2.evaluate(test_x, test_y)\n",
    "print(\"Accuracy: \", accuracy_lam_2*100)\n",
    "print(\"Loss: \", loss_lam_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00f6e396-fce1-4efa-a815-056123f28d20",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}