Optimizers & AutoEncoder.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AaronThomasJoseph/DL-Projects/blob/main/CS671_A2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "63636K6XwY-C"
      },
      "outputs": [],
      "source": [
        "#Libraries\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import zipfile\n",
        "import os\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import tensorflow\n",
        "from tensorflow import keras\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense\n",
        "import keras.layers\n",
        "from keras.callbacks import EarlyStopping\n",
        "from keras import optimizers\n",
        "from keras.layers import GaussianNoise\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import recall_score\n",
        "from sklearn.metrics import f1_score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vE7dNvLUzphm"
      },
      "outputs": [],
      "source": [
        "#Loading Image Data\n",
        "with zipfile.ZipFile(\"Group_21_Assignment2.zip\",\"r\") as zip_ref:\n",
        "    zip_ref.extractall()\n",
        "zip_ref.close()\n",
        "\n",
        "def load_images_from_folder(folder):\n",
        "    images = []\n",
        "    for filename in os.listdir(folder):\n",
        "        img = plt.imread(os.path.join(folder,filename))\n",
        "        if img is not None:\n",
        "            images.append(img)\n",
        "    return images"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gUY-mADu9KsQ"
      },
      "source": [
        "#**`Dataset`**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ozv0jEV_1xo1"
      },
      "outputs": [],
      "source": [
        "#Train Data\n",
        "train0 = np.array(load_images_from_folder(\"Group_21/train/0\"))\n",
        "train2 = np.array(load_images_from_folder(\"Group_21/train/2\"))\n",
        "train4 = np.array(load_images_from_folder(\"Group_21/train/4\"))\n",
        "train6 = np.array(load_images_from_folder(\"Group_21/train/6\"))\n",
        "train7 = np.array(load_images_from_folder(\"Group_21/train/7\"))\n",
        "\n",
        "train0 = train0.reshape((train0.shape[0],28*28))\n",
        "train2 = train2.reshape((train2.shape[0],28*28))\n",
        "train4 = train4.reshape((train4.shape[0],28*28))\n",
        "train6 = train6.reshape((train6.shape[0],28*28))\n",
        "train7 = train7.reshape((train7.shape[0],28*28))\n",
        "\n",
        "x_train = np.concatenate([train0,train2,train4,train6,train7])\n",
        "y_train = np.concatenate([0+np.zeros(len(train0)),2+np.zeros(len(train2)),4+np.zeros(len(train4)),\n",
        "                          6+np.zeros(len(train6)),7+np.zeros(len(train7))])\n",
        "\n",
        "#Validation Data\n",
        "val0 = np.array(load_images_from_folder(\"Group_21/val/0\"))\n",
        "val2 = np.array(load_images_from_folder(\"Group_21/val/2\"))\n",
        "val4 = np.array(load_images_from_folder(\"Group_21/val/4\"))\n",
        "val6 = np.array(load_images_from_folder(\"Group_21/val/6\"))\n",
        "val7 = np.array(load_images_from_folder(\"Group_21/val/7\"))\n",
        "\n",
        "val0 = val0.reshape((val0.shape[0],28*28))\n",
        "val2 = val2.reshape((val2.shape[0],28*28))\n",
        "val4 = val4.reshape((val4.shape[0],28*28))\n",
        "val6 = val6.reshape((val6.shape[0],28*28))\n",
        "val7 = val7.reshape((val7.shape[0],28*28))\n",
        "\n",
        "x_val = np.concatenate([val0,val2,val4,val6,val7])\n",
        "y_val = np.concatenate([0+np.zeros(len(val0)),2+np.zeros(len(val2)),4+np.zeros(len(val4)),\n",
        "                          6+np.zeros(len(val6)),7+np.zeros(len(val7))])\n",
        "\n",
        "#Test Data\n",
        "test0 = np.array(load_images_from_folder(\"Group_21/test/0\"))\n",
        "test2 = np.array(load_images_from_folder(\"Group_21/test/2\"))\n",
        "test4 = np.array(load_images_from_folder(\"Group_21/test/4\"))\n",
        "test6 = np.array(load_images_from_folder(\"Group_21/test/6\"))\n",
        "test7 = np.array(load_images_from_folder(\"Group_21/test/7\"))\n",
        "\n",
        "test0 = test0.reshape((test0.shape[0],28*28))\n",
        "test2 = test2.reshape((test2.shape[0],28*28))\n",
        "test4 = test4.reshape((test4.shape[0],28*28))\n",
        "test6 = test6.reshape((test6.shape[0],28*28))\n",
        "test7 = test7.reshape((test7.shape[0],28*28))\n",
        "\n",
        "x_test = np.concatenate([test0,test2,test4,test6,test7])\n",
        "y_test = np.concatenate([0+np.zeros(len(test0)),2+np.zeros(len(test2)),4+np.zeros(len(test4)),\n",
        "                          6+np.zeros(len(test6)),7+np.zeros(len(test7))])\n",
        "\n",
        "del train0,train2,train4,train6,train7\n",
        "del val0,val2,val4,val6,val7\n",
        "del test0,test2,test4,test6,test7\n",
        "\n",
        "#Normalizing Input\n",
        "x_train,x_val,x_test = x_train/255,x_val/255,x_test/255"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VDjvB3_KCHVk"
      },
      "source": [
        "#**`FCNN with 3 Hidden Layers`**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ly8I2yc5PfUm"
      },
      "outputs": [],
      "source": [
        "#Converting one hot encoded test label to label\n",
        "def ohetolabel(array):\n",
        "  output = list()\n",
        "  for i in range(len(array)):\n",
        "    output.append(np.argmax(array[i]))\n",
        "  return np.array(output)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OveSX2Q-1Lsi"
      },
      "outputs": [],
      "source": [
        "#Plot cf matrix\n",
        "def cf_matrix_plot(cf_matrix,title):\n",
        "  #group_names = ['True Neg','False Pos','False Neg','True Pos','True Pos','True Pos','True Pos','True Pos','True Pos']\n",
        "\n",
        "  group_counts = [\"{0:0.0f}\".format(value) for value in\n",
        "                  cf_matrix.flatten()]\n",
        "\n",
        "  group_percentages = [\"{0:.2%}\".format(value) for value in\n",
        "                      cf_matrix.flatten()/np.sum(cf_matrix)]\n",
        "\n",
        "  labels = [f\"{v1}\\n{v2}\\n\" for v1, v2 in\n",
        "            zip(group_counts,group_percentages)]\n",
        "\n",
        "  labels = np.asarray(labels).reshape(5,5)\n",
        "\n",
        "  ax = sns.heatmap(cf_matrix, annot=labels, fmt='', cmap='Blues')\n",
        "\n",
        "  ax.set_title(title);\n",
        "  ax.set_xlabel('\\nPredicted Image Class')\n",
        "  ax.set_ylabel('Actual Image Class');\n",
        "\n",
        "  ## Ticket labels - List must be in alphabetical order\n",
        "  ax.xaxis.set_ticklabels(['0','2','4','6','7'])\n",
        "  ax.yaxis.set_ticklabels(['0','2','4','6','7'])"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Neural network\n",
        "def clf_1_layer(x_train,y_train,x_val,y_val,x_test,y_test,node1,C,optimizer,batch_size,error_th,p,epochs=1e3):\n",
        "  model = Sequential()\n",
        "  model.add(Dense(node1, input_dim=x_train.shape[1], activation='sigmoid'))\n",
        "  model.add(Dense(C, activation='softmax'))\n",
        "\n",
        "  es = EarlyStopping(monitor='loss',mode='min',min_delta = error_th,verbose=1,patience=p)\n",
        "  if optimizer not in [\"SGD\",\"RMSProp\",\"Adam\"]:\n",
        "    opt = \"SGD\"\n",
        "  else:\n",
        "    opt = optimizer\n",
        "\n",
        "  #One Hot Encoding\n",
        "  ohe = OneHotEncoder()\n",
        "  y_train = ohe.fit_transform(y_train.reshape(-1,1)).toarray()\n",
        "  y_val = ohe.fit_transform(y_val.reshape(-1,1)).toarray()\n",
        "  y_test = ohe.fit_transform(y_test.reshape(-1,1)).toarray()\n",
        "\n",
        "  model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])\n",
        "  history = model.fit(x_train, y_train,validation_data = (x_val,y_val) ,epochs=int(epochs), batch_size=batch_size,callbacks=[es])\n",
        "\n",
        "  y_train_pred = model.predict(x_train)\n",
        "  y_val_pred   = model.predict(x_val)\n",
        "  y_test_pred  = model.predict(x_test)\n",
        "\n",
        "  y_train_pred = ohetolabel(y_train_pred)\n",
        "  y_val_pred   = ohetolabel(y_val_pred)\n",
        "  y_test_pred  = ohetolabel(y_test_pred)\n",
        "\n",
        "  y_train = ohetolabel(y_train)\n",
        "  y_val   = ohetolabel(y_val)\n",
        "  y_test  = ohetolabel(y_test)\n",
        "\n",
        "  print(\"No of Epochs =\",len(history.history[\"loss\"]))\n",
        "  plt.figure(figsize=(12.5,8))\n",
        "  plt.suptitle(optimizer,fontweight=\"bold\",y=1+1e-2)\n",
        "  plt.subplot(1,2,2)\n",
        "  plt.plot(history.history['accuracy'],marker='.')\n",
        "  plt.plot(history.history['val_accuracy'],marker='.')\n",
        "  plt.title('Model accuracy')\n",
        "  plt.ylabel('Accuracy')\n",
        "  plt.xlabel('Epoch')\n",
        "  plt.legend(['Train', 'Validation'], loc='upper left')\n",
        "  plt.grid()\n",
        "\n",
        "  plt.subplot(1,2,1)\n",
        "  plt.plot(history.history['loss'],marker='.')\n",
        "  plt.plot(history.history['val_loss'],marker='.')\n",
        "  plt.title('Model loss')\n",
        "  plt.ylabel('Loss')\n",
        "  plt.xlabel('Epoch')\n",
        "  plt.legend(['Train', 'Validation'], loc='upper left')\n",
        "  plt.grid()\n",
        "  plt.tight_layout();plt.show()\n",
        "\n",
        "  print(\"Training Accuracy =\\t\",history.history['accuracy'][-1])\n",
        "  print(\"Validation Accuracy =\\t\",history.history['val_accuracy'][-1])\n",
        "\n",
        "\n",
        "  #Generate the confusion matrix\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(optimizer,fontweight=\"bold\",y=1+1e-2)\n",
        "  plt.subplot(1,3,1)\n",
        "  cf_matrix = confusion_matrix(y_train, y_train_pred)\n",
        "  cf_matrix_plot(cf_matrix,\"Training Data\")\n",
        "\n",
        "  plt.subplot(1,3,2)\n",
        "  cf_matrix = confusion_matrix(y_val, y_val_pred)\n",
        "  cf_matrix_plot(cf_matrix,\"Validation Data\")\n",
        "\n",
        "  plt.subplot(1,3,3)\n",
        "  cf_matrix = confusion_matrix(y_test, y_test_pred)\n",
        "  cf_matrix_plot(cf_matrix,\"Test Data\")\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  print(\"\\tTraining Data\\t\\t\\t\\tValidation Data\\t\\t\\t\\tTest Data\")\n",
        "  print(\"Accuracy:\",end=\" \")\n",
        "  print(np.around(accuracy_score(y_train , y_train_pred),4),end=\"\\t\"*4)\n",
        "  print(np.around(accuracy_score(y_val , y_val_pred),4),end=\"\\t\"*5)\n",
        "  print(np.around(accuracy_score(y_test , y_test_pred),4))\n",
        "\n",
        "  print(\"Precision:\",end=\"\")\n",
        "  print(np.around(precision_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "  print(np.around(precision_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "  print(np.around(precision_score(y_test , y_test_pred,average=\"macro\"),4))\n",
        "\n",
        "  print(\"Recall:\",end=\"   \")\n",
        "  print(np.around(recall_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "  print(np.around(recall_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "  print(np.around(recall_score(y_test , y_test_pred,average=\"macro\"),4))\n",
        "\n",
        "  print(\"F1-Score:\",end=\" \")\n",
        "  print(np.around(f1_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "  print(np.around(f1_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "  print(np.around(f1_score(y_test , y_test_pred,average=\"macro\"),4))\n"
      ],
      "metadata": {
        "id": "vaoZT_O-EGa2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Neural network\n",
        "def clf_2_layer(x_train,y_train,x_val,y_val,x_test,y_test,node1,node2,C,optimizer,batch_size,error_th,p,epochs=1e3):\n",
        "  model = Sequential()\n",
        "  model.add(Dense(node1, input_dim=x_train.shape[1], activation='sigmoid'))\n",
        "  model.add(Dense(node2, activation=\"sigmoid\"))\n",
        "  model.add(Dense(C, activation='softmax'))\n",
        "\n",
        "  es = EarlyStopping(monitor='loss',mode='min',min_delta = error_th,verbose=1,patience=p)\n",
        "  if optimizer not in [\"SGD\",\"RMSProp\",\"Adam\"]:\n",
        "    opt = \"SGD\"\n",
        "  else:\n",
        "    opt = optimizer\n",
        "\n",
        "  #One Hot Encoding\n",
        "  ohe = OneHotEncoder()\n",
        "  y_train = ohe.fit_transform(y_train.reshape(-1,1)).toarray()\n",
        "  y_val = ohe.fit_transform(y_val.reshape(-1,1)).toarray()\n",
        "  y_test = ohe.fit_transform(y_test.reshape(-1,1)).toarray()\n",
        "\n",
        "  model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])\n",
        "  history = model.fit(x_train, y_train,validation_data = (x_val,y_val) ,epochs=int(epochs), batch_size=batch_size,callbacks=[es])\n",
        "\n",
        "  y_train_pred = model.predict(x_train)\n",
        "  y_val_pred   = model.predict(x_val)\n",
        "  y_test_pred  = model.predict(x_test)\n",
        "\n",
        "  y_train_pred = ohetolabel(y_train_pred)\n",
        "  y_val_pred   = ohetolabel(y_val_pred)\n",
        "  y_test_pred  = ohetolabel(y_test_pred)\n",
        "\n",
        "  y_train = ohetolabel(y_train)\n",
        "  y_val   = ohetolabel(y_val)\n",
        "  y_test  = ohetolabel(y_test)\n",
        "\n",
        "  print(\"No of Epochs =\",len(history.history[\"loss\"]))\n",
        "  plt.figure(figsize=(12.5,8))\n",
        "  plt.suptitle(optimizer,fontweight=\"bold\",y=1+1e-2)\n",
        "  plt.subplot(1,2,2)\n",
        "  plt.plot(history.history['accuracy'],marker='.')\n",
        "  plt.plot(history.history['val_accuracy'],marker='.')\n",
        "  plt.title('Model accuracy')\n",
        "  plt.ylabel('Accuracy')\n",
        "  plt.xlabel('Epoch')\n",
        "  plt.legend(['Train', 'Validation'], loc='upper left')\n",
        "  plt.grid()\n",
        "\n",
        "  plt.subplot(1,2,1)\n",
        "  plt.plot(history.history['loss'],marker='.')\n",
        "  plt.plot(history.history['val_loss'],marker='.')\n",
        "  plt.title('Model loss')\n",
        "  plt.ylabel('Loss')\n",
        "  plt.xlabel('Epoch')\n",
        "  plt.legend(['Train', 'Validation'], loc='upper left')\n",
        "  plt.grid()\n",
        "  plt.tight_layout();plt.show()\n",
        "\n",
        "  print(\"Training Accuracy =\\t\",history.history['accuracy'][-1])\n",
        "  print(\"Validation Accuracy =\\t\",history.history['val_accuracy'][-1])\n",
        "\n",
        "\n",
        "  #Generate the confusion matrix\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(optimizer,fontweight=\"bold\",y=1+1e-2)\n",
        "  plt.subplot(1,3,1)\n",
        "  cf_matrix = confusion_matrix(y_train, y_train_pred)\n",
        "  cf_matrix_plot(cf_matrix,\"Training Data\")\n",
        "\n",
        "  plt.subplot(1,3,2)\n",
        "  cf_matrix = confusion_matrix(y_val, y_val_pred)\n",
        "  cf_matrix_plot(cf_matrix,\"Validation Data\")\n",
        "\n",
        "  plt.subplot(1,3,3)\n",
        "  cf_matrix = confusion_matrix(y_test, y_test_pred)\n",
        "  cf_matrix_plot(cf_matrix,\"Test Data\")\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  print(\"\\tTraining Data\\t\\t\\t\\tValidation Data\\t\\t\\t\\tTest Data\")\n",
        "  print(\"Accuracy:\",end=\" \")\n",
        "  print(np.around(accuracy_score(y_train , y_train_pred),4),end=\"\\t\"*4)\n",
        "  print(np.around(accuracy_score(y_val , y_val_pred),4),end=\"\\t\"*5)\n",
        "  print(np.around(accuracy_score(y_test , y_test_pred),4))\n",
        "\n",
        "  print(\"Precision:\",end=\"\")\n",
        "  print(np.around(precision_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "  print(np.around(precision_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "  print(np.around(precision_score(y_test , y_test_pred,average=\"macro\"),4))\n",
        "\n",
        "  print(\"Recall:\",end=\"   \")\n",
        "  print(np.around(recall_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "  print(np.around(recall_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "  print(np.around(recall_score(y_test , y_test_pred,average=\"macro\"),4))\n",
        "\n",
        "  print(\"F1-Score:\",end=\" \")\n",
        "  print(np.around(f1_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "  print(np.around(f1_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "  print(np.around(f1_score(y_test , y_test_pred,average=\"macro\"),4))"
      ],
      "metadata": {
        "id": "OSHcbSqsD8-n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QzN3m70aDNc3"
      },
      "outputs": [],
      "source": [
        "# Neural network\n",
        "def clf_3_layer(x_train,y_train,x_val,y_val,x_test,y_test,node1,node2,node3,C,optimizer,batch_size,error_th,p,epochs=1e3):\n",
        "\n",
        "  tf.keras.initializers.RandomNormal(mean=0.0, stddev=1, seed=42)\n",
        "  model = Sequential()\n",
        "  model.add(Dense(node1, input_dim=x_train.shape[1], activation='sigmoid',\n",
        "                  kernel_initializer='random_normal',bias_initializer='zeros'))\n",
        "  model.add(Dense(node2, activation=\"sigmoid\",\n",
        "                  kernel_initializer='random_normal',bias_initializer='zeros'))\n",
        "  model.add(Dense(node3, activation='sigmoid',\n",
        "                  kernel_initializer='random_normal',bias_initializer='zeros'))\n",
        "  model.add(Dense(C, activation='softmax',\n",
        "                  kernel_initializer='random_normal',bias_initializer='zeros'))\n",
        "\n",
        "  es = EarlyStopping(monitor='loss',mode='min',min_delta = error_th,verbose=1,patience=p)\n",
        "  if optimizer not in [\"SGD\",\"RMSProp\",\"Adam\"]:\n",
        "    opt = \"SGD\"\n",
        "  else:\n",
        "    opt = optimizer\n",
        "\n",
        "  #One Hot Encoding\n",
        "  ohe = OneHotEncoder()\n",
        "  y_train = ohe.fit_transform(y_train.reshape(-1,1)).toarray()\n",
        "  y_val = ohe.fit_transform(y_val.reshape(-1,1)).toarray()\n",
        "  y_test = ohe.fit_transform(y_test.reshape(-1,1)).toarray()\n",
        "\n",
        "  model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])\n",
        "  history = model.fit(x_train, y_train,validation_data = (x_val,y_val) ,epochs=int(epochs), batch_size=batch_size,callbacks=[es])\n",
        "\n",
        "  y_train_pred = model.predict(x_train)\n",
        "  y_val_pred   = model.predict(x_val)\n",
        "  y_test_pred  = model.predict(x_test)\n",
        "\n",
        "  y_train_pred = ohetolabel(y_train_pred)\n",
        "  y_val_pred   = ohetolabel(y_val_pred)\n",
        "  y_test_pred  = ohetolabel(y_test_pred)\n",
        "\n",
        "  y_train = ohetolabel(y_train)\n",
        "  y_val   = ohetolabel(y_val)\n",
        "  y_test  = ohetolabel(y_test)\n",
        "\n",
        "  print(\"No of Epochs =\",len(history.history[\"loss\"]))\n",
        "  plt.figure(figsize=(12.5,8))\n",
        "  plt.suptitle(optimizer,fontweight=\"bold\",y=1+1e-2)\n",
        "  plt.subplot(1,2,2)\n",
        "  plt.plot(history.history['accuracy'],marker='.')\n",
        "  plt.plot(history.history['val_accuracy'],marker='.')\n",
        "  plt.title('Model accuracy')\n",
        "  plt.ylabel('Accuracy')\n",
        "  plt.xlabel('Epoch')\n",
        "  plt.legend(['Train', 'Validation'], loc='upper left')\n",
        "  plt.grid()\n",
        "\n",
        "  plt.subplot(1,2,1)\n",
        "  plt.plot(history.history['loss'],marker='.')\n",
        "  plt.plot(history.history['val_loss'],marker='.')\n",
        "  plt.title('Model loss')\n",
        "  plt.ylabel('Loss')\n",
        "  plt.xlabel('Epoch')\n",
        "  plt.legend(['Train', 'Validation'], loc='upper left')\n",
        "  plt.grid()\n",
        "  plt.tight_layout();plt.show()\n",
        "\n",
        "  print(\"Training Accuracy =\\t\",history.history['accuracy'][-1])\n",
        "  print(\"Validation Accuracy =\\t\",history.history['val_accuracy'][-1])\n",
        "\n",
        "\n",
        "  #Generate the confusion matrix\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(optimizer,fontweight=\"bold\",y=1+1e-2)\n",
        "  plt.subplot(1,3,1)\n",
        "  cf_matrix = confusion_matrix(y_train, y_train_pred)\n",
        "  cf_matrix_plot(cf_matrix,\"Training Data\")\n",
        "\n",
        "  plt.subplot(1,3,2)\n",
        "  cf_matrix = confusion_matrix(y_val, y_val_pred)\n",
        "  cf_matrix_plot(cf_matrix,\"Validation Data\")\n",
        "\n",
        "  plt.subplot(1,3,3)\n",
        "  cf_matrix = confusion_matrix(y_test, y_test_pred)\n",
        "  cf_matrix_plot(cf_matrix,\"Test Data\")\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  print(\"\\tTraining Data\\t\\t\\t\\tValidation Data\\t\\t\\t\\tTest Data\")\n",
        "  print(\"Accuracy:\",end=\" \")\n",
        "  print(np.around(accuracy_score(y_train , y_train_pred),4),end=\"\\t\"*4)\n",
        "  print(np.around(accuracy_score(y_val , y_val_pred),4),end=\"\\t\"*5)\n",
        "  print(np.around(accuracy_score(y_test , y_test_pred),4))\n",
        "\n",
        "  print(\"Precision:\",end=\"\")\n",
        "  print(np.around(precision_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "  print(np.around(precision_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "  print(np.around(precision_score(y_test , y_test_pred,average=\"macro\"),4))\n",
        "\n",
        "  print(\"Recall:\",end=\"   \")\n",
        "  print(np.around(recall_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "  print(np.around(recall_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "  print(np.around(recall_score(y_test , y_test_pred,average=\"macro\"),4))\n",
        "\n",
        "  print(\"F1-Score:\",end=\" \")\n",
        "  print(np.around(f1_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "  print(np.around(f1_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "  print(np.around(f1_score(y_test , y_test_pred,average=\"macro\"),4))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "q8zsQ_8zNncM"
      },
      "outputs": [],
      "source": [
        "node1 = 30\n",
        "node2 = 20\n",
        "node3 = 10\n",
        "C = 5\n",
        "error_th = 1e-4\n",
        "p = 1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "D2NTpwZnD3MY"
      },
      "outputs": [],
      "source": [
        "optimizer ='SGD'\n",
        "batch_size = 1\n",
        "tensorflow.keras.optimizers.SGD(learning_rate=0.001)\n",
        "clf_3_layer(x_train,y_train,x_val,y_val,x_test,y_test,node1,node2,node3,C,optimizer,batch_size,error_th,p,epochs=1e5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wuXsyFNdDNeF"
      },
      "outputs": [],
      "source": [
        "optimizer  ='SGD with momentum (NAG)'\n",
        "batch_size = 32\n",
        "tensorflow.keras.optimizers.SGD(learning_rate=0.001,momentum = 0.9,nesterov=True)\n",
        "clf_3_layer(x_train,y_train,x_val,y_val,x_test,y_test,node1,node2,node3,C,optimizer,batch_size,error_th,p,epochs=1e5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YhQUw_0QDhlw"
      },
      "outputs": [],
      "source": [
        "optimizer  ='Vanilla Gradient Descent'\n",
        "batch_size = len(x_train)\n",
        "tensorflow.keras.optimizers.SGD(learning_rate=0.001)\n",
        "clf_3_layer(x_train,y_train,x_val,y_val,x_test,y_test,node1,node2,node3,C,optimizer,batch_size,error_th,p=1000,epochs=1e5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0WQLKxEMCaWj"
      },
      "outputs": [],
      "source": [
        "optimizer  ='RMSprop'\n",
        "batch_size = 32\n",
        "tensorflow.keras.optimizers.RMSprop(learning_rate=0.001, momentum=0.9, rho = 0.99)\n",
        "clf_3_layer(x_train,y_train,x_val,y_val,x_test,y_test,node1,node2,node3,C,optimizer,batch_size,error_th,p,epochs=1e5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0CTpJA9rCfFM"
      },
      "outputs": [],
      "source": [
        "optimizer  ='Adam'\n",
        "batch_size = 32\n",
        "tensorflow.keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)\n",
        "clf_3_layer(x_train,y_train,x_val,y_val,x_test,y_test,node1,node2,node3,C,optimizer,batch_size,error_th,p,epochs=1e5)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qb31YLbm5whF"
      },
      "source": [
        "#**`AutoEncoder`**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oZbkchiy8wVT"
      },
      "source": [
        "##**`AutoEncoder with 1 Hidden Layers`**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2iB2M_1G59gv"
      },
      "outputs": [],
      "source": [
        "# Auto Encoder\n",
        "def autoencoder_1_layer(x_train,x_val,x_test,y_train,y_val,y_test,node1,optimizer,batch_size,error_th,epochs=1e3):\n",
        "  model = Sequential()\n",
        "  model.add(Dense(node1, input_dim=x_train.shape[1], activation='sigmoid'))\n",
        "  model.add(Dense(x_train.shape[1], activation='linear'))\n",
        "\n",
        "  es = EarlyStopping(monitor='loss',mode='min',min_delta = error_th,verbose=1,patience=1)\n",
        "\n",
        "  model.compile(loss='mean_squared_error', optimizer=optimizer, metrics=['mean_squared_error'])\n",
        "  history = model.fit(x_train, x_train,validation_data = (x_val,x_val) ,epochs=int(epochs), batch_size=batch_size,callbacks=[es])\n",
        "\n",
        "  print(\"No of Epochs =\",len(history.history[\"loss\"]))\n",
        "  plt.figure(figsize=(12.5,8))\n",
        "  plt.suptitle(\"1 Hidden Layer\",fontweight=\"bold\",y=1+1e-2)\n",
        "  plt.subplot(1,1,1)\n",
        "  plt.plot(history.history['loss'],marker='.')\n",
        "  plt.plot(history.history['val_loss'],marker='.')\n",
        "  plt.title('Model loss')\n",
        "  plt.ylabel('Loss')\n",
        "  plt.xlabel('Epoch')\n",
        "  plt.legend(['Train', 'Validation'], loc='upper left')\n",
        "  plt.grid()\n",
        "  plt.tight_layout();plt.show()\n",
        "\n",
        "  print(\"Training Loss\\t=\",history.history['loss'][-1])\n",
        "  print(\"Validation Loss\\t=\",history.history['val_loss'][-1])\n",
        "\n",
        "  x_train_pred = model.predict(x_train)\n",
        "  x_val_pred   = model.predict(x_val)\n",
        "  x_test_pred  = model.predict(x_test)\n",
        "  test_loss = ((x_test-x_test_pred)**2).mean()\n",
        "  print(\"Test Loss\\t=\",test_loss)\n",
        "\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(\"Average Reconstruction Error vs Data\",fontweight=\"bold\",y=1+1e-2)\n",
        "  plt.subplot(1,1,1)\n",
        "  plt.bar(height = [history.history['loss'][-1],history.history['val_loss'][-1],test_loss],x = ['Training Data','Validation Data','Test Data'])\n",
        "  plt.ylabel('Average Reconstruction Error')\n",
        "  plt.xlabel('Data')\n",
        "  plt.grid()\n",
        "  plt.tight_layout();plt.show()\n",
        "\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(\"Training Set\\nOriginal Image\",fontweight=\"bold\",y=1+1e-2)\n",
        "  index = 1\n",
        "  C = set(y_train)\n",
        "  for i in C:\n",
        "    plt.subplot(1,5,index)\n",
        "    plt.imshow(x_train[y_train==i][0].reshape(28,28),cmap='gray')\n",
        "    plt.title(\"Class \"+str(int(i)))\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "    index+=1\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(\"Training Set\\nReconstructed Image\",fontweight=\"bold\",y=1+1e-2)\n",
        "  index = 1\n",
        "  for i in C:\n",
        "    plt.subplot(1,5,index)\n",
        "    plt.imshow(x_train_pred[y_train==i][0].reshape(28,28),cmap='gray')\n",
        "    plt.title(\"Class \"+str(int(i)))\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "    index+=1\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(\"Validation Set\\nOriginal Image\",fontweight=\"bold\",y=1+1e-2)\n",
        "  index = 1\n",
        "  for i in C:\n",
        "    plt.subplot(1,5,index)\n",
        "    plt.imshow(x_val[y_val==i][0].reshape(28,28),cmap='gray')\n",
        "    plt.title(\"Class \"+str(int(i)))\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "    index+=1\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(\"Validation Set\\nReconstructed Image\",fontweight=\"bold\",y=1+1e-2)\n",
        "  index = 1\n",
        "  for i in C:\n",
        "    plt.subplot(1,5,index)\n",
        "    plt.imshow(x_val_pred[y_val==i][0].reshape(28,28),cmap='gray')\n",
        "    plt.title(\"Class \"+str(int(i)))\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "    index+=1\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  #Weight Visualization\n",
        "  plt.figure(figsize=(12.5,node1))\n",
        "  plt.suptitle(\"Weight Vizualization\",fontweight=\"bold\",y=1+1e-2)\n",
        "  weights = model.layers[0].get_weights()\n",
        "  index = 1\n",
        "  for i in range(node1):\n",
        "    plt.subplot(node1//4,4,index)\n",
        "    img = weights[0].T[i]\n",
        "    img /= np.sqrt(sum(img**2))\n",
        "    plt.imshow(img.reshape(28,28),cmap='gray')\n",
        "    plt.title(\"Hidden Node \"+str(int(i+1)))\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "    index+=1\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  #Encoder\n",
        "  model2 = Sequential()\n",
        "  model2.add(Dense(node1, input_dim=x_train.shape[1], activation='sigmoid', weights=model.layers[0].get_weights()))\n",
        "\n",
        "  x_train_compress = model2.predict(x_train)\n",
        "  x_val_compress = model2.predict(x_val)\n",
        "  x_test_compress = model2.predict(x_test)\n",
        "  return x_train_compress,x_val_compress,x_test_compress"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LJnV71IGffUf"
      },
      "outputs": [],
      "source": [
        "node1 = 64\n",
        "epochs = 1e5\n",
        "error_th = 1e-4\n",
        "optimizer = 'Adam'\n",
        "batch_size = 32\n",
        "tensorflow.keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)\n",
        "x_train_compress,x_val_compress,x_test_compress = autoencoder_1_layer(x_train,x_val,x_test,y_train,y_val,y_test,node1,optimizer,batch_size,error_th,epochs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "L9ga-GC1t2TZ"
      },
      "outputs": [],
      "source": [
        "#FCNN using Adam optimizer for classification.\n",
        "node1 = 30\n",
        "node2 = 20\n",
        "node3 = 10\n",
        "C = 5\n",
        "error_th = 1e-4\n",
        "p = 1\n",
        "\n",
        "optimizer  ='Adam'\n",
        "batch_size = 32\n",
        "tensorflow.keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)\n",
        "clf_3_layer(x_train_compress,y_train,x_val_compress,y_val,x_test_compress,y_test,\n",
        "            node1,node2,node3,C,optimizer,batch_size,error_th,p,epochs=1e5)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AhxV9CgKfUbs"
      },
      "source": [
        "##**`AutoEncoder with 3 Hidden Layers`**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XrmSTnvePXfX"
      },
      "outputs": [],
      "source": [
        "# Auto Encoder\n",
        "def autoencoder_3_layer(x_train,x_val,x_test,y_train,y_val,y_test,node1,node2,optimizer,batch_size,error_th,epochs=1e3):\n",
        "  model = Sequential()\n",
        "  model.add(Dense(node1, input_dim=x_train.shape[1], activation='sigmoid'))\n",
        "  model.add(Dense(node2, activation='sigmoid'))\n",
        "  model.add(Dense(node1, activation='sigmoid'))\n",
        "  model.add(Dense(x_train.shape[1], activation='linear'))\n",
        "\n",
        "  es = EarlyStopping(monitor='loss',mode='min',min_delta = error_th,verbose=1,patience=1)\n",
        "\n",
        "  model.compile(loss='mean_squared_error', optimizer=optimizer, metrics=['mean_squared_error'])\n",
        "  history = model.fit(x_train, x_train,validation_data = (x_val,x_val) ,epochs=int(epochs), batch_size=batch_size,callbacks=[es])\n",
        "\n",
        "  print(\"No of Epochs =\",len(history.history[\"loss\"]))\n",
        "  plt.figure(figsize=(12.5,8))\n",
        "  plt.suptitle(\"3 Hidden Layer\",fontweight=\"bold\",y=1+1e-2)\n",
        "  plt.subplot(1,1,1)\n",
        "  plt.plot(history.history['loss'],marker='.')\n",
        "  plt.plot(history.history['val_loss'],marker='.')\n",
        "  plt.title('Model loss')\n",
        "  plt.ylabel('Loss')\n",
        "  plt.xlabel('Epoch')\n",
        "  plt.legend(['Train', 'Validation'], loc='upper left')\n",
        "  plt.grid()\n",
        "  plt.tight_layout();plt.show()\n",
        "\n",
        "  print(\"Training Loss\\t=\",history.history['loss'][-1])\n",
        "  print(\"Validation Loss\\t=\",history.history['val_loss'][-1])\n",
        "\n",
        "  x_train_pred = model.predict(x_train)\n",
        "  x_val_pred   = model.predict(x_val)\n",
        "  x_test_pred  = model.predict(x_test)\n",
        "  test_loss = ((x_test-x_test_pred)**2).mean()\n",
        "  print(\"Test Loss\\t=\",test_loss)\n",
        "\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(\"Average Reconstruction Error vs Data\",fontweight=\"bold\",y=1+1e-2)\n",
        "  plt.subplot(1,1,1)\n",
        "  plt.bar(height = [history.history['loss'][-1],history.history['val_loss'][-1],test_loss],x = ['Training Data','Validation Data','Test Data'])\n",
        "  plt.ylabel('Average Reconstruction Error')\n",
        "  plt.xlabel('Data')\n",
        "  plt.grid()\n",
        "  plt.tight_layout();plt.show()\n",
        "\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(\"Training Set\\nOriginal Image\",fontweight=\"bold\",y=1+1e-2)\n",
        "  index = 1\n",
        "  C = set(y_train)\n",
        "  for i in C:\n",
        "    plt.subplot(1,5,index)\n",
        "    plt.imshow(x_train[y_train==i][0].reshape(28,28),cmap='gray')\n",
        "    plt.title(\"Class \"+str(int(i)))\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "    index+=1\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(\"Training Set\\nReconstructed Image\",fontweight=\"bold\",y=1+1e-2)\n",
        "  index = 1\n",
        "  for i in C:\n",
        "    plt.subplot(1,5,index)\n",
        "    plt.imshow(x_train_pred[y_train==i][0].reshape(28,28),cmap='gray')\n",
        "    plt.title(\"Class \"+str(int(i)))\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "    index+=1\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(\"Validation Set\\nOriginal Image\",fontweight=\"bold\",y=1+1e-2)\n",
        "  index = 1\n",
        "  for i in C:\n",
        "    plt.subplot(1,5,index)\n",
        "    plt.imshow(x_val[y_val==i][0].reshape(28,28),cmap='gray')\n",
        "    plt.title(\"Class \"+str(int(i)))\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "    index+=1\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(\"Validation Set\\nReconstructed Image\",fontweight=\"bold\",y=1+1e-2)\n",
        "  index = 1\n",
        "  for i in C:\n",
        "    plt.subplot(1,5,index)\n",
        "    plt.imshow(x_val_pred[y_val==i][0].reshape(28,28),cmap='gray')\n",
        "    plt.title(\"Class \"+str(int(i)))\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "    index+=1\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  #Weight Visualization\n",
        "  plt.figure(figsize=(12.5,node2))\n",
        "  plt.suptitle(\"Weight Vizualization\",fontweight=\"bold\",y=1+1e-2)\n",
        "  weights = model.layers[1].get_weights()\n",
        "  index = 1\n",
        "  for i in range(node2):\n",
        "    plt.subplot(node2//4,4,index)\n",
        "    img = weights[0].T[i]\n",
        "    img /= np.sqrt(sum(img**2))\n",
        "    plt.imshow(img.reshape(16,16),cmap='gray')\n",
        "    plt.title(\"Hidden Node \"+str(int(i+1)))\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "    index+=1\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  #Encoder\n",
        "  model2 = Sequential()\n",
        "  model2.add(Dense(node1, input_dim=x_train.shape[1], activation='sigmoid', weights=model.layers[0].get_weights()))\n",
        "  model2.add(Dense(node2, activation='sigmoid',weights=model.layers[1].get_weights()))\n",
        "\n",
        "  x_train_compress = model2.predict(x_train)\n",
        "  x_val_compress = model2.predict(x_val)\n",
        "  x_test_compress = model2.predict(x_test)\n",
        "  return x_train_compress,x_val_compress,x_test_compress"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cNn2dB98QgZ5"
      },
      "outputs": [],
      "source": [
        "node1 = 256\n",
        "node2 = 64\n",
        "epochs = 1e5\n",
        "error_th = 1e-4\n",
        "optimizer = 'Adam'\n",
        "batch_size = 32\n",
        "tensorflow.keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)\n",
        "x_train_compress,x_val_compress,x_test_compress = autoencoder_3_layer(x_train,x_val,x_test,y_train,y_val,y_test,node1,node2,optimizer,batch_size,error_th,epochs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TNgGr14pwAwe"
      },
      "outputs": [],
      "source": [
        "#FCNN using Adam optimizer for classification.\n",
        "node1 = 30\n",
        "node2 = 20\n",
        "node3 = 10\n",
        "C = 5\n",
        "error_th = 1e-4\n",
        "p = 1\n",
        "\n",
        "optimizer  ='Adam'\n",
        "batch_size = 32\n",
        "tensorflow.keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)\n",
        "clf_3_layer(x_train_compress,y_train,x_val_compress,y_val,x_test_compress,y_test,\n",
        "            node1,node2,node3,C,optimizer,batch_size,error_th,p,epochs=1e5)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cG7fnwoIq3zQ"
      },
      "source": [
        "##**`Denoising AutoEncoder with 1 Hidden Layers`**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "d4meXZi0nYgz"
      },
      "outputs": [],
      "source": [
        "class noiseLayer(tf.keras.layers.Layer):\n",
        "  def __init__(self,noise_pct,mean,std):\n",
        "    super(noiseLayer, self).__init__()\n",
        "    self.noise_pct = noise_pct\n",
        "    self.mean = mean\n",
        "    self.std  = std\n",
        "\n",
        "  def call(self, input):\n",
        "    mean = self.mean\n",
        "    std  = self.std\n",
        "    noise = np.random.normal(size = [784],loc = mean,scale = std)\n",
        "    #if np.random.rand()*100>noise_pct:\n",
        "    #  noise *= 0\n",
        "    index = list(range(len(noise)))\n",
        "    np.random.shuffle(index)\n",
        "    noise[index[int(self.noise_pct/100*len(noise)):]] = 0\n",
        "    noise = tf.convert_to_tensor(noise,dtype=np.float32)\n",
        "    return input + noise\n",
        "\n",
        "# Denoising Auto Encoder\n",
        "def denoising_autoencoder_1_layer(x_train,x_val,x_test,y_train,y_val,y_test,node1,optimizer,batch_size,error_th,noise_pct,epochs=1e3):\n",
        "  model = Sequential()\n",
        "  model.add(noiseLayer(noise_pct,mean=0,std=1/np.sqrt(784)))\n",
        "  model.add(Dense(node1, input_dim=x_train.shape[1], activation='sigmoid'))\n",
        "  model.add(Dense(x_train.shape[1], activation='linear'))\n",
        "\n",
        "  es = EarlyStopping(monitor='loss',mode='min',min_delta = error_th,verbose=1,patience=1)\n",
        "\n",
        "  model.compile(loss='mean_squared_error', optimizer=optimizer, metrics=['mean_squared_error'])\n",
        "  history = model.fit(x_train, x_train,validation_data = (x_val,x_val) ,epochs=int(epochs),\n",
        "                      batch_size=batch_size,callbacks=[es])\n",
        "\n",
        "  print(\"No of Epochs =\",len(history.history[\"loss\"]))\n",
        "  plt.figure(figsize=(12.5,8))\n",
        "  plt.suptitle(\"1 Hidden Layer\",fontweight=\"bold\",y=1+1e-2)\n",
        "  plt.subplot(1,1,1)\n",
        "  plt.plot(history.history['loss'],marker='.')\n",
        "  plt.plot(history.history['val_loss'],marker='.')\n",
        "  plt.title('Model loss')\n",
        "  plt.ylabel('Loss')\n",
        "  plt.xlabel('Epoch')\n",
        "  plt.legend(['Train', 'Validation'], loc='upper left')\n",
        "  plt.grid()\n",
        "  plt.tight_layout();plt.show()\n",
        "\n",
        "  print(\"Training Loss\\t=\",history.history['loss'][-1])\n",
        "  print(\"Validation Loss\\t=\",history.history['val_loss'][-1])\n",
        "\n",
        "  x_train_pred = model.predict(x_train)\n",
        "  x_val_pred   = model.predict(x_val)\n",
        "  x_test_pred  = model.predict(x_test)\n",
        "  test_loss = ((x_test-x_test_pred)**2).mean()\n",
        "  print(\"Test Loss\\t=\",test_loss)\n",
        "\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(\"Average Reconstruction Error vs Data\",fontweight=\"bold\",y=1+1e-2)\n",
        "  plt.subplot(1,1,1)\n",
        "  plt.bar(height = [history.history['loss'][-1],history.history['val_loss'][-1],test_loss],x = ['Training Data','Validation Data','Test Data'])\n",
        "  plt.ylabel('Average Reconstruction Error')\n",
        "  plt.xlabel('Data')\n",
        "  plt.grid()\n",
        "  plt.tight_layout();plt.show()\n",
        "\n",
        "\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(\"Training Set\\nOriginal Image\",fontweight=\"bold\",y=1+1e-2)\n",
        "  index = 1\n",
        "  C = set(y_train)\n",
        "  for i in C:\n",
        "    plt.subplot(1,5,index)\n",
        "    plt.imshow(x_train[y_train==i][0].reshape(28,28),cmap='gray')\n",
        "    plt.title(\"Class \"+str(int(i)))\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "    index+=1\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(\"Training Set\\nReconstructed Image\",fontweight=\"bold\",y=1+1e-2)\n",
        "  index = 1\n",
        "  for i in C:\n",
        "    plt.subplot(1,5,index)\n",
        "    plt.imshow(x_train_pred[y_train==i][0].reshape(28,28),cmap='gray')\n",
        "    plt.title(\"Class \"+str(int(i)))\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "    index+=1\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(\"Validation Set\\nOriginal Image\",fontweight=\"bold\",y=1+1e-2)\n",
        "  index = 1\n",
        "  for i in C:\n",
        "    plt.subplot(1,5,index)\n",
        "    plt.imshow(x_val[y_val==i][0].reshape(28,28),cmap='gray')\n",
        "    plt.title(\"Class \"+str(int(i)))\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "    index+=1\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle(\"Validation Set\\nReconstructed Image\",fontweight=\"bold\",y=1+1e-2)\n",
        "  index = 1\n",
        "  for i in C:\n",
        "    plt.subplot(1,5,index)\n",
        "    plt.imshow(x_val_pred[y_val==i][0].reshape(28,28),cmap='gray')\n",
        "    plt.title(\"Class \"+str(int(i)))\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "    index+=1\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  #Weight Visualization\n",
        "  plt.figure(figsize=(12.5,node1))\n",
        "  plt.suptitle(\"Weight Vizualization\",fontweight=\"bold\",y=1+1e-2)\n",
        "  weights = model.layers[1].get_weights()\n",
        "\n",
        "  index = 1\n",
        "  for i in range(8):#node1):\n",
        "    plt.subplot(node1//4,4,index)\n",
        "    img = weights[0].T[i]\n",
        "    img /= np.sqrt(sum(img**2))\n",
        "    plt.imshow(img.reshape(28,28),cmap='gray')\n",
        "    plt.title(\"Hidden Node \"+str(int(i+1)))\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "    index+=1\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  #Encoder\n",
        "  model2 = Sequential()\n",
        "  model.add(noiseLayer(noise_pct,mean=0,std=1/np.sqrt(784)))\n",
        "  model2.add(Dense(node1, input_dim=x_train.shape[1], activation='sigmoid', weights=model.layers[1].get_weights()))\n",
        "\n",
        "  x_train_compress = model2.predict(x_train)\n",
        "  x_val_compress = model2.predict(x_val)\n",
        "  x_test_compress = model2.predict(x_test)\n",
        "  return x_train_compress,x_val_compress,x_test_compress"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dIGOO9oBrDdB"
      },
      "outputs": [],
      "source": [
        "node1 = 64\n",
        "epochs = 1e5\n",
        "error_th = 1e-4\n",
        "optimizer = 'Adam'\n",
        "batch_size = 32\n",
        "noise_pct = 100\n",
        "tensorflow.keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)\n",
        "x_train_compress,x_val_compress,x_test_compress = denoising_autoencoder_1_layer(x_train,x_val,x_test,y_train,y_val,y_test,node1,optimizer,batch_size,error_th,noise_pct,epochs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8IlRzpkwr7oc"
      },
      "outputs": [],
      "source": [
        "#FCNN using Adam optimizer for classification.\n",
        "node1 = 30\n",
        "node2 = 20\n",
        "node3 = 10\n",
        "C = 5\n",
        "error_th = 1e-4\n",
        "p = 1\n",
        "\n",
        "optimizer  ='Adam'\n",
        "batch_size = 32\n",
        "tensorflow.keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)\n",
        "clf_3_layer(x_train_compress,y_train,x_val_compress,y_val,x_test_compress,y_test,\n",
        "            node1,node2,node3,C,optimizer,batch_size,error_th,p,epochs=1e5)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "node1 = 64\n",
        "epochs = 1e5\n",
        "error_th = 1e-4\n",
        "optimizer = 'Adam'\n",
        "batch_size = 32\n",
        "noise_pct = 40\n",
        "tensorflow.keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)\n",
        "x_train_compress,x_val_compress,x_test_compress = denoising_autoencoder_1_layer(x_train,x_val,x_test,y_train,y_val,y_test,node1,optimizer,batch_size,error_th,noise_pct,epochs)"
      ],
      "metadata": {
        "id": "3Gx6_nn0fkcu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#FCNN using Adam optimizer for classification.\n",
        "node1 = 30\n",
        "node2 = 20\n",
        "node3 = 10\n",
        "C = 5\n",
        "error_th = 1e-4\n",
        "p = 1\n",
        "\n",
        "optimizer  ='Adam'\n",
        "batch_size = 32\n",
        "tensorflow.keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)\n",
        "clf_3_layer(x_train_compress,y_train,x_val_compress,y_val,x_test_compress,y_test,\n",
        "            node1,node2,node3,C,optimizer,batch_size,error_th,p,epochs=1e5)"
      ],
      "metadata": {
        "id": "9vgHhEecfz1A"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}