final_train9.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "source": [
    "import os\n",
    "import sys\n",
    "import tensorflow as tf\n",
    "import scipy.io.wavfile as wav\n",
    "import glob\n",
    "import numpy as np\n",
    "import six\n",
    "from six.moves import xrange as range\n",
    "import json\n",
    "from python_speech_features import mfcc\n",
    "from sklearn.model_selection import train_test_split\n",
    "import mlflow\n",
    "import math"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "# tf.debugging.set_log_device_placement(True)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "source": [
    "# Constants \n",
    "SPACE_TOKEN = '<space>'\n",
    "SPACE_INDEX = 0\n",
    "FIRST_INDEX = 1\n",
    "FEAT_MASK_VALUE = 1e+10\n",
    "\n",
    "# Some configs\n",
    "num_features = 13\n",
    "num_units = 100\n",
    "num_classes = 285 + 1 # 285(including space) + blamk label = 286\n",
    "\n",
    "# Hyper-parameters\n",
    "num_epochs = 100\n",
    "num_layers = 1\n",
    "batch_size = 2\n",
    "initial_learning_rate = 0.005\n",
    "momentum = 0.9"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "source": [
    "# Loading the data\n",
    "file_path = glob.glob('../data/train/wav/orginal*/*.wav')\n",
    "file_path = file_path[28:32]\n",
    "audio_list = []\n",
    "fs_list = []\n",
    "\n",
    "for file_name in file_path:\n",
    "    fs, audio = wav.read(file_name)\n",
    "    audio_list.append(audio)\n",
    "    fs_list.append(fs)\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "source": [
    "\n",
    "# Create a dataset composed of data with variable lengths\n",
    "inputs_list = []\n",
    "for index in range(len(audio_list)):\n",
    "    input_val = mfcc(audio_list[index], samplerate=fs_list[index])\n",
    "    input_val = (input_val - np.mean(input_val)) / np.std(input_val)\n",
    "    inputs_list.append(input_val)\n",
    "\n",
    "# Transform in 3D Array\n",
    "inputs_list\n",
    "     \n",
    "# train_inputs = tf.ragged.constant([i for i in inputs_list], dtype=np.float32)\n",
    "# tr= tf.RaggedTensor.from_row_lengths(train_inputs ,row_lengths= [i,i])\n",
    "# train_seq_len = tf.cast(tr , tf.int32)\n",
    "# train_seq_len\n",
    "# #train_inputs = train_inputs.to_tensor(default_value=FEAT_MASK_VALUE)"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "metadata": {},
     "execution_count": 5
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "source": [
    "with open('../data/train.json', 'r', encoding='UTF-8') as label_file:\n",
    "    labels = json.load(label_file)\n",
    "with open('../data/language_model.json', 'r', encoding='UTF-8') as language_file:\n",
    "    alphabets = json.load(language_file)\n"
   ],
   "outputs": [
    {
     "output_type": "error",
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '../data/labels.json'",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-b8f00dc514ed>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'../data/labels.json'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'r'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'UTF-8'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mlabel_file\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'../data/language_model.json'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'r'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'UTF-8'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mlanguage_file\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0malphabets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlanguage_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '../data/labels.json'"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "source": [
    "# Reading Targets\n",
    "original_list = []\n",
    "targets_list = []\n",
    "\n",
    "for path in file_path:\n",
    "    file_name = path[:-4].split('wav')[1][1:]\n",
    "    # Read Label\n",
    "    label = labels[file_name]\n",
    "    original = \" \".join(label.strip().split(' '))\n",
    "    original_list.append(original)\n",
    "    # print(original)\n",
    "    target = original.replace(' ', '  ')\n",
    "    # print('step-1. ',target)\n",
    "    target = target.split(' ')\n",
    "    # print('step-2. ', target)\n",
    "    # Adding blank label\n",
    "    target = np.hstack([SPACE_TOKEN if x == '' else list(x) for x in target])\n",
    "    # print('step-3. ', target)\n",
    "    # Transform char into index\n",
    "    target = np.asarray([alphabets['char_to_num'][x] for x in target])\n",
    "    # print('step-4. ', target)\n",
    "    targets_list.append(target)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "source": [
    "# Creating sparse representation to feed the placeholder\n",
    "train_targets = tf.ragged.constant([i for i in targets_list], dtype=np.int32)\n",
    "train_targets_len = tf.cast(train_targets.row_lengths(), tf.int32)\n",
    "train_targets = train_targets.to_sparse()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "source": [
    "train_targets.shape"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "TensorShape([4, 38])"
      ]
     },
     "metadata": {},
     "execution_count": 9
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "source": [
    "# Split Training and Validation sets\n",
    "# train_inputs, val_inputs = train_inputs[:800], train_inputs[800:]\n",
    "# train_seq_len, val_seq_len = train_seq_len[:800], train_seq_len[800:]\n",
    "# train_targets, val_targets = tf.sparse.slice(train_targets, start=[0, 0], size=[800, 163]), tf.sparse.slice(train_targets, start=[800, 0], size=[200, 163])\n",
    "# train_targets_len, val_targets_len = train_targets_len[:800], train_targets_len[800:]\n",
    "\n",
    "# train_inputs, val_inputs = train_inputs[:5], train_inputs[5:]\n",
    "# train_seq_len, val_seq_len = train_seq_len[:5], train_seq_len[5:]\n",
    "# train_targets, val_targets = tf.sparse.slice(train_targets, start=[0, 0], size=[\n",
    "#                                              5, 73]), tf.sparse.slice(train_targets, start=[5, 0], size=[5, 73])\n",
    "# train_targets_len, val_targets_len = train_targets_len[:5], train_targets_len[5:]\n",
    "\n",
    "val_inputs, val_targets, val_seq_len, val_targets_len = train_inputs, train_targets, train_seq_len, train_targets_len\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "source": [
    "class CTCLossLayer(tf.keras.layers.Layer):\n",
    "    def call(self, inputs):\n",
    "        labels = inputs[0]\n",
    "        logits = inputs[1]\n",
    "        label_len = inputs[2]\n",
    "        logit_len = inputs[3]\n",
    "\n",
    "        logits_trans = tf.transpose(logits, (1,0,2))\n",
    "        label_len = tf.reshape(label_len, (-1,))\n",
    "        logit_len = tf.reshape(logit_len, (-1,))\n",
    "        loss = tf.reduce_mean(tf.nn.ctc_loss(labels, logits_trans, label_len, logit_len, blank_index=-1))\n",
    "        # define loss here instead of in compile\n",
    "        self.add_loss(loss)\n",
    "\n",
    "        # Decode\n",
    "        decoded, _ = tf.nn.ctc_greedy_decoder(logits_trans, logit_len)\n",
    "\n",
    "        # Inaccuracy: label error rate\n",
    "        ler = tf.reduce_mean(tf.edit_distance(tf.cast(decoded[0], tf.int32),labels))\n",
    "        self.add_metric(ler, name='ler', aggregation='mean')\n",
    "\n",
    "        return logits\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "source": [
    "# Defining Training Cells\n",
    "cells = []\n",
    "for _ in range(num_layers):\n",
    "    cell = tf.keras.layers.LSTMCell(num_units)\n",
    "    cells.append(cell)\n",
    "\n",
    "stack = tf.keras.layers.StackedRNNCells(cells)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "source": [
    "# Definning Input Parameters\n",
    "input_feature = tf.keras.layers.Input((None, num_features), name='input_feature')\n",
    "input_label = tf.keras.layers.Input((None,), dtype=tf.int32, sparse=True, name='input_label')\n",
    "input_feature_len = tf.keras.layers.Input((1,), dtype=tf.int32, name='input_feature_len')\n",
    "input_label_len =tf.keras.layers.Input((1,), dtype=tf.int32, name='input_label_len')\n",
    "\n",
    "layer_masking = tf.keras.layers.Masking(FEAT_MASK_VALUE)(input_feature)\n",
    "layer_rnn = tf.keras.layers.RNN(stack, return_sequences=True)(layer_masking)\n",
    "# layer_drop = tf.keras.layers.Dropout(0.2, seed=42)(layer_rnn)\n",
    "layer_output = tf.keras.layers.Dense(num_classes, kernel_initializer=tf.keras.initializers.TruncatedNormal(0.0,0.1), bias_initializer='zeros', name='logit')(layer_rnn)\n",
    "\n",
    "layer_loss = CTCLossLayer()([input_label, layer_output, input_label_len, input_feature_len])\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "source": [
    "# Create models for training and prediction\n",
    "model_train = tf.keras.models.Model(inputs=[input_feature, input_label, input_feature_len, input_label_len],\n",
    "            outputs=layer_loss)\n",
    "\n",
    "model_predict = tf.keras.models.Model(inputs=input_feature, outputs=layer_output)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "source": [
    "# Compile Training Model with selected optimizer\n",
    "optimizer = tf.keras.optimizers.SGD(initial_learning_rate, momentum)\n",
    "model_train.compile(optimizer=optimizer)\n",
    "\n",
    "# Training, Our y is already defined so no need\n",
    "try:\n",
    "    experiment_id = mlflow.create_experiment(\"Stacked RNN(LSTM): 50 Cells\")\n",
    "    experiment = mlflow.get_experiment(experiment_id)\n",
    "except mlflow.exceptions.MlflowException:\n",
    "    experiment = mlflow.get_experiment_by_name(\"Stacked RNN(LSTM): 50 Cells\")\n",
    "\n",
    "mlflow.tensorflow.autolog()\n",
    "model_train.fit(x=[train_inputs, train_targets, train_seq_len, train_targets_len], y=None,\n",
    "                validation_data=([val_inputs, val_targets, val_seq_len, val_targets_len], None),\n",
    "                batch_size=batch_size, epochs=num_epochs)\n"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "2021/08/11 11:30:05 INFO mlflow.utils.autologging_utils: Created MLflow autologging run with ID '7ec1556c7e64453a94be3426dc3985f7', which will track hyperparameters, performance metrics, model artifacts, and lineage information for the current tensorflow workflow\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/100\n",
      "2/2 [==============================] - 11s 5s/step - loss: 1809.0740 - ler: 2.0046 - val_loss: 248.9128 - val_ler: 0.8165\n",
      "Epoch 2/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 304.3698 - ler: 0.8165 - val_loss: 383.4819 - val_ler: 0.8184\n",
      "Epoch 3/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 324.9138 - ler: 0.8005 - val_loss: 177.8816 - val_ler: 0.7978\n",
      "Epoch 4/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 162.0337 - ler: 0.7689 - val_loss: 147.5302 - val_ler: 0.7755\n",
      "Epoch 5/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 143.6159 - ler: 0.7689 - val_loss: 135.6169 - val_ler: 0.7558\n",
      "Epoch 6/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 133.5174 - ler: 0.7654 - val_loss: 125.9642 - val_ler: 0.7921\n",
      "Epoch 7/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 124.8065 - ler: 0.7921 - val_loss: 118.6310 - val_ler: 0.7676\n",
      "Epoch 8/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 120.3079 - ler: 0.7745 - val_loss: 119.8236 - val_ler: 0.7540\n",
      "Epoch 9/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 120.8973 - ler: 0.7475 - val_loss: 114.0546 - val_ler: 0.7208\n",
      "Epoch 10/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 113.0667 - ler: 0.7374 - val_loss: 111.9184 - val_ler: 0.7606\n",
      "Epoch 11/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 114.2562 - ler: 0.7676 - val_loss: 116.2967 - val_ler: 0.7374\n",
      "Epoch 12/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 115.7376 - ler: 0.7278 - val_loss: 110.2590 - val_ler: 0.7707\n",
      "Epoch 13/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 110.2649 - ler: 0.7677 - val_loss: 107.6720 - val_ler: 0.7514\n",
      "Epoch 14/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 109.7617 - ler: 0.7418 - val_loss: 106.2204 - val_ler: 0.7616\n",
      "Epoch 15/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 111.2095 - ler: 0.7485 - val_loss: 105.5136 - val_ler: 0.7339\n",
      "Epoch 16/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 105.1996 - ler: 0.7339 - val_loss: 103.8662 - val_ler: 0.7430\n",
      "Epoch 17/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 103.3219 - ler: 0.7430 - val_loss: 99.5751 - val_ler: 0.7676\n",
      "Epoch 18/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 99.8402 - ler: 0.7676 - val_loss: 98.1566 - val_ler: 0.7745\n",
      "Epoch 19/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 97.8315 - ler: 0.7676 - val_loss: 94.0043 - val_ler: 0.7308\n",
      "Epoch 20/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 94.0318 - ler: 0.7176 - val_loss: 91.5319 - val_ler: 0.7160\n",
      "Epoch 21/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 90.9541 - ler: 0.7226 - val_loss: 90.3021 - val_ler: 0.7226\n",
      "Epoch 22/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 90.2624 - ler: 0.7226 - val_loss: 88.8348 - val_ler: 0.7160\n",
      "Epoch 23/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 89.8764 - ler: 0.7187 - val_loss: 87.2676 - val_ler: 0.7256\n",
      "Epoch 24/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 88.3731 - ler: 0.7077 - val_loss: 85.2014 - val_ler: 0.7073\n",
      "Epoch 25/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 84.6840 - ler: 0.7073 - val_loss: 84.6697 - val_ler: 0.7034\n",
      "Epoch 26/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 83.9632 - ler: 0.7103 - val_loss: 81.2714 - val_ler: 0.7142\n",
      "Epoch 27/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 81.4940 - ler: 0.7212 - val_loss: 78.8717 - val_ler: 0.6941\n",
      "Epoch 28/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 78.8729 - ler: 0.6941 - val_loss: 77.6533 - val_ler: 0.6815\n",
      "Epoch 29/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 78.1831 - ler: 0.6815 - val_loss: 75.5470 - val_ler: 0.6706\n",
      "Epoch 30/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 76.2473 - ler: 0.6706 - val_loss: 73.8952 - val_ler: 0.6772\n",
      "Epoch 31/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 73.7043 - ler: 0.6772 - val_loss: 73.8379 - val_ler: 0.7153\n",
      "Epoch 32/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 74.9176 - ler: 0.7069 - val_loss: 72.5948 - val_ler: 0.6740\n",
      "Epoch 33/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 72.4297 - ler: 0.6737 - val_loss: 71.0789 - val_ler: 0.6475\n",
      "Epoch 34/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 71.1257 - ler: 0.6475 - val_loss: 69.5663 - val_ler: 0.6194\n",
      "Epoch 35/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 69.1665 - ler: 0.6221 - val_loss: 68.3023 - val_ler: 0.6189\n",
      "Epoch 36/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 68.6731 - ler: 0.6452 - val_loss: 66.1114 - val_ler: 0.6273\n",
      "Epoch 37/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 66.3814 - ler: 0.6412 - val_loss: 62.4959 - val_ler: 0.5972\n",
      "Epoch 38/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 62.6682 - ler: 0.6242 - val_loss: 59.6515 - val_ler: 0.6063\n",
      "Epoch 39/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 59.7251 - ler: 0.5835 - val_loss: 58.6825 - val_ler: 0.5411\n",
      "Epoch 40/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 57.9375 - ler: 0.5302 - val_loss: 56.4016 - val_ler: 0.5403\n",
      "Epoch 41/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 55.9581 - ler: 0.5319 - val_loss: 54.8987 - val_ler: 0.5188\n",
      "Epoch 42/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 55.0935 - ler: 0.5105 - val_loss: 51.4644 - val_ler: 0.5216\n",
      "Epoch 43/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 51.4116 - ler: 0.5413 - val_loss: 49.3221 - val_ler: 0.5045\n",
      "Epoch 44/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 49.2330 - ler: 0.4844 - val_loss: 47.8355 - val_ler: 0.4632\n",
      "Epoch 45/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 47.9949 - ler: 0.4563 - val_loss: 47.0598 - val_ler: 0.4660\n",
      "Epoch 46/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 47.0582 - ler: 0.4952 - val_loss: 44.0693 - val_ler: 0.4496\n",
      "Epoch 47/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 44.5974 - ler: 0.4298 - val_loss: 42.7668 - val_ler: 0.4187\n",
      "Epoch 48/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 42.7652 - ler: 0.4201 - val_loss: 40.8360 - val_ler: 0.3751\n",
      "Epoch 49/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 40.7196 - ler: 0.3913 - val_loss: 40.2006 - val_ler: 0.4162\n",
      "Epoch 50/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 39.3367 - ler: 0.3831 - val_loss: 37.7335 - val_ler: 0.3657\n",
      "Epoch 51/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 37.7852 - ler: 0.3723 - val_loss: 36.3647 - val_ler: 0.3208\n",
      "Epoch 52/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 36.6733 - ler: 0.3110 - val_loss: 34.0471 - val_ler: 0.3135\n",
      "Epoch 53/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 34.3230 - ler: 0.3315 - val_loss: 32.8048 - val_ler: 0.3091\n",
      "Epoch 54/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 32.5107 - ler: 0.3021 - val_loss: 30.6552 - val_ler: 0.2356\n",
      "Epoch 55/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 30.6129 - ler: 0.2422 - val_loss: 29.5035 - val_ler: 0.2272\n",
      "Epoch 56/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 29.4314 - ler: 0.2619 - val_loss: 28.0127 - val_ler: 0.2331\n",
      "Epoch 57/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 28.8717 - ler: 0.2510 - val_loss: 28.5591 - val_ler: 0.2236\n",
      "Epoch 58/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 29.6021 - ler: 0.1771 - val_loss: 28.2658 - val_ler: 0.2073\n",
      "Epoch 59/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 28.4103 - ler: 0.2201 - val_loss: 25.7407 - val_ler: 0.2425\n",
      "Epoch 60/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 26.0892 - ler: 0.2494 - val_loss: 24.0179 - val_ler: 0.1917\n",
      "Epoch 61/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 23.9885 - ler: 0.1917 - val_loss: 22.6336 - val_ler: 0.1369\n",
      "Epoch 62/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 22.6214 - ler: 0.1501 - val_loss: 21.3220 - val_ler: 0.1273\n",
      "Epoch 63/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 21.2523 - ler: 0.1543 - val_loss: 19.9570 - val_ler: 0.1447\n",
      "Epoch 64/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 19.9631 - ler: 0.1543 - val_loss: 18.7904 - val_ler: 0.1128\n",
      "Epoch 65/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 18.7552 - ler: 0.1128 - val_loss: 18.0154 - val_ler: 0.0962\n",
      "Epoch 66/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 18.1099 - ler: 0.0962 - val_loss: 17.2288 - val_ler: 0.0892\n",
      "Epoch 67/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 17.0945 - ler: 0.0629 - val_loss: 16.2381 - val_ler: 0.0629\n",
      "Epoch 68/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 16.1864 - ler: 0.0695 - val_loss: 15.4842 - val_ler: 0.0791\n",
      "Epoch 69/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 15.2930 - ler: 0.0857 - val_loss: 14.5696 - val_ler: 0.0928\n",
      "Epoch 70/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 14.5192 - ler: 0.0928 - val_loss: 13.9146 - val_ler: 0.0665\n",
      "Epoch 71/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 13.9298 - ler: 0.0730 - val_loss: 13.3192 - val_ler: 0.0538\n",
      "Epoch 72/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 13.1870 - ler: 0.0634 - val_loss: 12.6602 - val_ler: 0.0538\n",
      "Epoch 73/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 12.6057 - ler: 0.0604 - val_loss: 12.0881 - val_ler: 0.0437\n",
      "Epoch 74/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 12.0492 - ler: 0.0437 - val_loss: 11.6816 - val_ler: 0.0371\n",
      "Epoch 75/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 11.5950 - ler: 0.0371 - val_loss: 11.1589 - val_ler: 0.0371\n",
      "Epoch 76/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 11.0803 - ler: 0.0564 - val_loss: 10.6782 - val_ler: 0.0564\n",
      "Epoch 77/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 10.6436 - ler: 0.0564 - val_loss: 10.3340 - val_ler: 0.0306\n",
      "Epoch 78/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 10.2780 - ler: 0.0306 - val_loss: 9.8520 - val_ler: 0.0371\n",
      "Epoch 79/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 9.7752 - ler: 0.0371 - val_loss: 9.4483 - val_ler: 0.0371\n",
      "Epoch 80/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 9.3970 - ler: 0.0371 - val_loss: 9.1077 - val_ler: 0.0306\n",
      "Epoch 81/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 9.0549 - ler: 0.0306 - val_loss: 8.7787 - val_ler: 0.0306\n",
      "Epoch 82/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 8.6968 - ler: 0.0306 - val_loss: 8.4218 - val_ler: 0.0306\n",
      "Epoch 83/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 8.3789 - ler: 0.0306 - val_loss: 8.1528 - val_ler: 0.0306\n",
      "Epoch 84/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 8.1040 - ler: 0.0306 - val_loss: 7.8759 - val_ler: 0.0306\n",
      "Epoch 85/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 7.8504 - ler: 0.0306 - val_loss: 7.5820 - val_ler: 0.0306\n",
      "Epoch 86/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 7.5462 - ler: 0.0306 - val_loss: 7.3341 - val_ler: 0.0306\n",
      "Epoch 87/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 7.3088 - ler: 0.0306 - val_loss: 7.0848 - val_ler: 0.0306\n",
      "Epoch 88/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 7.0346 - ler: 0.0306 - val_loss: 6.8569 - val_ler: 0.0306\n",
      "Epoch 89/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 6.8350 - ler: 0.0306 - val_loss: 6.6389 - val_ler: 0.0306\n",
      "Epoch 90/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 6.6085 - ler: 0.0167 - val_loss: 6.4352 - val_ler: 0.0167\n",
      "Epoch 91/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 6.3930 - ler: 0.0167 - val_loss: 6.2341 - val_ler: 0.0167\n",
      "Epoch 92/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 6.2126 - ler: 0.0167 - val_loss: 6.0549 - val_ler: 0.0167\n",
      "Epoch 93/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 6.0185 - ler: 0.0306 - val_loss: 5.8784 - val_ler: 0.0167\n",
      "Epoch 94/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 5.8532 - ler: 0.0167 - val_loss: 5.8180 - val_ler: 0.0167\n",
      "Epoch 95/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 5.8345 - ler: 0.0167 - val_loss: 5.7982 - val_ler: 0.0167\n",
      "Epoch 96/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 5.7533 - ler: 0.0167 - val_loss: 5.5339 - val_ler: 0.0167\n",
      "Epoch 97/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 5.5360 - ler: 0.0167 - val_loss: 5.4108 - val_ler: 0.0167\n",
      "Epoch 98/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 5.4465 - ler: 0.0167 - val_loss: 5.3669 - val_ler: 0.0000e+00\n",
      "Epoch 99/100\n",
      "2/2 [==============================] - 4s 2s/step - loss: 5.3939 - ler: 0.0000e+00 - val_loss: 5.1152 - val_ler: 0.0000e+00\n",
      "Epoch 100/100\n",
      "2/2 [==============================] - 3s 2s/step - loss: 5.0762 - ler: 0.0000e+00 - val_loss: 4.9377 - val_ler: 0.0000e+00\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "WARNING:absl:Function `_wrapped_model` contains input name(s) args_0 with unsupported characters which will be renamed to args_0_5 in the SavedModel.\n",
      "WARNING:absl:Found untraced functions such as lstm_cell_layer_call_and_return_conditional_losses, lstm_cell_layer_call_fn, lstm_cell_layer_call_fn, lstm_cell_layer_call_and_return_conditional_losses, lstm_cell_layer_call_and_return_conditional_losses while saving (showing 5 of 5). These functions will not be directly callable after loading.\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "INFO:tensorflow:Assets written to: C:\\Users\\milky\\AppData\\Local\\Temp\\tmpzobwoqy2\\model\\data\\model\\assets\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "INFO:tensorflow:Assets written to: C:\\Users\\milky\\AppData\\Local\\Temp\\tmpzobwoqy2\\model\\data\\model\\assets\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x2a9b126b220>"
      ]
     },
     "metadata": {},
     "execution_count": 15
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "source": [
    "# Decoding\n",
    "print('Original:')\n",
    "print(original_list[0])\n",
    "print(original_list[1])\n",
    "print(original_list[2])\n",
    "print(original_list[3])\n",
    "print('Decoded:')\n",
    "\n",
    "\n",
    "# train_inputs = tf.ragged.constant([i for i in inputs_list[:6]], dtype=np.float32)\n",
    "# train_seq_len = tf.cast(train_inputs.row_lengths(), tf.int32)\n",
    "# train_inputs = train_inputs.to_tensor(default_value=FEAT_MASK_VALUE)\n",
    "\n",
    "decoded, _ = tf.nn.ctc_greedy_decoder(tf.transpose(\n",
    "    model_predict.predict(train_inputs), (1, 0, 2)), train_seq_len)\n",
    "\n",
    "d = tf.sparse.to_dense(decoded[0], default_value=-1).numpy()\n",
    "str_decoded = [''.join([alphabets['num_to_char'][str(x)]\n",
    "                       for x in np.asarray(row) if x != -1]) for row in d]\n",
    "\n",
    "# print('decoded',str_decoded)\n",
    "for s in str_decoded:\n",
    "    # Replacing blank label to none\n",
    "    # s = s.replace(chr(ord('z') + 1), '')\n",
    "    # Replacing space label to space\n",
    "    s = s.replace(alphabets['num_to_char']['0'], ' ')\n",
    "    print(s)\n"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Original:\n",
      "ሰውዬው ሚስቱ ን በጣም እንደሚ ወድ በ ለ ሆሳ ስ ነገራት\n",
      "ዛሬ የ በላ ና ት ቀይ ወጥ ሁ ርጥ ያለች ና ት\n",
      "አ በ በ ለ ዶክተሩ ሁለመና ዬን ነው የሚያመ ኝ ብሎ ነገረው\n",
      "የ ቻይና ህዝብ ሁ ልቆ መሳፍርት የ ለውም\n",
      "Decoded:\n",
      "ሰውዬው ሚስቱ ን በጣም እንደሚ ወድ በ ለ ሆሳ ስ ነገራት\n",
      "ዛሬ የ በላ ና ት ቀይ ወጥ ሁ ርጥ ያለች ና ት\n",
      "አ በ በ ለ ዶክተሩ ሁለመና ዬን ነው የሚያመ ኝ ብሎ ነገረው\n",
      "የ ቻይና ህዝብ ሁ ልቆ መሳፍርት የ ለውም\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "source": [
    "alphabets['num_to_char']['0']"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'<space>'"
      ]
     },
     "metadata": {},
     "execution_count": 17
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.9.6",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.9.6 64-bit ('tf-gpu': conda)"
  },
  "interpreter": {
   "hash": "9e09945fa8d15c93b0c117548153c35f8a8702e830281d3d09dfb9256247df3a"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}