ModelClean.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cb49d0c-c079-4381-8850-1b1d1eddd5ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "##ALL WE NEED IS STUFF BELOW HERE:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22832a9c-1d6d-4f8a-9034-182255c53291",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_12\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " encoder_inputs (InputLayer)    [(None, None)]       0           []                               \n",
      "                                                                                                  \n",
      " decoder_inputs (InputLayer)    [(None, None)]       0           []                               \n",
      "                                                                                                  \n",
      " encoder_embedding (Embedding)  (None, None, 256)    1325312     ['encoder_inputs[0][0]']         \n",
      "                                                                                                  \n",
      " decoder_embedding (Embedding)  (None, None, 256)    1325312     ['decoder_inputs[0][0]']         \n",
      "                                                                                                  \n",
      " encoder_lstm (LSTM)            [(None, 256),        525312      ['encoder_embedding[0][0]']      \n",
      "                                 (None, 256),                                                     \n",
      "                                 (None, 256)]                                                     \n",
      "                                                                                                  \n",
      " decoder_lstm (LSTM)            [(None, None, 256),  525312      ['decoder_embedding[0][0]',      \n",
      "                                 (None, 256),                     'encoder_lstm[0][1]',           \n",
      "                                 (None, 256)]                     'encoder_lstm[0][2]']           \n",
      "                                                                                                  \n",
      " decoder_dense (Dense)          (None, None, 5177)   1330489     ['decoder_lstm[0][0]']           \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 5,031,737\n",
      "Trainable params: 5,031,737\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/100\n",
      "49/49 [==============================] - 23s 454ms/step - loss: 2.1507 - accuracy: 0.7708 - val_loss: 0.7544 - val_accuracy: 0.8927\n",
      "Epoch 2/100\n",
      "49/49 [==============================] - 24s 481ms/step - loss: 1.3468 - accuracy: 0.7964 - val_loss: 0.6942 - val_accuracy: 0.9019\n",
      "Epoch 3/100\n",
      "49/49 [==============================] - 24s 497ms/step - loss: 1.2739 - accuracy: 0.8061 - val_loss: 0.6798 - val_accuracy: 0.9070\n",
      "Epoch 4/100\n",
      "49/49 [==============================] - 25s 516ms/step - loss: 1.2193 - accuracy: 0.8089 - val_loss: 0.6815 - val_accuracy: 0.9054\n",
      "Epoch 5/100\n",
      "49/49 [==============================] - 25s 507ms/step - loss: 1.1811 - accuracy: 0.8115 - val_loss: 0.6826 - val_accuracy: 0.9049\n",
      "Epoch 6/100\n",
      "49/49 [==============================] - 25s 507ms/step - loss: 1.1446 - accuracy: 0.8147 - val_loss: 0.6843 - val_accuracy: 0.9072\n",
      "Epoch 7/100\n",
      "49/49 [==============================] - 25s 506ms/step - loss: 1.1097 - accuracy: 0.8172 - val_loss: 0.6914 - val_accuracy: 0.9044\n",
      "Epoch 8/100\n",
      "49/49 [==============================] - 24s 488ms/step - loss: 1.0704 - accuracy: 0.8209 - val_loss: 0.6846 - val_accuracy: 0.9054\n",
      "Epoch 9/100\n",
      "49/49 [==============================] - 24s 494ms/step - loss: 1.0334 - accuracy: 0.8239 - val_loss: 5.1830 - val_accuracy: 0.5889\n",
      "Epoch 10/100\n",
      "49/49 [==============================] - 24s 499ms/step - loss: 1.0412 - accuracy: 0.8235 - val_loss: 0.6747 - val_accuracy: 0.9072\n",
      "Epoch 11/100\n",
      "49/49 [==============================] - 24s 494ms/step - loss: 0.9637 - accuracy: 0.8287 - val_loss: 0.6787 - val_accuracy: 0.9113\n",
      "Epoch 12/100\n",
      "49/49 [==============================] - 25s 505ms/step - loss: 0.9306 - accuracy: 0.8309 - val_loss: 0.6750 - val_accuracy: 0.9109\n",
      "Epoch 13/100\n",
      "49/49 [==============================] - 25s 505ms/step - loss: 0.8987 - accuracy: 0.8329 - val_loss: 0.6798 - val_accuracy: 0.9106\n",
      "Epoch 14/100\n",
      "49/49 [==============================] - 25s 503ms/step - loss: 0.8671 - accuracy: 0.8354 - val_loss: 0.6810 - val_accuracy: 0.9116\n",
      "Epoch 15/100\n",
      "49/49 [==============================] - 25s 510ms/step - loss: 0.8354 - accuracy: 0.8386 - val_loss: 0.6894 - val_accuracy: 0.9110\n",
      "Epoch 16/100\n",
      "49/49 [==============================] - 24s 487ms/step - loss: 0.8037 - accuracy: 0.8425 - val_loss: 0.6853 - val_accuracy: 0.9120\n",
      "Epoch 17/100\n",
      "49/49 [==============================] - 24s 486ms/step - loss: 0.7729 - accuracy: 0.8466 - val_loss: 0.6972 - val_accuracy: 0.9101\n",
      "Epoch 18/100\n",
      "49/49 [==============================] - 25s 503ms/step - loss: 0.7407 - accuracy: 0.8516 - val_loss: 0.6984 - val_accuracy: 0.9108\n",
      "Epoch 19/100\n",
      "49/49 [==============================] - 26s 522ms/step - loss: 0.7105 - accuracy: 0.8567 - val_loss: 0.6941 - val_accuracy: 0.9107\n",
      "Epoch 20/100\n",
      "49/49 [==============================] - 25s 517ms/step - loss: 0.6795 - accuracy: 0.8628 - val_loss: 0.7002 - val_accuracy: 0.9083\n",
      "Epoch 21/100\n",
      "49/49 [==============================] - 24s 500ms/step - loss: 0.6502 - accuracy: 0.8680 - val_loss: 0.7021 - val_accuracy: 0.9111\n",
      "Epoch 22/100\n",
      "49/49 [==============================] - 24s 491ms/step - loss: 0.6227 - accuracy: 0.8735 - val_loss: 0.7039 - val_accuracy: 0.9110\n",
      "Epoch 23/100\n",
      "49/49 [==============================] - 25s 502ms/step - loss: 0.5955 - accuracy: 0.8788 - val_loss: 0.7097 - val_accuracy: 0.9101\n",
      "Epoch 24/100\n",
      "49/49 [==============================] - 24s 495ms/step - loss: 0.5692 - accuracy: 0.8839 - val_loss: 0.7095 - val_accuracy: 0.9107\n",
      "Epoch 25/100\n",
      "49/49 [==============================] - 25s 511ms/step - loss: 0.5436 - accuracy: 0.8889 - val_loss: 0.7095 - val_accuracy: 0.9108\n",
      "Epoch 26/100\n",
      "49/49 [==============================] - 26s 525ms/step - loss: 0.5189 - accuracy: 0.8939 - val_loss: 0.7255 - val_accuracy: 0.9108\n",
      "Epoch 27/100\n",
      "49/49 [==============================] - 25s 509ms/step - loss: 0.4962 - accuracy: 0.8987 - val_loss: 0.7168 - val_accuracy: 0.9103\n",
      "Epoch 28/100\n",
      "49/49 [==============================] - 25s 509ms/step - loss: 0.4739 - accuracy: 0.9032 - val_loss: 0.7223 - val_accuracy: 0.9107\n",
      "Epoch 29/100\n",
      "49/49 [==============================] - 25s 510ms/step - loss: 0.4520 - accuracy: 0.9074 - val_loss: 0.7272 - val_accuracy: 0.9106\n",
      "Epoch 30/100\n",
      "49/49 [==============================] - 25s 520ms/step - loss: 0.4324 - accuracy: 0.9118 - val_loss: 0.7270 - val_accuracy: 0.9103\n",
      "Epoch 31/100\n",
      "49/49 [==============================] - 25s 515ms/step - loss: 0.4121 - accuracy: 0.9159 - val_loss: 0.7320 - val_accuracy: 0.9112\n",
      "Epoch 32/100\n",
      "49/49 [==============================] - 24s 501ms/step - loss: 0.3929 - accuracy: 0.9202 - val_loss: 0.7404 - val_accuracy: 0.9114\n",
      "Epoch 33/100\n",
      "49/49 [==============================] - 25s 514ms/step - loss: 0.3755 - accuracy: 0.9239 - val_loss: 0.7367 - val_accuracy: 0.9110\n",
      "Epoch 34/100\n",
      "49/49 [==============================] - 25s 515ms/step - loss: 0.3578 - accuracy: 0.9272 - val_loss: 0.7392 - val_accuracy: 0.9113\n",
      "Epoch 35/100\n",
      "49/49 [==============================] - 354s 7s/step - loss: 0.3419 - accuracy: 0.9305 - val_loss: 0.7448 - val_accuracy: 0.9110\n",
      "Epoch 36/100\n",
      "49/49 [==============================] - 25s 508ms/step - loss: 0.3254 - accuracy: 0.9340 - val_loss: 0.7509 - val_accuracy: 0.9115\n",
      "Epoch 37/100\n",
      "49/49 [==============================] - 25s 502ms/step - loss: 0.3116 - accuracy: 0.9366 - val_loss: 0.7522 - val_accuracy: 0.9112\n",
      "Epoch 38/100\n",
      "49/49 [==============================] - 25s 508ms/step - loss: 0.2968 - accuracy: 0.9395 - val_loss: 0.7516 - val_accuracy: 0.9118\n",
      "Epoch 39/100\n",
      "49/49 [==============================] - 25s 510ms/step - loss: 0.2832 - accuracy: 0.9424 - val_loss: 0.7578 - val_accuracy: 0.9112\n",
      "Epoch 40/100\n",
      "49/49 [==============================] - 26s 526ms/step - loss: 0.2703 - accuracy: 0.9446 - val_loss: 0.7609 - val_accuracy: 0.9105\n",
      "Epoch 41/100\n",
      "49/49 [==============================] - 25s 514ms/step - loss: 0.2580 - accuracy: 0.9473 - val_loss: 0.7600 - val_accuracy: 0.9112\n",
      "Epoch 42/100\n",
      "49/49 [==============================] - 24s 497ms/step - loss: 0.2467 - accuracy: 0.9494 - val_loss: 0.7616 - val_accuracy: 0.9110\n",
      "Epoch 43/100\n",
      "49/49 [==============================] - 25s 509ms/step - loss: 0.2356 - accuracy: 0.9516 - val_loss: 0.7617 - val_accuracy: 0.9112\n",
      "Epoch 44/100\n",
      "49/49 [==============================] - 25s 513ms/step - loss: 0.2255 - accuracy: 0.9538 - val_loss: 0.7652 - val_accuracy: 0.9118\n",
      "Epoch 45/100\n",
      "49/49 [==============================] - 25s 511ms/step - loss: 0.2156 - accuracy: 0.9554 - val_loss: 0.7726 - val_accuracy: 0.9116\n",
      "Epoch 46/100\n",
      "49/49 [==============================] - 25s 522ms/step - loss: 0.2057 - accuracy: 0.9573 - val_loss: 0.7759 - val_accuracy: 0.9126\n",
      "Epoch 47/100\n",
      "49/49 [==============================] - 25s 514ms/step - loss: 0.1968 - accuracy: 0.9591 - val_loss: 0.7733 - val_accuracy: 0.9116\n",
      "Epoch 48/100\n",
      "49/49 [==============================] - 24s 494ms/step - loss: 0.1884 - accuracy: 0.9608 - val_loss: 0.7735 - val_accuracy: 0.9123\n",
      "Epoch 49/100\n",
      "49/49 [==============================] - 26s 532ms/step - loss: 0.1800 - accuracy: 0.9624 - val_loss: 0.7838 - val_accuracy: 0.9119\n",
      "Epoch 50/100\n",
      "49/49 [==============================] - 25s 503ms/step - loss: 0.1722 - accuracy: 0.9639 - val_loss: 0.7867 - val_accuracy: 0.9117\n",
      "Epoch 51/100\n",
      "49/49 [==============================] - 26s 525ms/step - loss: 0.1652 - accuracy: 0.9651 - val_loss: 0.7867 - val_accuracy: 0.9115\n",
      "Epoch 52/100\n",
      "49/49 [==============================] - 26s 522ms/step - loss: 0.1585 - accuracy: 0.9662 - val_loss: 0.7845 - val_accuracy: 0.9121\n",
      "Epoch 53/100\n",
      "49/49 [==============================] - 25s 518ms/step - loss: 0.1519 - accuracy: 0.9678 - val_loss: 0.7880 - val_accuracy: 0.9120\n",
      "Epoch 54/100\n",
      "49/49 [==============================] - 541s 11s/step - loss: 0.1460 - accuracy: 0.9687 - val_loss: 0.7883 - val_accuracy: 0.9121\n",
      "Epoch 55/100\n",
      "49/49 [==============================] - 24s 494ms/step - loss: 0.1398 - accuracy: 0.9699 - val_loss: 0.7964 - val_accuracy: 0.9122\n",
      "Epoch 56/100\n",
      "49/49 [==============================] - 24s 494ms/step - loss: 0.1341 - accuracy: 0.9713 - val_loss: 0.7955 - val_accuracy: 0.9125\n",
      "Epoch 57/100\n",
      "49/49 [==============================] - 25s 501ms/step - loss: 0.1294 - accuracy: 0.9720 - val_loss: 0.8005 - val_accuracy: 0.9124\n",
      "Epoch 58/100\n",
      "49/49 [==============================] - 25s 502ms/step - loss: 0.1238 - accuracy: 0.9732 - val_loss: 0.8053 - val_accuracy: 0.9122\n",
      "Epoch 59/100\n",
      "49/49 [==============================] - 25s 517ms/step - loss: 0.1186 - accuracy: 0.9743 - val_loss: 0.8068 - val_accuracy: 0.9124\n",
      "Epoch 60/100\n",
      "49/49 [==============================] - 27s 546ms/step - loss: 0.1138 - accuracy: 0.9751 - val_loss: 0.8084 - val_accuracy: 0.9121\n",
      "Epoch 61/100\n",
      "49/49 [==============================] - 26s 526ms/step - loss: 0.1095 - accuracy: 0.9760 - val_loss: 0.8127 - val_accuracy: 0.9121\n",
      "Epoch 62/100\n",
      "49/49 [==============================] - 25s 504ms/step - loss: 0.1053 - accuracy: 0.9767 - val_loss: 0.8138 - val_accuracy: 0.9125\n",
      "Epoch 63/100\n",
      "49/49 [==============================] - 25s 512ms/step - loss: 0.1013 - accuracy: 0.9775 - val_loss: 0.8150 - val_accuracy: 0.9121\n",
      "Epoch 64/100\n",
      "49/49 [==============================] - 25s 513ms/step - loss: 0.0975 - accuracy: 0.9784 - val_loss: 0.8201 - val_accuracy: 0.9122\n",
      "Epoch 65/100\n",
      "49/49 [==============================] - 25s 521ms/step - loss: 0.0936 - accuracy: 0.9790 - val_loss: 0.8218 - val_accuracy: 0.9122\n",
      "Epoch 66/100\n",
      "49/49 [==============================] - 26s 525ms/step - loss: 0.0902 - accuracy: 0.9799 - val_loss: 0.8285 - val_accuracy: 0.9126\n",
      "Epoch 67/100\n",
      "49/49 [==============================] - 26s 529ms/step - loss: 0.0867 - accuracy: 0.9804 - val_loss: 0.8295 - val_accuracy: 0.9123\n",
      "Epoch 68/100\n",
      "49/49 [==============================] - 25s 520ms/step - loss: 0.0834 - accuracy: 0.9810 - val_loss: 0.8363 - val_accuracy: 0.9121\n",
      "Epoch 69/100\n",
      "49/49 [==============================] - 26s 535ms/step - loss: 0.0811 - accuracy: 0.9815 - val_loss: 0.8343 - val_accuracy: 0.9122\n",
      "Epoch 70/100\n",
      "49/49 [==============================] - 26s 529ms/step - loss: 0.0779 - accuracy: 0.9822 - val_loss: 0.8312 - val_accuracy: 0.9126\n",
      "Epoch 71/100\n",
      "49/49 [==============================] - 26s 526ms/step - loss: 0.0755 - accuracy: 0.9826 - val_loss: 0.8411 - val_accuracy: 0.9125\n",
      "Epoch 72/100\n",
      "49/49 [==============================] - 25s 515ms/step - loss: 0.0726 - accuracy: 0.9833 - val_loss: 0.8462 - val_accuracy: 0.9123\n",
      "Epoch 73/100\n",
      "49/49 [==============================] - 25s 518ms/step - loss: 0.0697 - accuracy: 0.9838 - val_loss: 0.8501 - val_accuracy: 0.9124\n",
      "Epoch 74/100\n",
      "49/49 [==============================] - 25s 510ms/step - loss: 0.0678 - accuracy: 0.9842 - val_loss: 0.8477 - val_accuracy: 0.9132\n",
      "Epoch 75/100\n",
      "49/49 [==============================] - 25s 510ms/step - loss: 0.0650 - accuracy: 0.9846 - val_loss: 0.8537 - val_accuracy: 0.9121\n",
      "Epoch 76/100\n",
      "49/49 [==============================] - 25s 509ms/step - loss: 0.0628 - accuracy: 0.9850 - val_loss: 0.8542 - val_accuracy: 0.9126\n",
      "Epoch 77/100\n",
      "49/49 [==============================] - 25s 521ms/step - loss: 0.0614 - accuracy: 0.9854 - val_loss: 0.8553 - val_accuracy: 0.9129\n",
      "Epoch 78/100\n",
      "49/49 [==============================] - 26s 526ms/step - loss: 0.0588 - accuracy: 0.9858 - val_loss: 0.8642 - val_accuracy: 0.9121\n",
      "Epoch 79/100\n",
      "49/49 [==============================] - 26s 537ms/step - loss: 0.0568 - accuracy: 0.9862 - val_loss: 0.8725 - val_accuracy: 0.9119\n",
      "Epoch 80/100\n",
      "49/49 [==============================] - 26s 531ms/step - loss: 0.0554 - accuracy: 0.9865 - val_loss: 0.8633 - val_accuracy: 0.9121\n",
      "Epoch 81/100\n",
      "49/49 [==============================] - 26s 538ms/step - loss: 0.0536 - accuracy: 0.9866 - val_loss: 0.8674 - val_accuracy: 0.9119\n",
      "Epoch 82/100\n",
      "49/49 [==============================] - 26s 528ms/step - loss: 0.0522 - accuracy: 0.9872 - val_loss: 0.8675 - val_accuracy: 0.9124\n",
      "Epoch 83/100\n",
      "49/49 [==============================] - 25s 514ms/step - loss: 0.0500 - accuracy: 0.9872 - val_loss: 0.8689 - val_accuracy: 0.9120\n",
      "Epoch 84/100\n",
      "49/49 [==============================] - 26s 528ms/step - loss: 0.0494 - accuracy: 0.9873 - val_loss: 0.8754 - val_accuracy: 0.9121\n",
      "Epoch 85/100\n",
      "49/49 [==============================] - 26s 535ms/step - loss: 0.0474 - accuracy: 0.9876 - val_loss: 0.8850 - val_accuracy: 0.9115\n",
      "Epoch 86/100\n",
      "49/49 [==============================] - 25s 520ms/step - loss: 0.0464 - accuracy: 0.9877 - val_loss: 0.8821 - val_accuracy: 0.9114\n",
      "Epoch 87/100\n",
      "49/49 [==============================] - 25s 513ms/step - loss: 0.0452 - accuracy: 0.9879 - val_loss: 0.8876 - val_accuracy: 0.9118\n",
      "Epoch 88/100\n",
      "49/49 [==============================] - 25s 500ms/step - loss: 0.0440 - accuracy: 0.9880 - val_loss: 0.8929 - val_accuracy: 0.9119\n",
      "Epoch 89/100\n",
      "49/49 [==============================] - 25s 517ms/step - loss: 0.0431 - accuracy: 0.9882 - val_loss: 0.8912 - val_accuracy: 0.9117\n",
      "Epoch 90/100\n",
      "49/49 [==============================] - 26s 533ms/step - loss: 0.0419 - accuracy: 0.9884 - val_loss: 0.8936 - val_accuracy: 0.9120\n",
      "Epoch 91/100\n",
      "49/49 [==============================] - 25s 517ms/step - loss: 0.0410 - accuracy: 0.9886 - val_loss: 0.8897 - val_accuracy: 0.9123\n",
      "Epoch 92/100\n",
      "49/49 [==============================] - 24s 497ms/step - loss: 0.0399 - accuracy: 0.9889 - val_loss: 0.9001 - val_accuracy: 0.9126\n",
      "Epoch 93/100\n",
      "49/49 [==============================] - 25s 504ms/step - loss: 0.0390 - accuracy: 0.9889 - val_loss: 0.8993 - val_accuracy: 0.9120\n",
      "Epoch 94/100\n",
      "49/49 [==============================] - 26s 527ms/step - loss: 0.0386 - accuracy: 0.9890 - val_loss: 0.9020 - val_accuracy: 0.9121\n",
      "Epoch 95/100\n",
      "49/49 [==============================] - 26s 537ms/step - loss: 0.0374 - accuracy: 0.9892 - val_loss: 0.8987 - val_accuracy: 0.9121\n",
      "Epoch 96/100\n",
      "49/49 [==============================] - 25s 512ms/step - loss: 0.0368 - accuracy: 0.9892 - val_loss: 0.9041 - val_accuracy: 0.9125\n",
      "Epoch 97/100\n",
      "49/49 [==============================] - 26s 532ms/step - loss: 0.0360 - accuracy: 0.9894 - val_loss: 0.9075 - val_accuracy: 0.9124\n",
      "Epoch 98/100\n",
      "49/49 [==============================] - 25s 513ms/step - loss: 0.0352 - accuracy: 0.9894 - val_loss: 0.9079 - val_accuracy: 0.9124\n",
      "Epoch 99/100\n",
      "49/49 [==============================] - 25s 511ms/step - loss: 0.0350 - accuracy: 0.9894 - val_loss: 0.9133 - val_accuracy: 0.9128\n",
      "Epoch 100/100\n",
      "49/49 [==============================] - 25s 520ms/step - loss: 0.0344 - accuracy: 0.9893 - val_loss: 0.9183 - val_accuracy: 0.9127\n",
      "1/1 [==============================] - 0s 143ms/step\n",
      "1/1 [==============================] - 0s 133ms/step\n",
      "1/1 [==============================] - 0s 8ms/step\n",
      "1/1 [==============================] - 0s 7ms/step\n",
      "1/1 [==============================] - 0s 8ms/step\n",
      "1/1 [==============================] - 0s 7ms/step\n",
      "1/1 [==============================] - 0s 7ms/step\n",
      "1/1 [==============================] - 0s 36ms/step\n",
      "1/1 [==============================] - 0s 9ms/step\n",
      "1/1 [==============================] - 0s 10ms/step\n",
      "Input: start Ah doh know wat ah doing end\n",
      "Output: I do not know what I am doing\n",
      "Output: i do not know what i am doing.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import Input, LSTM, Dense, Embedding\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "import pickle\n",
    "\n",
    "\n",
    "# Suppress TensorFlow warnings (optional)\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'  # Suppress all logs except errors\n",
    "\n",
    "# Load your CSV file containing Trini slang and their English meanings\n",
    "data = pd.read_csv('/Users/aidenramgoolam/utschack/caribe_tec_to_eng_dataset.csv')\n",
    "\n",
    "# Remove the \"tec:\" prefix from the 'input' column if present\n",
    "data['input'] = data['input'].str.replace('tec:', '', regex=False)\n",
    "\n",
    "# Add start and end tokens with spaces\n",
    "data['input'] = data['input'].apply(lambda x: 'start ' + x + ' end')\n",
    "data['target'] = data['target'].apply(lambda x: 'start ' + x + ' end')\n",
    "\n",
    "# Combine all texts for building the vocabulary\n",
    "all_texts = list(data['input']) + list(data['target'])\n",
    "\n",
    "# Initialize and fit the tokenizer\n",
    "tokenizer = Tokenizer(filters='', oov_token='<OOV>')\n",
    "tokenizer.fit_on_texts(all_texts)\n",
    "\n",
    "# Save the tokenizer for future use\n",
    "with open('tokenizer.pkl', 'wb') as f:\n",
    "    pickle.dump(tokenizer, f)\n",
    "\n",
    "# Convert texts to sequences\n",
    "input_sequences = tokenizer.texts_to_sequences(data['input'])\n",
    "target_sequences = tokenizer.texts_to_sequences(data['target'])\n",
    "\n",
    "# Determine the maximum sequence length\n",
    "max_sequence_length = max(max(len(seq) for seq in input_sequences),\n",
    "                          max(len(seq) for seq in target_sequences))\n",
    "\n",
    "# Pad the sequences\n",
    "input_sequences = pad_sequences(input_sequences, maxlen=max_sequence_length, padding='post')\n",
    "target_sequences = pad_sequences(target_sequences, maxlen=max_sequence_length, padding='post')\n",
    "\n",
    "# Prepare decoder input and output sequences\n",
    "decoder_input_sequences = target_sequences[:, :-1]\n",
    "decoder_output_sequences = target_sequences[:, 1:]\n",
    "\n",
    "# Define vocabulary size\n",
    "vocab_size = len(tokenizer.word_index) + 1  # +1 for padding token\n",
    "\n",
    "# Build the model with Embedding layers\n",
    "embedding_dim = 256  # You can adjust this value\n",
    "latent_dim = 256     # Latent dimensionality of the encoding space\n",
    "\n",
    "# Encoder\n",
    "encoder_inputs = Input(shape=(None,), name='encoder_inputs')\n",
    "encoder_embedding = Embedding(input_dim=vocab_size, output_dim=embedding_dim, name='encoder_embedding')\n",
    "encoder_embed = encoder_embedding(encoder_inputs)\n",
    "encoder_lstm = LSTM(latent_dim, return_state=True, name='encoder_lstm')\n",
    "encoder_outputs, state_h, state_c = encoder_lstm(encoder_embed)\n",
    "encoder_states = [state_h, state_c]\n",
    "\n",
    "# Decoder\n",
    "decoder_inputs = Input(shape=(None,), name='decoder_inputs')\n",
    "decoder_embedding = Embedding(input_dim=vocab_size, output_dim=embedding_dim, name='decoder_embedding')\n",
    "decoder_embed = decoder_embedding(decoder_inputs)\n",
    "decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True, name='decoder_lstm')\n",
    "decoder_outputs, _, _ = decoder_lstm(decoder_embed, initial_state=encoder_states)\n",
    "decoder_dense = Dense(vocab_size, activation='softmax', name='decoder_dense')\n",
    "decoder_outputs = decoder_dense(decoder_outputs)\n",
    "\n",
    "# Define the model\n",
    "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
    "\n",
    "# Compile the model with sparse categorical crossentropy\n",
    "model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "# Summarize the model\n",
    "model.summary()\n",
    "\n",
    "# Prepare the target data for training (decoder outputs need to be 2D arrays)\n",
    "decoder_output_data = decoder_output_sequences.reshape(-1, decoder_output_sequences.shape[1], 1)\n",
    "\n",
    "# Train the model\n",
    "model.fit([input_sequences, decoder_input_sequences], decoder_output_data,\n",
    "          batch_size=64,\n",
    "          epochs=100,\n",
    "          validation_split=0.2)\n",
    "\n",
    "# Save the model\n",
    "model.save('seq2seq_trini_translation.h5')\n",
    "\n",
    "# Define sampling models for inference\n",
    "\n",
    "# Encoder model\n",
    "encoder_model = Model(encoder_inputs, encoder_states)\n",
    "\n",
    "# Decoder model\n",
    "# Define inputs for the decoder's initial states\n",
    "decoder_state_input_h = Input(shape=(latent_dim,), name='decoder_state_input_h')\n",
    "decoder_state_input_c = Input(shape=(latent_dim,), name='decoder_state_input_c')\n",
    "decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
    "\n",
    "# Get embeddings of decoder input\n",
    "decoder_inputs_single = Input(shape=(1,), name='decoder_inputs_single')  # Input shape is (batch_size, sequence_length)\n",
    "decoder_embed_single = decoder_embedding(decoder_inputs_single)\n",
    "\n",
    "# Run the decoder LSTM\n",
    "decoder_outputs, state_h, state_c = decoder_lstm(decoder_embed_single, initial_state=decoder_states_inputs)\n",
    "decoder_states = [state_h, state_c]\n",
    "decoder_outputs = decoder_dense(decoder_outputs)\n",
    "\n",
    "# Define the decoder model\n",
    "decoder_model = Model(\n",
    "    [decoder_inputs_single] + decoder_states_inputs,\n",
    "    [decoder_outputs] + decoder_states)\n",
    "\n",
    "# Reverse-lookup token index to decode sequences back to words\n",
    "reverse_word_index = {idx: word for word, idx in tokenizer.word_index.items()}\n",
    "\n",
    "# Decoding function\n",
    "def decode_sequence(input_seq):\n",
    "    # Encode the input as state vectors\n",
    "    states_value = encoder_model.predict(input_seq)\n",
    "\n",
    "    # Generate empty target sequence of length 1 with only the start token\n",
    "    target_seq = np.array([tokenizer.word_index['start']])\n",
    "\n",
    "    # Initialize variables\n",
    "    stop_condition = False\n",
    "    decoded_sentence = ''\n",
    "    while not stop_condition:\n",
    "        # Predict the next token\n",
    "        output_tokens, h, c = decoder_model.predict(\n",
    "            [target_seq] + states_value)\n",
    "\n",
    "        # Sample the token with the highest probability\n",
    "        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
    "        sampled_word = reverse_word_index.get(sampled_token_index, '<OOV>')\n",
    "\n",
    "        # Exit condition: either hit max length or find the stop token\n",
    "        if (sampled_word == 'end' or\n",
    "           len(decoded_sentence.split()) > max_sequence_length):\n",
    "            stop_condition = True\n",
    "        else:\n",
    "            if sampled_word != '<OOV>':\n",
    "                decoded_sentence += ' ' + sampled_word\n",
    "\n",
    "            # Update the target sequence (length 1)\n",
    "            target_seq = np.array([sampled_token_index])\n",
    "\n",
    "            # Update states\n",
    "            states_value = [h, c]\n",
    "\n",
    "    return decoded_sentence.strip()\n",
    "\n",
    "# Test the model with an example input sentence\n",
    "input_sentence = \"Ah doh know wat ah doing\"\n",
    "\n",
    "# Add start and end tokens\n",
    "input_sentence = 'start ' + input_sentence + ' end'\n",
    "\n",
    "# Convert the sentence to a sequence\n",
    "input_sequence = tokenizer.texts_to_sequences([input_sentence])\n",
    "\n",
    "# Pad the sequence\n",
    "input_sequence = pad_sequences(input_sequence, maxlen=max_sequence_length, padding='post')\n",
    "\n",
    "# Decode the input sequence\n",
    "decoded_sentence = decode_sequence(input_sequence)\n",
    "\n",
    "print(f\"Input: {input_sentence}\")\n",
    "print(\"Output: I do not know what I am doing\")\n",
    "print(f\"Output: {decoded_sentence}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "daf6df5d-5b78-49fe-850f-fe8fd1cf8184",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 15ms/step\n",
      "1/1 [==============================] - 0s 11ms/step\n",
      "1/1 [==============================] - 0s 8ms/step\n",
      "1/1 [==============================] - 0s 8ms/step\n",
      "1/1 [==============================] - 0s 8ms/step\n",
      "1/1 [==============================] - 0s 8ms/step\n",
      "1/1 [==============================] - 0s 8ms/step\n",
      "1/1 [==============================] - 0s 9ms/step\n",
      "Input: start Yuh does rel maco end\n",
      "Output: you like to eavesdrop a lot\n"
     ]
    }
   ],
   "source": [
    "# Test the model with an example input sentence\n",
    "input_sentence1 = \"Yuh does rel maco\"\n",
    "\n",
    "# Add start and end tokens\n",
    "input_sentence1 = 'start ' + input_sentence1 + ' end'\n",
    "\n",
    "# Convert the sentence to a sequence\n",
    "input_sequence1 = tokenizer.texts_to_sequences([input_sentence1])\n",
    "\n",
    "# Pad the sequence\n",
    "input_sequence1 = pad_sequences(input_sequence1, maxlen=max_sequence_length, padding='post')\n",
    "\n",
    "# Decode the input sequence\n",
    "decoded_sentence1 = decode_sequence(input_sequence1)\n",
    "\n",
    "print(f\"Input: {input_sentence1}\")\n",
    "# print(\"Output: I do not know what I am doing\")\n",
    "print(f\"Output: {decoded_sentence1}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8af70a97-4406-4974-8ca6-f5b30c1c6253",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 19ms/step\n",
      "1/1 [==============================] - 0s 12ms/step\n",
      "1/1 [==============================] - 0s 8ms/step\n",
      "1/1 [==============================] - 0s 9ms/step\n",
      "Input: start My g end\n",
      "Output: my friend\n"
     ]
    }
   ],
   "source": [
    "def translate(input_str):\n",
    "\n",
    "    # Test the model with an example input sentence\n",
    "    input_sentence2 = input_str\n",
    "\n",
    "    # Add start and end tokens\n",
    "    input_sentence2 = 'start ' + input_sentence2 + ' end'\n",
    "\n",
    "    # Convert the sentence to a sequence\n",
    "    input_sequence2 = tokenizer.texts_to_sequences([input_sentence2])\n",
    "\n",
    "    # Pad the sequence\n",
    "    input_sequence2 = pad_sequences(input_sequence2, maxlen=max_sequence_length, padding='post')\n",
    "\n",
    "    # Decode the input sequence\n",
    "    decoded_sentence2 = decode_sequence(input_sequence2)\n",
    "    print(decoded_sentence2)\n",
    "    return decoded_sentence2\n",
    "    # print(\"Output: I do not know what I am doing\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3dbe1a2a-acc3-46a7-8c7c-7bf1611b6e63",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}