Prediction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "import pyarabic.araby as araby\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical\n",
    "import farasa\n",
    "from farasa.segmenter import FarasaSegmenter \n",
    "import unicodedata\n",
    "import torch\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Embedding, Bidirectional, LSTM, Dense, Dropout, TimeDistributed, BatchNormalization\n",
    "from tensorflow.keras.optimizers import Adamax\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.activations import relu,linear\n",
    "\n",
    "from gensim.models.word2vec import LineSentence\n",
    "from gensim.models import FastText"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_data(file_path):\n",
    "    \"\"\"\n",
    "    Read the contents of the file located at file_path \n",
    "    and append each line to the list data\n",
    "    \"\"\"\n",
    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
    "        data = file.readlines()\n",
    "\n",
    "        # remove '\\n' from each line\n",
    "        data = [line.strip() for line in data]\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_pickle_file(file_path):\n",
    "    \"\"\"\n",
    "    Read the contents of the pickle file located at file_path \n",
    "    and append each line to the list data\n",
    "    \"\"\"\n",
    "    with open(file_path, 'rb') as file:\n",
    "        data = pickle.load(file)\n",
    "    return data\n",
    "\n",
    "def save_words_in_file(path, words, permission='w'):\n",
    "    \"\"\"\n",
    "    Save the words in the file located at path \n",
    "    \"\"\"\n",
    "    with open(path, permission, encoding='utf-8') as file:\n",
    "            for word in words:\n",
    "                file.write(word + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36\n",
      "{'ح', 'ى', 'و', 'ا', 'ظ', 'ن', 'ج', 'د', 'خ', 'ث', 'ر', 'س', 'ئ', 'ع', 'ه', 'آ', 'ط', 'ز', 'م', 'ذ', 'ق', 'غ', 'ف', 'ي', 'ص', 'ض', 'ت', 'ب', 'ل', 'ك', 'إ', 'ء', 'ؤ', 'أ', 'ش', 'ة'}\n"
     ]
    }
   ],
   "source": [
    "# set for arabic letters\n",
    "arabic_letters = set(read_pickle_file(\"./Delivery/arabic_letters.pickle\"))\n",
    "\n",
    "print(len(arabic_letters))\n",
    "print(arabic_letters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n",
      "{'َ', 'ٍ', 'ٌ', 'ُ', 'ْ', 'ً', 'ِ', 'ّ'}\n"
     ]
    }
   ],
   "source": [
    "# set for arabic letters\n",
    "diacritics = set(read_pickle_file(\"./Delivery/diacritics.pickle\"))\n",
    "\n",
    "print(len(diacritics))\n",
    "print(diacritics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove diacritics\n",
    "def remove_diacritics(text):\n",
    "    text = araby.strip_tashkeel(text)\n",
    "    return text\n",
    "\n",
    "# Remove any letters not found in set arabic_letters and not found in set diacritics\n",
    "def remove_non_arabic(text):\n",
    "    text = re.sub(r'[^\\s' + ''.join(arabic_letters) + ''.join(diacritics) + ']', '', text)\n",
    "    return text\n",
    "\n",
    "def input_preprocessing_text(text):\n",
    "    # Correct most common errors on word like repetetion of harakats, or tanween before alef\n",
    "    text = araby.autocorrect(text)\n",
    "\n",
    "    # Remove any non-Arabic letters\n",
    "    text = remove_non_arabic(text)\n",
    "\n",
    "    # Remove diacritics\n",
    "    text = remove_diacritics(text)\n",
    "\n",
    "    # Tokenize\n",
    "    text = araby.tokenize(text)\n",
    "\n",
    "    return text\n",
    "\n",
    "def save_tokenized_input(text,path=\"./generatedFiles/training/tokenized_input.txt\", permission='w'):\n",
    "    words = input_preprocessing_text(text)\n",
    "    save_words_in_file(path, words, permission)\n",
    "    \n",
    "\n",
    "def save_gold_output(text,path=\"./generatedFiles/training/gold_output.txt\", permission='w'):\n",
    "    # Remove any non-Arabic letters and extra spaces\n",
    "    text = remove_non_arabic(text)\n",
    "\n",
    "    # Tokenize\n",
    "    text = araby.tokenize(text)\n",
    "\n",
    "    save_words_in_file(path, text, permission)\n",
    "\n",
    "\n",
    "def is_not_arabic_diacritic(char):\n",
    "   category = unicodedata.category(char)\n",
    "   return not (category == 'Mn' or category == 'Mc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2024-01-02 17:45:44,149 - farasapy_logger - WARNING]: Be careful with large lines as they may break on interactive mode. You may switch to Standalone mode for such cases.\n"
     ]
    }
   ],
   "source": [
    "segmenter = FarasaSegmenter(interactive=True) # The default behaviour\n",
    "\n",
    "def get_seg_tags(word):                 # word = \"wAlktAb\"\n",
    "    segments = segmenter.segment(word)  # segments will be a list: [\"w\", \"Al\", \"ktAb\"]\n",
    "    segments = segments.split('+')\n",
    "    seg_tags = []\n",
    "    for segment in segments:\n",
    "        if len(segment) == 1:\n",
    "            seg_tags.append(\"S\")\n",
    "        else:\n",
    "            seg_tags.append(\"B\")  # First letter\n",
    "            seg_tags.extend(\"M\" * (len(segment) - 2))  # Middle letters\n",
    "            seg_tags.append(\"E\")  # Last letter\n",
    "    return segments, seg_tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_size = 40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the tokenizer_char model\n",
    "with open('./generatedFiles/test/tokenizer_char.pickle', 'rb') as handle:\n",
    "    tokenizer_char = pickle.load(handle)\n",
    "\n",
    "# Load the sequences_char model\n",
    "with open('./generatedFiles/test/sequences_char.pickle', 'rb') as handle:\n",
    "    sequences_char = pickle.load(handle)\n",
    "\n",
    "# Load the char_embeddings array from a pickle file\n",
    "with open('./generatedFiles/test/char_embeddings.pickle', 'rb') as handle:\n",
    "    char_embeddings = pickle.load(handle)\n",
    "\n",
    "# Load the char_features array from a pickle file\n",
    "with open('./generatedFiles/test/char_features.pickle', 'rb') as handle:\n",
    "    char_features = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the tokenizer_tags model\n",
    "with open('./generatedFiles/test/tokenizer_tags.pickle', 'rb') as handle:\n",
    "    tokenizer_tags = pickle.load(handle)\n",
    "\n",
    "# Load the tags_embeddings array in a pickle file\n",
    "with open('./generatedFiles/test/tags_embeddings.pickle', 'rb') as handle:\n",
    "    tags_embeddings = pickle.load(handle)\n",
    "\n",
    "# Load the sequences_char model\n",
    "with open('./generatedFiles/test/sequences_tags.pickle', 'rb') as handle:\n",
    "    sequences_tags = pickle.load(handle)\n",
    "\n",
    "# Load the tags_features array in a pickle file\n",
    "with open('./generatedFiles/test/tags_features.pickle', 'rb') as handle:\n",
    "    tags_features = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_before_preprocessing = read_data(\"./dataset/test.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(test_data_before_preprocessing)):\n",
    "    save_tokenized_input(test_data_before_preprocessing[i], path=\"./generatedFiles/test/test_tokenized_input.txt\", permission='a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_tokenized_input = read_data(\"./generatedFiles/test/test_tokenized_input.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(test_tokenized_input)):\n",
    "    test_segments, test_seg_tags = get_seg_tags(test_tokenized_input[i])\n",
    "    with open('./generatedFiles/test/test_input_segments.txt', 'a', encoding='utf-8') as file:\n",
    "        for tag in test_seg_tags:\n",
    "            file.write(tag)\n",
    "        file.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_input_segments = read_data(\"./generatedFiles/test/test_input_segments.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_map(file_path, number_of_keys=2):\n",
    "    \"\"\"\n",
    "    Read the contents of the file located at file_path \n",
    "    and append to the dictionary prior_feature\n",
    "    \"\"\"\n",
    "    prior_feature = {}\n",
    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
    "        for line in file:\n",
    "            key, value = line.strip().split(':')\n",
    "            key = key.strip()\n",
    "            value = value.strip()\n",
    "            key = key[1:-1].split(',')\n",
    "            value = value[1:-1].split(',')\n",
    "            if number_of_keys == 2:\n",
    "                key = (key[0][1:-1], key[1][2:-1], int(key[2]))\n",
    "            else:\n",
    "                key = (key[0][1:-1], key[1][2:-1], int(key[2]), int(key[3]))\n",
    "            \n",
    "            value = [int(i) for i in value]\n",
    "            prior_feature[key] = value\n",
    "    return prior_feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "prior_feature = read_map('./generatedFiles/training/prior_feature.txt', 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(417359, 120)\n"
     ]
    }
   ],
   "source": [
    "test_char_features_vector=[]\n",
    "test_tag_features_vector=[]\n",
    "test_prior_features_vector=[]\n",
    "test_embeddings = []\n",
    "for i in range(len(test_tokenized_input)):\n",
    "    for j in range(len(test_tokenized_input[i])):    \n",
    "        char_index = tokenizer_char.word_index.get(test_tokenized_input[i][j])\n",
    "        test_char_features_vector= char_embeddings[char_index]\n",
    "        if (len(test_tokenized_input[i]) != len(test_input_segments[i])):\n",
    "            test_input_segments[i] = \"S\" * (len(test_tokenized_input[i]) - len(test_input_segments[i])) + test_input_segments[i]\n",
    "        tag_index = tokenizer_tags.word_index.get(test_input_segments[i][j].lower())\n",
    "        test_tag_features_vector= tags_embeddings[tag_index]\n",
    "        test_prior_features_vector= (prior_feature[(test_tokenized_input[i], test_tokenized_input[i][j], j)]) if (test_tokenized_input[i], test_tokenized_input[i][j], j) in prior_feature else [1, 1, 1, 1, 1, 1, 1, 1]\n",
    "        # pad the prior feature vector with zeros to have the same length as the other features\n",
    "        test_prior_features_vector = np.pad(test_prior_features_vector, (0, 32), 'constant')\n",
    "        # concatenate the 3 features vectors to have a matrix of 3 columns\n",
    "        # test_embeddings.append(np.vstack((test_char_features_vector, test_tag_features_vector, test_prior_features_vector)))\n",
    "        # word_embedding = model1.wv[test_tokenized_input[i]]\n",
    "        test_embeddings.append(np.concatenate((test_char_features_vector, test_tag_features_vector, test_prior_features_vector)))\n",
    "\n",
    "test_embeddings = np.array(test_embeddings)\n",
    "\n",
    "print(test_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pad the embeddings to be of size 418000\n",
    "test_embeddings = np.pad(test_embeddings, ((0, 418000 - test_embeddings.shape[0]), (0, 0)), 'constant')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(418000, 120)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_embeddings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(418, 1000, 120)\n"
     ]
    }
   ],
   "source": [
    "test_embeddings_reshape = test_embeddings[:418000]\n",
    "test_embeddings_reshape = test_embeddings_reshape.reshape((-1, 1000, 120))\n",
    "test_embeddings_reshape = np.array(test_embeddings_reshape)\n",
    "print(test_embeddings_reshape.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From c:\\Users\\hlaha\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\keras\\src\\backend.py:1398: The name tf.executing_eagerly_outside_functions is deprecated. Please use tf.compat.v1.executing_eagerly_outside_functions instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# load the model\n",
    "model = tf.keras.models.load_model('./generatedFiles/model_shakkala.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14/14 [==============================] - 10s 471ms/step\n"
     ]
    }
   ],
   "source": [
    "# predict the test dataset\n",
    "predictions = model.predict(test_embeddings_reshape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(418, 1000, 15)\n"
     ]
    }
   ],
   "source": [
    "print(predictions.shape)\n",
    "predictions = predictions.reshape(-1, 15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[9.96443689e-01 3.66844215e-05 7.48699880e-04 4.39207543e-05\n",
      "  9.42222076e-04 5.09472593e-05 6.08946779e-04 1.76945454e-04\n",
      "  5.69241936e-04 2.79082233e-05 3.04720288e-05 3.11689000e-05\n",
      "  9.55952928e-05 2.91507749e-05 1.64438941e-04]\n",
      " [1.26982059e-05 6.93961738e-07 2.64693226e-05 3.46447860e-06\n",
      "  9.11959869e-05 3.71064857e-06 9.99723494e-01 1.67853466e-06\n",
      "  1.16646879e-06 2.62588799e-07 5.43247097e-06 1.24023450e-06\n",
      "  5.20431854e-07 2.14748070e-06 1.25671853e-04]\n",
      " [9.97247159e-01 7.11207176e-05 2.60321773e-04 2.22980161e-04\n",
      "  3.29572911e-04 9.70155525e-05 8.79902916e-04 3.03097295e-05\n",
      "  6.32464362e-05 1.31812403e-05 3.28825154e-05 2.45149677e-05\n",
      "  2.82575129e-05 4.41194134e-05 6.55433338e-04]\n",
      " [1.24083990e-05 7.47032402e-07 9.99879241e-01 1.53134752e-05\n",
      "  1.20202358e-05 1.62061710e-07 2.28520548e-05 1.05064305e-06\n",
      "  3.51975757e-07 4.44376064e-06 2.08391812e-05 1.80776078e-06\n",
      "  5.67607003e-07 3.37026449e-07 2.77936433e-05]\n",
      " [3.92807488e-05 1.54813472e-06 2.93963039e-05 6.49212379e-05\n",
      "  1.91576582e-05 5.43786291e-06 9.98587847e-01 3.32888249e-06\n",
      "  9.42668612e-06 4.08466212e-06 6.09324597e-05 7.35307549e-05\n",
      "  9.98603809e-07 5.08917810e-06 1.09511451e-03]\n",
      " [9.99352872e-01 1.75733821e-05 2.36068354e-05 9.89049367e-05\n",
      "  6.59573425e-06 1.28080092e-05 6.77986463e-05 1.67852158e-05\n",
      "  1.19623692e-05 1.42164790e-05 7.20985327e-06 1.44722035e-05\n",
      "  1.37974139e-05 8.18093667e-06 3.33265692e-04]\n",
      " [8.69030464e-06 3.56028380e-07 9.99415159e-01 5.52586971e-06\n",
      "  5.36891966e-05 2.97326721e-07 4.56355929e-06 5.60011131e-06\n",
      "  9.27443239e-07 1.80261959e-05 3.97572818e-04 4.20761398e-06\n",
      "  1.96747965e-06 7.20825767e-07 8.25944735e-05]\n",
      " [2.73454680e-06 3.10914061e-06 1.02211989e-03 3.01649288e-06\n",
      "  2.56204876e-05 5.86120314e-06 7.82615007e-06 6.82272457e-06\n",
      "  5.45328885e-06 5.49506658e-05 4.10067005e-04 2.61222194e-05\n",
      "  1.24251337e-05 1.24005464e-05 9.98401582e-01]\n",
      " [5.22426990e-06 7.33177046e-07 9.99168396e-01 4.89934428e-06\n",
      "  2.56512430e-05 6.83483449e-07 3.08118456e-06 4.15068507e-06\n",
      "  5.32645288e-07 4.95717832e-05 3.14798788e-04 6.57802866e-06\n",
      "  4.01792113e-06 1.25412146e-06 4.10460227e-04]\n",
      " [1.05865265e-05 1.48358768e-05 9.98450875e-01 5.91540520e-05\n",
      "  4.00112876e-05 6.65231710e-06 3.57898862e-05 5.51945868e-06\n",
      "  4.23923268e-07 1.27500811e-04 4.35522845e-04 2.88335323e-05\n",
      "  2.22854260e-06 1.52949019e-06 7.80594826e-04]]\n"
     ]
    }
   ],
   "source": [
    "print(predictions[:10])    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the index of the maximum value in each row\n",
    "predictions = np.argmax(predictions, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(418000,)\n",
      "[ 0  6  0  2  6  0  2 14  2  2]\n"
     ]
    }
   ],
   "source": [
    "print(predictions.shape)\n",
    "print(predictions[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Map each diacrtics to its unicode\n",
    "diacritics_mapping = {\n",
    "    'FATHA': '\\u064E',\n",
    "    'DAMMA': '\\u064F',\n",
    "    'KASRA': '\\u0650',\n",
    "    'SHADDA': '\\u0651',\n",
    "    'SUKUN': '\\u0652',\n",
    "    'FATHATAN': '\\u064B',\n",
    "    'DAMMATAN': '\\u064C',\n",
    "    'KASRATAN': '\\u064D'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make a dictionary to map the predictions to the corresponding unicode \n",
    "\n",
    "predictions_map = {\n",
    "    0 : diacritics_mapping['FATHA'],\n",
    "    1 : diacritics_mapping['FATHATAN'],\n",
    "    2 : diacritics_mapping['KASRA'],\n",
    "    3 : diacritics_mapping['KASRATAN'],\n",
    "    4 : diacritics_mapping['DAMMA'],\n",
    "    5 : diacritics_mapping['DAMMATAN'],\n",
    "    6 : diacritics_mapping['SUKUN'],\n",
    "    7 : diacritics_mapping['SHADDA'],\n",
    "    8 : diacritics_mapping['SHADDA'] + diacritics_mapping['FATHA'],\n",
    "    9 : diacritics_mapping['SHADDA'] + diacritics_mapping['FATHATAN'],\n",
    "    10 : diacritics_mapping['SHADDA'] + diacritics_mapping['KASRA'],\n",
    "    11 : diacritics_mapping['SHADDA'] + diacritics_mapping['KASRATAN'],\n",
    "    12 : diacritics_mapping['SHADDA'] + diacritics_mapping['DAMMA'],\n",
    "    13 : diacritics_mapping['SHADDA'] + diacritics_mapping['DAMMATAN'],\n",
    "    14 : ''\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# truncate the predictions to 417470\n",
    "predictions = predictions[:417470]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "417359\n",
      "['لَ', 'يْ', 'سَ', 'لِ', 'لْ', 'وَ', 'كِ', 'ي', 'لِ', 'بِ']\n"
     ]
    }
   ],
   "source": [
    "# loop over the letters and concatenate it with the corresponding prediction\n",
    "predicted_diacritized_text = []\n",
    "count = 0\n",
    "for i in range(len(test_tokenized_input)):\n",
    "    for j in range(len(test_tokenized_input[i])):\n",
    "        predicted_diacritized_text.append(test_tokenized_input[i][j] + predictions_map[predictions[count]])\n",
    "        count += 1\n",
    "        \n",
    "print(len(predicted_diacritized_text))\n",
    "print(predicted_diacritized_text[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./generatedFiles/test/predictions.csv', 'w', encoding='utf-8') as file:\n",
    "    file.write('ID,label\\n')\n",
    "    id = 0\n",
    "    id2 = 0\n",
    "    for i in range(len(test_tokenized_input)):\n",
    "        for j in range(len(test_tokenized_input[i])):\n",
    "            # check if test_tokenized_input[i][j] is an arabic letter\n",
    "            if test_tokenized_input[i][j] in arabic_letters:\n",
    "                file.write(f'{id2},{predictions[id]}\\n')\n",
    "                id2 += 1\n",
    "            id += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "x ={\n",
    "    0 : 0,\n",
    "    1 : 1,\n",
    "    2 : 4,\n",
    "    3 : 5,\n",
    "    4 : 2,\n",
    "    5 : 3,\n",
    "    6 : 6,\n",
    "    7 : 7,\n",
    "    8 : 8,\n",
    "    9 : 9,\n",
    "    10 : 12,\n",
    "    11 : 13,\n",
    "    12 : 10,\n",
    "    13 : 11,\n",
    "    14 : 14\n",
    "    \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read csv file, and loop over the csv, for the second column replace each value with its corresponding value in x\n",
    "import csv\n",
    "data = []\n",
    "with open('./generatedFiles/test/predictions.csv', 'r') as file:\n",
    "    reader = csv.reader(file)\n",
    "    for row in reader:\n",
    "        data.append(row)\n",
    "        \n",
    "for i in range(1, len(data)):\n",
    "    data[i][1] = x[int(data[i][1])]\n",
    "    \n",
    "    \n",
    "    \n",
    "# write the new data to the csv file\n",
    "with open(\"./generatedFiles/test/predictions_updated.csv\", mode='w', newline='') as file:\n",
    "    writer = csv.writer(file)\n",
    "    writer.writerows(data)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}