Arabic_Process.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "###First aquiring the most common words\n",
        "\n",
        "Link: https://raw.githubusercontent.com/hermitdave/FrequencyWords/master/content/2016/ar/ar_50k.txt\n"
      ],
      "metadata": {
        "id": "y1yWyVRpefMF"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "skhtzXpHeacR"
      },
      "outputs": [],
      "source": [
        "file1 = open('ar_50k.txt', 'r')\n",
        "Lines = file1.readlines()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with open('arabic_nouns.txt', 'w') as f:\n",
        "  for line in Lines:\n",
        "    #print ()\n",
        "    word = line.split(\" \")[0]\n",
        "    f.write(f\"{word}\\n\")\n",
        "  #break"
      ],
      "metadata": {
        "id": "otJiwIVQez12"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Trying to Get rid of Stop Words and Stemming\n",
        "\n",
        "Link: https://arabicstemmer.com/"
      ],
      "metadata": {
        "id": "EEvALWrNe0mR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install snowballstemmer-1.1.0.tar.gz"
      ],
      "metadata": {
        "id": "9u4J4ey7e5MC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from snowballstemmer import stemmer"
      ],
      "metadata": {
        "id": "6Q0liiyVfGHd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "file1 = open('arabic_nouns.txt', 'r')\n",
        "Lines = file1.readlines()"
      ],
      "metadata": {
        "id": "vKvbDa7efKw2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "stemmed_list = []\n",
        "for line in Lines:\n",
        "  stemmed_list.append(ar_stemmer.stemWord(line))"
      ],
      "metadata": {
        "id": "XAD9-URQfMFt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "final_nouns =list(set(stemmed_list))"
      ],
      "metadata": {
        "id": "YYe3Jyl2fNeo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with open('final_nouns.txt', 'w') as f:\n",
        "    for word in final_nouns:\n",
        "        f.write(f\"{word}\\n\")"
      ],
      "metadata": {
        "id": "khv13dt-fQu5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Second: Trying diffrent approaches for the model\n",
        "\n",
        "link:https://github.com/bakrianoo/aravec"
      ],
      "metadata": {
        "id": "L4yQZAExfVvX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import gensim \n",
        "import re"
      ],
      "metadata": {
        "id": "uoefMULWfbuM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def clean_str(text):\n",
        "    search = [\"أ\",\"إ\",\"آ\",\"ة\",\"_\",\"-\",\"/\",\".\",\"،\",\" و \",\" يا \",'\"',\"ـ\",\"'\",\"ى\",\"\\\\\",'\\n', '\\t','&quot;','?','؟','!']\n",
        "    replace = [\"ا\",\"ا\",\"ا\",\"ه\",\" \",\" \",\"\",\"\",\"\",\" و\",\" يا\",\"\",\"\",\"\",\"ي\",\"\",' ', ' ',' ',' ? ',' ؟ ',' ! ']\n",
        "    \n",
        "    #remove tashkeel\n",
        "    p_tashkeel = re.compile(r'[\\u0617-\\u061A\\u064B-\\u0652]')\n",
        "    text = re.sub(p_tashkeel,\"\", text)\n",
        "    \n",
        "    #remove longation\n",
        "    p_longation = re.compile(r'(.)\\1+')\n",
        "    subst = r\"\\1\\1\"\n",
        "    text = re.sub(p_longation, subst, text)\n",
        "    \n",
        "    text = text.replace('وو', 'و')\n",
        "    text = text.replace('يي', 'ي')\n",
        "    text = text.replace('اا', 'ا')\n",
        "    \n",
        "    for i in range(0, len(search)):\n",
        "        text = text.replace(search[i], replace[i])\n",
        "    \n",
        "    #trim    \n",
        "    text = text.strip()\n",
        "\n",
        "    return text"
      ],
      "metadata": {
        "id": "y6uDhkr-fx74"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#!wget \"https://bakrianoo.ewr1.vultrobjects.com/aravec/full_uni_cbow_300_twitter.zip\"\n",
        "!wget \"https://bakrianoo.ewr1.vultrobjects.com/aravec/full_uni_cbow_100_wiki.zip\""
      ],
      "metadata": {
        "id": "YXJC8xQefzzA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#!unzip \"full_uni_cbow_300_twitter.zip\"\n",
        "!unzip \"full_uni_cbow_100_wiki.zip\""
      ],
      "metadata": {
        "id": "Pmx1msXVf0vI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#t_model = gensim.models.Word2Vec.load('full_uni_cbow_300_twitter.mdl')\n",
        "t_model = gensim.models.Word2Vec.load('full_uni_cbow_100_wiki.mdl')"
      ],
      "metadata": {
        "id": "_IGq9og2f1wh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "nouns = list(t_model.wv.index_to_key)\n",
        "print(nouns[:100])"
      ],
      "metadata": {
        "id": "WILd_i5kf2wT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "t_model.wv.sort_by_descending_frequency()"
      ],
      "metadata": {
        "id": "P4hwartIf4Vl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# print(t_model.wv.vocab.keys())\n",
        "nouns = list(t_model.wv.index_to_key)\n",
        "print(nouns[:100])"
      ],
      "metadata": {
        "id": "4B-DBn1pf5nV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# python 3.X\n",
        "token = clean_str(u'قطة')\n",
        "# python 2.7\n",
        "# token = clean_str('تونس'.decode('utf8', errors='ignore'))\n",
        "\n",
        "most_similar = t_model.wv.most_similar( token, topn=50 )\n",
        "for term, score in most_similar:\n",
        "    print(term, score)"
      ],
      "metadata": {
        "id": "alogoqZjf6yy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from time import sleep"
      ],
      "metadata": {
        "id": "mQRbSAWgf-n2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def start_game(key_word = None, cold_threshold = 0.4, hot_threshold = 0.65, get_similarity = False):\n",
        "    \n",
        "    if key_word is None:\n",
        "        # key_word = np.random.choice(nouns)\n",
        "        token = clean_str(u'القطه')\n",
        "    \n",
        "    lag = 1.0\n",
        "    print(\"Welcome to the game!\")\n",
        "    sleep(lag)\n",
        "    print(\"I thought of a word. Your goal is to guess it!\")\n",
        "    sleep(lag)\n",
        "    print(\"You will have an unlimited number of attempts to guess it\")\n",
        "    sleep(lag)\n",
        "    print(\"I will say how close (hot) or far (cold) your assumption is relative to my word\")\n",
        "    sleep(lag)\n",
        "    print(\"If you want to give up, enter 0 or 'give up'\")\n",
        "    sleep(lag)\n",
        "    print(\"Good luck!\")\n",
        "    sleep(lag)\n",
        "    \n",
        "    win_flag = False\n",
        "    give_up_flag = False\n",
        "    attempts = 0\n",
        "    \n",
        "    while not (win_flag or give_up_flag):\n",
        "        print()\n",
        "        print(\"Type the word\")\n",
        "        word = input()\n",
        "        \n",
        "        if word == key_word:\n",
        "            attempts += 1\n",
        "            win_flag = True\n",
        "            break\n",
        "        if word == \"give up\" or word == '0':\n",
        "            give_up_flag = True\n",
        "            break\n",
        "            \n",
        "        if word in nouns:\n",
        "            \n",
        "            attempts += 1\n",
        "            # similarity\n",
        "            score = t_model.wv.similarity(key_word, word)\n",
        "        \n",
        "            if score < cold_threshold:\n",
        "                print(\"It's cold! Your assumption is very far from the intended word\")\n",
        "            elif score > hot_threshold:\n",
        "                print(\"HOT!!! Your assumption is very close to the intended word\")\n",
        "            else:\n",
        "                print(\"It's warm. There is a slight connection between your assumption and the intended word\")\n",
        "            \n",
        "            if get_similarity:\n",
        "                print(\"similarity is \" + str(np.round(score, 4)))\n",
        "            \n",
        "        else:\n",
        "            \n",
        "            print(\"Your word is not a singular noun!\")\n",
        "            print(\"Try again!\")\n",
        "    \n",
        "    print()\n",
        "    if win_flag:\n",
        "        print(\"Congrats! You have won!\")\n",
        "        \n",
        "    if give_up_flag:\n",
        "        print(\"It was a difficult word!\")\n",
        "        \n",
        "    print(\"The intended word is \" + key_word)\n",
        "    print(\"Number of attempts: \" + str(attempts))"
      ],
      "metadata": {
        "id": "h0pGBgapf_wI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Link: https://huggingface.co/asafaya/bert-base-arabic and https://huggingface.co/CAMeL-Lab/bert-base-arabic-camelbert-mix"
      ],
      "metadata": {
        "id": "YVmLfRjhgjm9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers"
      ],
      "metadata": {
        "id": "0QHYlxPEgu2l"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "import torch\n",
        "import torch.nn.functional as f\n",
        "\n",
        "from transformers import BertModel, BertTokenizer\n",
        "\n",
        "from time import sleep"
      ],
      "metadata": {
        "id": "IF3y0InWgzhW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "print(device)"
      ],
      "metadata": {
        "id": "4zR1M2N3g0mq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from abc import ABC\n",
        "class Analyzer(ABC):\n",
        "  def __init__(\n",
        "      self,\n",
        "      similarity_func, \n",
        "      bert_version : str,\n",
        "      device, \n",
        "      ):\n",
        "      self.similarity_func = similarity_func\n",
        "\n",
        "      self.bert_version = bert_version\n",
        "      self.tokenizer = BertTokenizer.from_pretrained(bert_version)\n",
        "      self.model = BertModel.from_pretrained(bert_version)\n",
        "\n",
        "      self.device=device\n",
        "\n",
        "      self.model = self.model.eval()\n",
        "\n",
        "  def get_word_emb(\n",
        "      self, \n",
        "      token : str\n",
        "      ):\n",
        "      encoding = self.tokenizer(\n",
        "          token, \n",
        "          padding=True, \n",
        "          return_tensors='pt'\n",
        "          ).to(self.device)\n",
        "\n",
        "      for tokens in encoding['input_ids']:\n",
        "        self.tokenizer.convert_ids_to_tokens(tokens)\n",
        "\n",
        "      with torch.no_grad():\n",
        "        embed = self.model(**encoding)[0]\n",
        "\n",
        "      avg_embed = embed.mean(dim=1)\n",
        "\n",
        "      return avg_embed\n",
        "    \n",
        "  def get_similarity(self, embed1, embed2):\n",
        "      return self.similarity_func(embed1, embed2)[0]"
      ],
      "metadata": {
        "id": "lJwMI7FXg1jp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"arabic_nouns.txt\", 'r', encoding='utf-8') as f:\n",
        "    nouns = f.read().split('\\n')"
      ],
      "metadata": {
        "id": "LTmP6Xksg20Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def start_game(\n",
        "    key_word = None, \n",
        "    cold_threshold = 0.5, \n",
        "    hot_threshold = 0.8, \n",
        "    get_similarity = False, \n",
        "    analyzer=None\n",
        "    ):\n",
        "    if analyzer is None:\n",
        "      embed_calc = Analyzer(\n",
        "        similarity_func=torch.nn.CosineSimilarity(),\n",
        "        bert_version='CAMeL-Lab/bert-base-arabic-camelbert-mix',\n",
        "        device=device\n",
        "      )\n",
        "    else: \n",
        "      embed_calc = analyzer\n",
        "\n",
        "    if key_word is None:\n",
        "        key_word = 'كلب'#np.random.choice(nouns)\n",
        "    \n",
        "    key_word_emb=embed_calc.get_word_emb(token=key_word)\n",
        "\n",
        "    lag = 1.0\n",
        "    print(\"Welcome to the game!\")\n",
        "    sleep(lag)\n",
        "    print(\"I thought of a word. Your goal is to guess it!\")\n",
        "    sleep(lag)\n",
        "    print(\"You will have an unlimited number of attempts to guess it\")\n",
        "    sleep(lag)\n",
        "    print(\"I will say how close (hot) or far (cold) your assumption is relative to my word\")\n",
        "    sleep(lag)\n",
        "    print(\"If you want to give up, enter 0 or 'give up'\")\n",
        "    sleep(lag)\n",
        "    print(\"Good luck!\")\n",
        "    sleep(lag)\n",
        "    \n",
        "    win_flag = False\n",
        "    give_up_flag = False\n",
        "    attempts = 0\n",
        "    \n",
        "    while not (win_flag or give_up_flag):\n",
        "        print()\n",
        "        print(\"Type the word\")\n",
        "        word = input()\n",
        "        \n",
        "        if word == key_word:\n",
        "            attempts += 1\n",
        "            win_flag = True\n",
        "            break\n",
        "        if word == \"give up\" or word == '0':\n",
        "            give_up_flag = True\n",
        "            break\n",
        "            \n",
        "        if word in nouns:\n",
        "            \n",
        "            attempts += 1\n",
        "\n",
        "            word_emb=embed_calc.get_word_emb(token=word)\n",
        "            score = embed_calc.get_similarity(embed1=key_word_emb, embed2=word_emb)\n",
        "        \n",
        "            if score < cold_threshold:\n",
        "                print(\"It's cold! Your assumption is very far from the intended word\")\n",
        "            elif score > hot_threshold:\n",
        "                print(\"HOT!!! Your assumption is very close to the intended word\")\n",
        "            else:\n",
        "                print(\"It's warm. There is a slight connection between your assumption and the intended word\")\n",
        "            \n",
        "            if get_similarity:\n",
        "                print(f\"similarity is {float(score.numpy()):.4f}\")\n",
        "            \n",
        "        else:\n",
        "            \n",
        "            print(\"Your word is not a singular noun!\")\n",
        "            print(\"Try again!\")\n",
        "    \n",
        "    print()\n",
        "    if win_flag:\n",
        "        print(\"Congrats! You have won!\")\n",
        "        \n",
        "    if give_up_flag:\n",
        "        print(\"It was a difficult word!\")\n",
        "        \n",
        "    print(f\"The intended word is {key_word}\")\n",
        "    print(f\"Number of attempts: {str(attempts)}\")"
      ],
      "metadata": {
        "id": "Mq9gN8yYg26Y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "embed_calc = Analyzer(\n",
        "    similarity_func=torch.nn.CosineSimilarity(),\n",
        "    bert_version='asafaya/bert-base-arabic',\n",
        "    device=device\n",
        "  )"
      ],
      "metadata": {
        "id": "-lXJPHsLg51k"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "start_game(\n",
        "    key_word = 'دمشق', \n",
        "    cold_threshold = 0.5, \n",
        "    hot_threshold = 0.88, \n",
        "    get_similarity = True, \n",
        "    analyzer=embed_calc\n",
        "    )"
      ],
      "metadata": {
        "id": "Mr3hEwPYg8JZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Another try to get the most common words"
      ],
      "metadata": {
        "id": "Bc54_1kyg_HS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "file1 = open('ar_50k.txt', 'r')\n",
        "Lines = file1.readlines()"
      ],
      "metadata": {
        "id": "2S_PhnVJhAwE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with open('arabic_nouns.txt', 'w') as f:\n",
        "  for line in Lines:\n",
        "    #print ()\n",
        "    word = line.split(\" \")[0]\n",
        "    f.write(f\"{word}\\n\")\n",
        "  #break"
      ],
      "metadata": {
        "id": "r2DZWFH7hExA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!wget \"https://bakrianoo.ewr1.vultrobjects.com/aravec/full_uni_cbow_100_wiki.zip\""
      ],
      "metadata": {
        "id": "pR-1P2KhhGxN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip \"full_uni_cbow_100_wiki.zip\""
      ],
      "metadata": {
        "id": "27ID9zSkhIBc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "t_model = gensim.models.Word2Vec.load('full_uni_cbow_100_wiki.mdl')"
      ],
      "metadata": {
        "id": "VZ1hTxwVhJhY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "f = open(\"arabic_nouns.txt\")\n",
        "lines = f.readlines()\n",
        "f.close()\n",
        "nouns = [line[:-1] for line in lines]\n",
        "N = len(nouns)\n",
        "print (N)"
      ],
      "metadata": {
        "id": "_Q9f-WBhhKlP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_nouns = list(t_model.wv.index_to_key)"
      ],
      "metadata": {
        "id": "Pc2w5Lx_hLui"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_nouns = list(t_model.wv.index_to_key)"
      ],
      "metadata": {
        "id": "7hGiaZSIhM8i"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "final_nouns =list(set(model_nouns).intersection(nouns))"
      ],
      "metadata": {
        "id": "wqBBPv-rhOEJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print (len(final_nouns))"
      ],
      "metadata": {
        "id": "exnFaSRBhPjG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with open('final_nouns.txt', 'w') as f:\n",
        "  for noun in final_nouns:\n",
        "    #print ()\n",
        "    f.write(f\"{noun}\\n\")"
      ],
      "metadata": {
        "id": "yNHlcHMdhQoq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Finalizing\n",
        "link : https://github.com/linuxscout/arramooz"
      ],
      "metadata": {
        "id": "D1UNAcR5n-3d"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "metadata": {
        "id": "Hgdmf4C5vWz0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# read text file into pandas DataFrame\n",
        "df = pd.read_csv(\"nouns.dict.txt\", sep=\"\\t\",on_bad_lines='skip')"
      ],
      "metadata": {
        "id": "EMb3tj_5oRX4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "metadata": {
        "id": "8taSRcE0oW6Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['مفرد/تكسير.1'].head()"
      ],
      "metadata": {
        "id": "A0NvghNcoY7p"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "np.savetxt(r'nouns.txt', df['مفرد/تكسير.1'].values, fmt='%s',encoding=\"utf-8\")"
      ],
      "metadata": {
        "id": "A8pI0cS6oarT"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}