TMRetrieval-FULL.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Translation Memory Retrieval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: Preprocessing is a separate module and must be done before using this!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import nltk\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     /Users/khannatanmai/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "stop_words = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I want to safely remove THE selected drive if it is POSSIBLE\n",
      "['want', 'safely', 'remove', 'selected', 'drive', 'possible']\n"
     ]
    }
   ],
   "source": [
    "input_line = input()\n",
    "\n",
    "#convert input to lowercase\n",
    "input_line = input_line.lower()\n",
    "\n",
    "#tokenise\n",
    "input_tokens = word_tokenize(input_line)\n",
    "\n",
    "content_words = [word for word in input_tokens if word not in stop_words] #Removing Stopwords\n",
    "\n",
    "print(content_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Edit Distance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load TM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "src_tm_words = [] #Content Words in Source TM\n",
    "\n",
    "with open('../tm_data/tm_src_pp.txt') as src_tm:\n",
    "    line = src_tm.readline()\n",
    "    \n",
    "    while line:\n",
    "        line = line.rstrip() #Removing Trailing Whitespace\n",
    "        \n",
    "        words = line.split('\\t')\n",
    "        src_tm_words.append(words)\n",
    "        \n",
    "        line = src_tm.readline()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Execute Edit Distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "207 ['safely', 'remove', 'selected', 'drive'] 2\n",
      "216241 ['safely', 'remove', 'drive'] 3\n",
      "750192 ['want', 'remove', 'package', 'continue', '?'] 4\n",
      "623225 ['remove', 'selected', 'account'] 4\n",
      "110189 ['remove', 'selected', 'bookmark'] 4\n"
     ]
    }
   ],
   "source": [
    "edit_distance_all = []\n",
    "N = 5 #Top N matches returned\n",
    "\n",
    "for element in src_tm_words:\n",
    "    ed = nltk.edit_distance(content_words, element) #Calculate Edit Distance with content words\n",
    "    edit_distance_all.append(ed)\n",
    "    \n",
    "#Get top N results\n",
    "edit_distance_all = np.array(edit_distance_all)\n",
    "\n",
    "sorted_indices = np.argsort(edit_distance_all) #Sorts in ascending order and returns the indices of elements in original array\n",
    "least_N_indices = sorted_indices[:N] #We want least edit distance\n",
    "\n",
    "for i in least_N_indices:\n",
    "    print(i+1, src_tm_words[i], edit_distance_all[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieval of Target from TM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "207 चयनित ड्राइव सुरक्षित रूप से निकालें\n",
      "\n",
      "216241 क्षैतिज विभाजन और स्तर विन्यास के वास्तविक अभिलोपन की लगभग सीमा तक दमन के ऐसे परिप्रेक्ष्य में शैली समाधान नहीं कर सकी .\n",
      "\n",
      "750192 कैल्शियम\n",
      "\n",
      "623225 ये शरीर और दिमाग में आक्सीजन का प्रवाह बढ़ाते हैं जिससे दिमाग की सक्रियता बढ़ती है ।\n",
      "\n",
      "110189 मुश्कबू\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tgt_tm_array = []\n",
    "\n",
    "with open('../tm_data/tm_tgt.txt') as tgt_tm:\n",
    "    line = tgt_tm.readline()\n",
    "    \n",
    "    while line:\n",
    "        tgt_tm_array.append(line)\n",
    "        line = tgt_tm.readline()\n",
    "        \n",
    "for i in least_N_indices:\n",
    "    print(i+1, tgt_tm_array[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}