Document Translator.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "r07kTYZd-okT"
   },
   "source": [
    "## Initial Stage\n",
    "### Importing Libraries and setting constraint values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Tkk_k3x33yOc"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "import re\n",
    "import json\n",
    "import pprint\n",
    "import sys\n",
    "from operator import itemgetter\n",
    "from copy import deepcopy\n",
    "import random\n",
    "from collections import defaultdict\n",
    "\n",
    "#constants\n",
    "MIN_PROB = 1.0e-12\n",
    "\n",
    "#input\n",
    "base = \"/content/drive/My Drive/\"\n",
    "lang_a = \"en\" \n",
    "infile_a = base + \"e.txt\"\n",
    "lang_b = \"du\"\n",
    "infile_b = base + \"d.txt\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zOyjx0OgEHw3"
   },
   "source": [
    "### Loading Data saved in Google Drive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "id": "EK_T0wim-w1z",
    "outputId": "2fb2c7cd-0659-40d1-fdd8-90748fb714ba"
   },
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading Data and Preprocessing\n",
    "### The corpus data is loaded. It is preprocessed by removing punctuations and normalizing the sentences. The list of pair of Dutch and English sentences is made and a sample from the list is returned as the output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OTVAaWGS88eP"
   },
   "outputs": [],
   "source": [
    "#generating corpus which is a vector of dictionaries where each dictionary has a sentence from src language \n",
    "#and its corresponding translation in trg (target) language \n",
    "\n",
    "def get_corpus(lang_a, infile_a, lang_b, infile_b, sentence_size = None):\n",
    "    '''\n",
    "    Load corpus from input file infile_a and infile_b\n",
    "    '''\n",
    "    corpus = []\n",
    "    with open(infile_a, 'r', encoding=\"utf8\") as a, open(infile_b, 'r', encoding=\"utf8\") as b:\n",
    "            while True:\n",
    "                try:\n",
    "                    a_sentence = (next(a)).lower()\n",
    "                    b_sentence = (next(b)).lower()\n",
    "                    if(sentence_size is not None and (len(a_sentence) > sentence_size or len(b_sentence) > sentence_size)):\n",
    "                      continue\n",
    "                    corpus.append({ \n",
    "                        lang_a : a_sentence.rstrip(),\n",
    "                        lang_b : b_sentence.rstrip()\n",
    "                        })\n",
    "                except StopIteration:\n",
    "                    break\n",
    "    return corpus\n",
    "\n",
    "#removes punctuations from the given sentence\n",
    "\n",
    "def to_normalized_sentence(sentence):\n",
    "  return re.sub(r\"[^\\w\\d'\\s]+\",'', sentence)\n",
    "\n",
    "#preprocess corpus\n",
    "\n",
    "def preprocess_corpus(corpus):\n",
    "  p_corpus = []\n",
    "  for pair in corpus:\n",
    "    a_sen = to_normalized_sentence(pair[lang_a])\n",
    "    b_sen = to_normalized_sentence(pair[lang_b])\n",
    "    if(a_sen == '' or b_sen == ''):\n",
    "      continue\n",
    "    p_pair = {lang_a : a_sen,\n",
    "              lang_b : b_sen\n",
    "              }\n",
    "    p_corpus.append(p_pair)\n",
    "  return p_corpus\n",
    "\n",
    "#generates random sample from the passed corpus\n",
    "\n",
    "def get_sample(input, size, seed):\n",
    "  random.seed(seed)\n",
    "  return random.sample(input, size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function call for preprocessing the loaded corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "ug0dReX4-vTf",
    "outputId": "b06526aa-857b-4755-ac9e-7731f3fed660"
   },
   "outputs": [],
   "source": [
    "corpus_original = get_corpus(lang_a, infile_a, lang_b, infile_b)\n",
    "##print(len(corpus_original))\n",
    "corpus = get_sample(corpus_original, 2, 2)\n",
    "##print(len(corpus))\n",
    "p_corpus = preprocess_corpus(corpus)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Phase"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Dn5R5qgUaQI8"
   },
   "source": [
    "### Initialising vocabulary for source and target language, and setting initial uniform probabilities for target vocabulary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NqJFKWBi6bHP"
   },
   "outputs": [],
   "source": [
    "#initialises the vocabulary sets for src and trg language\n",
    "\n",
    "def init_vocab(corpus, src_vocab, trg_vocab):\n",
    "        for sentence in corpus:\n",
    "            trg_vocab.update(sentence[lang_a].split())\n",
    "            src_vocab.update(sentence[lang_b].split())\n",
    "        # Add the NULL token\n",
    "        src_vocab.add(None)\n",
    "        src_vocab.add(None)\n",
    "\n",
    "#intialises the translation table probabilties and sets it to 1/(no. of unique words in trg language vocabulary)\n",
    "\n",
    "def set_initial_probabilities(corpus, translation_table):\n",
    "        src_vocab = set()\n",
    "        trg_vocab = set()\n",
    "        init_vocab(corpus, src_vocab, trg_vocab)\n",
    "        initial_prob = 1 / len(trg_vocab)\n",
    "\n",
    "        for t in trg_vocab:\n",
    "            translation_table[t] = defaultdict(lambda: initial_prob)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "PWgiKyoHLX6z"
   },
   "source": [
    "### Implementing the EM algorithm and defining training function which returns the translation table."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "S6nxE0K_1rVv"
   },
   "outputs": [],
   "source": [
    "#EM maximization step - updates probabilities with maximum likelihood estimate\n",
    "\n",
    "def max_lex_transl_probab(counts, translation_table):\n",
    "      for t, src_words in counts[\"t_given_s\"].items():\n",
    "          for s in src_words:\n",
    "              estimate = counts[\"t_given_s\"][t][s] / counts[\"any_t_given_s\"][s]\n",
    "              translation_table[t][s] = max(estimate, MIN_PROB)\n",
    "\n",
    "\n",
    "def maximize_lexical_translation_probabilities_innovate(counts, translation_table, n, corpus):\n",
    "      src_vocab = set()\n",
    "      for sentence in corpus:\n",
    "        src_vocab.update(sentence[lang_b].split())\n",
    "      for t, src_words in counts[\"t_given_s\"].items():\n",
    "          for s in src_words:\n",
    "              estimate = (counts[\"t_given_s\"][t][s] +n)/ (counts[\"any_t_given_s\"][s]+len(src_vocab)*n)\n",
    "              translation_table[t][s] = max(estimate, MIN_PROB)              \n",
    "\n",
    "#recomputes translation probababilites i.e. probability of translation of a word in src vocabulary into every word in trg vocabulary \n",
    "\n",
    "def train_iter_helper(corpus, translation_table):\n",
    "        counts = {}\n",
    "        counts[\"t_given_s\"] = defaultdict(lambda: defaultdict(lambda: 0.0))\n",
    "        counts[\"any_t_given_s\"] = defaultdict(lambda: 0.0)\n",
    "        for aligned_sentence in corpus:\n",
    "            trg_sentence = (aligned_sentence[lang_a]).split()\n",
    "            src_sentence = (aligned_sentence[lang_b]).split()\n",
    "\n",
    "            # E step - compute normalization factors to weigh counts\n",
    "            total_count = get_total_count(src_sentence, trg_sentence, translation_table)\n",
    "            # E step - compute counts\n",
    "            for t in trg_sentence:\n",
    "                for s in src_sentence:\n",
    "                    count = prob_alignment_point(s, t, translation_table)\n",
    "                    normalized_count = count / total_count[t]\n",
    "                    counts[\"t_given_s\"][t][s] += normalized_count\n",
    "                    counts[\"any_t_given_s\"][s] += normalized_count\n",
    "\n",
    "        # M step: Update probabilities with maximum likelihood estimate\n",
    "        max_lex_transl_probab(counts, translation_table)\n",
    "def generate_models(corpus, seeds, sample_size, iterations):\n",
    "  models = []\n",
    "  for s in seeds:\n",
    "    random.seed(s)\n",
    "    sample_corpus = random.sample(corpus, sample_size)\n",
    "    model = train(sample_corpus, iterations)\n",
    "    models.append(model)\n",
    "    print(\"Done for seed \" + str(s))\n",
    "  return models\n",
    "\n",
    "def ensemble(models):\n",
    "  translations = []\n",
    "  keys = set()\n",
    "  for model in models:\n",
    "    keys = keys.union(set(model.keys()))\n",
    "    # print(set(translation.keys()))\n",
    "    # print(keys)\n",
    "  ensemble_translations = {}\n",
    "\n",
    "  for key in keys:\n",
    "    temp = {} # keys are foreign words and values are their prob.\n",
    "    count = {}\n",
    "    for model in models:\n",
    "      try:\n",
    "        dict_val = model[key]\n",
    "        for k in dict_val.keys():\n",
    "          if k in temp: \n",
    "            temp[k] = (dict_val[k] + temp[val] * count[k])/(count[k] + 1)\n",
    "            count[k] = count[k] + 1\n",
    "          else:\n",
    "            temp[k] = dict_val[k]\n",
    "            count[k] = 1\n",
    "      except:\n",
    "        a = 0\n",
    "    # print(temp)\n",
    "    ensemble_translations[key] = max(temp.items(), key=operator.itemgetter(1))[0] \n",
    "  return ensemble_translations\n",
    "\n",
    "\n",
    "#trains the model on the given training corpus for the given number of iterations\n",
    "\n",
    "def train(corpus, iterations, dump_filename = None):\n",
    "  translation_table = {}\n",
    "  set_initial_probabilities(corpus, translation_table)\n",
    "  for i in range(iterations):\n",
    "    train_iter_helper(corpus, translation_table)\n",
    "    print(str(i) + \" iterations completed\")\n",
    "    if (i % 10 and dump_filename is not None):\n",
    "      dump_model(translation_table, filename = dump_filename + str(i) + \".json\")\n",
    "  return translation_table\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "QsPsnAV8dfMp"
   },
   "source": [
    "### Defining functions that compute the probability of all possible word alignments, expressed as a marginal distribution over target words t in a target sentence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "N1MQe-4G3l9J"
   },
   "outputs": [],
   "source": [
    "#returns probability of src language word 's' being translated to trg language word 't'\n",
    "\n",
    "def prob_alignment_point(source, target, translation_table):\n",
    "        return translation_table[target][source]\n",
    "\n",
    "#computes the sum of probability of all possible alignments, for the translation of src sentence into trg sentence\n",
    "\n",
    "def get_total_count(src_sentence, trg_sentence, translation_table):\n",
    "        \n",
    "        alignment_prob_for_t = defaultdict(lambda: 0.0)\n",
    "        for target in trg_sentence:\n",
    "            for source in src_sentence:\n",
    "                alignment_prob_for_t[target] += prob_alignment_point(source, target, translation_table)\n",
    "        return alignment_prob_for_t\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uvGaP8TkKdNH"
   },
   "source": [
    "## Testing Phase\n",
    "### Function for translating test sentence based on the trained model which contains word translations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "glpnIHwnh1M2"
   },
   "outputs": [],
   "source": [
    "#computes TF-IDF weights for words in src and trg language documents \n",
    "def get_doc_vectors(docA, docB, idf):\n",
    "\n",
    "  def computeTF(wordDict, bow):\n",
    "      tfDict = {}\n",
    "      bowCount = len(bow)\n",
    "      for word, count in wordDict.items():\n",
    "          if(count==0):\n",
    "            tfDict[word] = 0\n",
    "          else:\n",
    "            tfDict[word] = 1+math.log10(count)\n",
    "      return tfDict\n",
    "\n",
    "  def computeTFIDF(tfBow, idf):\n",
    "      tfidf = {}\n",
    "      for word, val in tfBow.items():\n",
    "          tfidf[word] = val*idf[word]\n",
    "      return tfidf\n",
    "\n",
    "  bowA = docA.split()\n",
    "  bowB = docB.split()\n",
    "  wordSet = set(bowA).union(set(bowB))\n",
    "  wordDictA = dict.fromkeys(wordSet, 0) \n",
    "  wordDictB = dict.fromkeys(wordSet, 0)\n",
    "\n",
    "  for word in bowA:\n",
    "    wordDictA[word]+=1\n",
    "  for word in bowB:\n",
    "    wordDictB[word]+=1\n",
    "\n",
    "  tfBowA = computeTF(wordDictA, bowA)\n",
    "  tfBowB = computeTF(wordDictB, bowB)\n",
    "  tfidfBowA = computeTFIDF(tfBowA, idf)\n",
    "  tfidfBowB = computeTFIDF(tfBowB, idf)\n",
    "\n",
    "  df = pd.DataFrame([tfidfBowA, tfidfBowB])\n",
    "  return df.values[0], df.values[1]\n",
    "\n",
    "#computes cosine similarity\n",
    "\n",
    "def cosine_similarity(idf, docA, docB):\n",
    "  try:\n",
    "    a, b = get_doc_vectors(docA, docB, idf)\n",
    "    dot_product = np.dot(a,b)\n",
    "    norm_a = np.linalg.norm(a)\n",
    "    norm_b = np.linalg.norm(b)\n",
    "    return dot_product/(norm_a*norm_b)\n",
    "  except:\n",
    "    return -1\n",
    "\n",
    "\n",
    "#computes IDF values corresonding to all the unique words in every document in docList\n",
    "\n",
    "def compute_IDFs(corpus):\n",
    "\n",
    "  def compute_IDF(docList):\n",
    "      idfDict = {}\n",
    "      word_set = set()\n",
    "      for sent in docList:\n",
    "        word_set = word_set.union(set(sent))\n",
    "      N = len(docList)\n",
    "      for word in word_set:\n",
    "          val = 0\n",
    "          for doc in docList:\n",
    "            if word in doc:\n",
    "              val = val+1\n",
    "          idfDict[word] = math.log10(N / float(val))\n",
    "      return idfDict\n",
    "\n",
    "  idfs = {}\n",
    "  docListA = []\n",
    "  docListB = []\n",
    "  for pair in corpus:\n",
    "    docListA.append(pair[lang_a].split())\n",
    "    docListB.append(pair[lang_b].split())\n",
    "  idfs[lang_a] = compute_IDF(docListA)\n",
    "  idfs[lang_b] = compute_IDF(docListB)\n",
    "  return idfs\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Utility Functions for computing Jaccard Coefficient and Cosine Similarity and would be used for evaluating the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#computes jaccard coefficient\n",
    "\n",
    "def jaccard_similarity(docA, docB):\n",
    "  list1 = docA.split()\n",
    "  list2 = docB.split()\n",
    "  intersection = len(list(set(list1).intersection(list2)))\n",
    "  union = (len(list1) + len(list2)) - intersection\n",
    "  return float(intersection) / union\n",
    "\n",
    "#returns a dictionary of cosine similarity and jaccard coefficient\n",
    "\n",
    "def similarity(idf, docA, docB):\n",
    "  return {\n",
    "      \"cosine\" : cosine_similarity(idf, docA, docB),\n",
    "      \"jaccard\" : jaccard_similarity(docA, docB)\n",
    "  }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nSDILWMQKkoW"
   },
   "source": [
    "### Functions for getting the translated sentences and the corresponding performance score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZFanAhCyWskr"
   },
   "outputs": [],
   "source": [
    "#returns dictionary which contains word translations from src language to trg language\n",
    "\n",
    "def get_translations(translation_table):\n",
    "  translations = {}\n",
    "  for t in translation_table.keys():\n",
    "    translations[t] = max(translation_table[t].items(), key=lambda a: a[1])[0]\n",
    "  return translations\n",
    "\n",
    "#accepts a sentence and returns its translation into trg language\n",
    "\n",
    "def translate_sentence(model, sentence):\n",
    "    def tokenize(sentence):\n",
    "      return sentence.split()\n",
    "\n",
    "    def translate(tokens, translations):\n",
    "        return [translations[word] if word in translations else word for word in tokens]\n",
    "\n",
    "    translations = get_translations(model)\n",
    "    tokens = tokenize(sentence)\n",
    "    translated_tokens = translate(tokens, translations)\n",
    "\n",
    "    return \" \".join(translated_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Result Phase\n",
    "### Calling Function for Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 190
    },
    "colab_type": "code",
    "id": "hZXWKnQl_YzE",
    "outputId": "c8aa53b3-6607-46fd-c51b-2b11f8565941"
   },
   "outputs": [],
   "source": [
    "model = train(p_corpus[:1000], 2)\n",
    "idfs = compute_IDFs(p_corpus[:1000])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Saving the generated model as a json file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "17Blf_FnwX_3"
   },
   "outputs": [],
   "source": [
    "def dump_model(model, filename, idfs = None):\n",
    "  data = {\"model\" : model, \n",
    "          \"idfs\" : idfs}\n",
    "  with open(filename, \"w+\") as f:\n",
    "    json.dump(data, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Rk4MHdJMKBXQ"
   },
   "source": [
    "### Computing and printing Average Cosine Similarity and Jaccard Coefficient for all the test cases."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4JJ170ou6amX"
   },
   "outputs": [],
   "source": [
    "cos_sim = 0\n",
    "jac_sim = 0\n",
    "for i in range(len(p_corpus)):\n",
    "  docA = p_corpus[i][lang_b]\n",
    "  docB = translate_sentence(model, p_corpus[i][lang_a])\n",
    "  sim = similarity(idfs[lang_b], docA, docB)\n",
    "  jac_sim += sim[\"jaccard\"]\n",
    "  cos_sim += sim[\"cosine\"]\n",
    "jac_sim /= len(p_corpus)\n",
    "cos_sim /= len(p_corpus)\n",
    "print(\"Jaccard: \" + str(jac_sim) + \" Cosine: \" + str(cos_sim))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "wy5CrhYTKnNp"
   },
   "source": [
    "## Ensemble Learning for the Innovation Part\n",
    "### Implementing Ensemble Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nQDX7mALKVcT"
   },
   "outputs": [],
   "source": [
    "import operator\n",
    "\n",
    "# generating sets of model for ensemble learning\n",
    "def generate_models(corpus, seeds, sample_size, iterations):\n",
    "  models = []\n",
    "  for s in seeds:\n",
    "    random.seed(s)\n",
    "    sample_corpus = random.sample(corpus, sample_size)\n",
    "    model = train(sample_corpus, iterations)\n",
    "    models.append(model)\n",
    "    print(\"Done for seed \" + str(s))\n",
    "  return models\n",
    "\n",
    "# get translation probabilites from the ensembler\n",
    "def ensemble(models):\n",
    "  translations = []\n",
    "  keys = set()\n",
    "  for model in models:\n",
    "    keys = keys.union(set(model.keys()))\n",
    "  ensemble_translations = {}\n",
    "\n",
    "  for key in keys:\n",
    "    temp = {} # keys are foreign words and values are their prob.\n",
    "    count = {}\n",
    "    for model in models:\n",
    "      try:\n",
    "        dict_val = model[key]\n",
    "        for k in dict_val.keys():\n",
    "          if k in temp: \n",
    "            temp[k] = (dict_val[k] + temp[val] * count[k])/(count[k] + 1)\n",
    "            count[k] = count[k] + 1\n",
    "          else:\n",
    "            temp[k] = dict_val[k]\n",
    "            count[k] = 1\n",
    "      except:\n",
    "        a = 0\n",
    "    ensemble_translations[key] = max(temp.items(), key=operator.itemgetter(1))[0] \n",
    "  return ensemble_translations\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 765
    },
    "colab_type": "code",
    "id": "KGDMXXI-LU7c",
    "outputId": "65f41e75-31de-4cf1-c75a-51901cdf6a9a"
   },
   "outputs": [],
   "source": [
    "ensemble_models = generate_models(p_corpus, [1, 2, 3, 4], 1000, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "id": "SGznomTTLgpq",
    "outputId": "29fff3ef-a224-46b7-dc1f-996fef8ba69c"
   },
   "outputs": [],
   "source": [
    "ensemble_translations = ensemble(ensemble_models)\n",
    "print(ensemble_translations)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Displaying the final Results Obtained"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5nk0GfFnH_4Z"
   },
   "outputs": [],
   "source": [
    "def read_output_json(filename):\n",
    "  data = None\n",
    "  with open(filename, \"r\") as f:\n",
    "    data = json.load(f)\n",
    "  return data\n",
    "\n",
    "def show_results(input_json):\n",
    "  print(input_json[\"title\"])\n",
    "  print(\"Corpus Length: \" + str(input_json[\"corpus_size\"]) )\n",
    "  pr_corpus = preprocess_corpus(input_json[\"corpus\"])\n",
    "  print(\"Similarity Scores: \")\n",
    "  print(\"Cosine Similarity: \" + input_json[\"score\"][\"cosine\"])\n",
    "  print(\"Jaccard Coefficient: \" + input_json[\"score\"][\"jaccard\"])"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "IR_ASSIGNMENT_GROUP_2.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}