Text_Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.corpus import stopwords \n",
    "from nltk.tokenize import word_tokenize \n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from google.cloud.language import enums\n",
    "from google.cloud.language import types\n",
    "import six\n",
    "from google.cloud import storage\n",
    "from google.cloud import language\n",
    "from google.oauth2 import service_account\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /home/jono/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package punkt to /home/jono/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to /home/jono/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "stop_words = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "#google cloud nlp api which we will use for speach tagging and extracting keywords\n",
    "#You will need a json file from google to use their language API\n",
    "creds = service_account.Credentials.from_service_account_file('xxxxxx.json')\n",
    "client = language.LanguageServiceClient(credentials=creds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "#removing stop words from a sentence, special characters and dublicates\n",
    "def preprocess(sentence):\n",
    "    sentence = remove_special_characters(sentence)\n",
    "    word_tokens = word_tokenize(sentence)\n",
    "    filtered_words = [w for w in word_tokens if not w in stopwords.words('english')]\n",
    "    \n",
    "    return remove_dublicates_words(\" \".join(filtered_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "#removing stop words from a sentence and special characters\n",
    "def preprocess_with_dublicates(sentence):\n",
    "    sentence = remove_special_characters(sentence)\n",
    "    word_tokens = word_tokenize(sentence)\n",
    "    filtered_words = [w for w in word_tokens if not w in stopwords.words('english')]\n",
    "    \n",
    "    return \" \".join(filtered_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "#removing dublicate words in a sentence\n",
    "def remove_dublicates_words(sent):\n",
    "    word_tokens = word_tokenize(sent)\n",
    "    tokens_list=[]\n",
    "    for w in word_tokens:\n",
    "        if w not in tokens_list:\n",
    "            tokens_list.append(w)\n",
    "    return \" \".join(tokens_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Removing special characters\n",
    "def remove_special_characters(s):\n",
    "    tokenizer = nltk.RegexpTokenizer('[a-zA-Z]\\w+\\'?\\w*')\n",
    "    tokens= tokenizer.tokenize(s)\n",
    "    return \" \".join(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for each word, we return the word and the number of its appearances\n",
    "def word_count(sent):\n",
    "    word_tokens = word_tokenize(remove_special_characters(sent))\n",
    "    non_dublicate_tokens=word_tokenize(remove_dublicates_words(remove_special_characters(sent)))\n",
    "    word_count={}\n",
    "    for n in non_dublicate_tokens:\n",
    "        count=0\n",
    "        for w in word_tokens:\n",
    "            if n.lower() == w.lower():\n",
    "                count+=1\n",
    "        word_count[n]=count\n",
    "        \n",
    "    return word_count\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_word_frequency(text):\n",
    "    x=word_count(text)\n",
    "    lst_labels=[]\n",
    "    lst_values=[]\n",
    "\n",
    "    for k,v in x.items():\n",
    "        if(v>1):\n",
    "            lst_labels.append(k)\n",
    "            lst_values.append(v)\n",
    "\n",
    "    # sort your values in descending order\n",
    "    indSort = np.argsort(lst_values)[::-1]\n",
    "\n",
    "    # rearrange your data\n",
    "    labels = np.array(lst_labels)[indSort]\n",
    "    values = np.array(lst_values)[indSort]\n",
    "\n",
    "    indexes = np.arange(len(labels))\n",
    "\n",
    "    bar_width = 0.\n",
    "\n",
    "    plt.bar(indexes, values)\n",
    "\n",
    "    # add labels\n",
    "    plt.xticks(indexes + bar_width, labels)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "def syntax_text(text):\n",
    "    my_token_list=[]\n",
    "    my_key_words=[]\n",
    "    \n",
    "    my_token_dict={}\n",
    "    my_keywords_dict={}\n",
    "    \n",
    "    if isinstance(text, six.binary_type):\n",
    "        text = text.decode('utf-8')\n",
    "\n",
    "    # Instantiates a plain text document.\n",
    "    document = types.Document(\n",
    "        content=text,\n",
    "        type=enums.Document.Type.PLAIN_TEXT)\n",
    "\n",
    "    # Detects syntax in the document. You can also analyze HTML with:\n",
    "    #   document.type == enums.Document.Type.HTML\n",
    "    tokens = client.analyze_syntax(document).tokens\n",
    "\n",
    "    # part-of-speech tags from enums.PartOfSpeech.Tag\n",
    "    pos_tag = ('UNKNOWN', 'ADJ', 'ADP', 'ADV', 'CONJ', 'DET', 'NOUN', 'NUM',\n",
    "               'PRON', 'PRT', 'PUNCT', 'VERB', 'X', 'AFFIX')\n",
    "    for token in tokens:\n",
    "        my_token_list.append((pos_tag[token.part_of_speech.tag],\n",
    "                               token.text.content))\n",
    "        \n",
    "        if pos_tag[token.part_of_speech.tag]=='NOUN':\n",
    "            my_key_words.append(token.text.content)\n",
    "            \n",
    "    my_token_dict['Token_Tags']= my_token_list\n",
    "    my_keywords_dict['keywords']=my_key_words\n",
    "    \n",
    "    return my_token_dict,my_keywords_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Extracts from an article on Jamal Khashoggi's case cited from https://www.bbc.com/news/world-europe-45812399 \n",
    "text=\"\"\"\n",
    "\n",
    "He first visited the Saudi consulate in Istanbul on 28 September to obtain a document certifying\n",
    "that he had divorced his ex-wife, so that he could marry his Turkish fiancée.\n",
    "\n",
    "But he was told he would have to return and arranged to come back on 2 October.\n",
    "\n",
    "\"He did not believe that something bad could happen on Turkish soil,\" his fiancée, \n",
    "Hatice Cengiz, wrote in the Washington Post.\n",
    "\n",
    "\"Jamal was hardly concerned ahead of his second visit.\"\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'He first visited Saudi consulate Istanbul September obtain document certifying divorced ex wife could marry Turkish fiancée But told would return arranged come back October believe something bad happen soil Hatice Cengiz wrote Washington Post Jamal hardly concerned ahead second visit'"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Here we remove dublicate words and special characters as well as english stop words\n",
    "preprocess(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'He first visited Saudi consulate Istanbul September obtain document certifying divorced ex wife could marry Turkish fiancée But told would return arranged come back October He believe something bad could happen Turkish soil fiancée Hatice Cengiz wrote Washington Post Jamal hardly concerned ahead second visit'"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess_with_dublicates(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'keywords': ['consulate',\n",
       "  'Istanbul',\n",
       "  'September',\n",
       "  'document',\n",
       "  'ex',\n",
       "  'wife',\n",
       "  'fiancée',\n",
       "  'October',\n",
       "  'something',\n",
       "  'soil',\n",
       "  'Hatice',\n",
       "  'Cengiz',\n",
       "  'Washington',\n",
       "  'Post',\n",
       "  'Jamal',\n",
       "  'visit']}"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#from the pre-processed text, we extract keywords\n",
    "syntax_text(preprocess(text))[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Token_Tags': [('PRON', 'He'),\n",
       "  ('ADV', 'first'),\n",
       "  ('VERB', 'visited'),\n",
       "  ('ADJ', 'Saudi'),\n",
       "  ('NOUN', 'consulate'),\n",
       "  ('NOUN', 'Istanbul'),\n",
       "  ('NOUN', 'September'),\n",
       "  ('VERB', 'obtain'),\n",
       "  ('NOUN', 'document'),\n",
       "  ('VERB', 'certifying'),\n",
       "  ('VERB', 'divorced'),\n",
       "  ('NOUN', 'ex'),\n",
       "  ('NOUN', 'wife'),\n",
       "  ('VERB', 'could'),\n",
       "  ('VERB', 'marry'),\n",
       "  ('ADJ', 'Turkish'),\n",
       "  ('NOUN', 'fiancée'),\n",
       "  ('CONJ', 'But'),\n",
       "  ('VERB', 'told'),\n",
       "  ('VERB', 'would'),\n",
       "  ('VERB', 'return'),\n",
       "  ('VERB', 'arranged'),\n",
       "  ('VERB', 'come'),\n",
       "  ('ADV', 'back'),\n",
       "  ('NOUN', 'October'),\n",
       "  ('VERB', 'believe'),\n",
       "  ('NOUN', 'something'),\n",
       "  ('ADJ', 'bad'),\n",
       "  ('VERB', 'happen'),\n",
       "  ('NOUN', 'soil'),\n",
       "  ('NOUN', 'Hatice'),\n",
       "  ('NOUN', 'Cengiz'),\n",
       "  ('VERB', 'wrote'),\n",
       "  ('NOUN', 'Washington'),\n",
       "  ('NOUN', 'Post'),\n",
       "  ('NOUN', 'Jamal'),\n",
       "  ('ADV', 'hardly'),\n",
       "  ('ADJ', 'concerned'),\n",
       "  ('ADP', 'ahead'),\n",
       "  ('ADJ', 'second'),\n",
       "  ('NOUN', 'visit')]}"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#from the preprocessed text, for each word in the text we print the word and its corresponding tag\n",
    "syntax_text(preprocess(text))[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAE8RJREFUeJzt3X+wXGd93/H3B/kXBQdkdEOpJVlm4lJMMDa5I6DuBNOCkPPDooVp7aHBJng0oThu0zStaTq2a09akrTQEByMUhQDbSzKr0YtIrLBIa7rOtY1KDa246Aobn0RGQvLJTh27ZHz7R97lBxf7dVd3btXV9Lzfs3s3D3Pj7PPHu1+9ujZs+ekqpAkteN5Sz0ASdKRZfBLUmMMfklqjMEvSY0x+CWpMQa/JDXG4Jekxhj8ktQYg1+SGnPCUg9gmBUrVtSaNWuWehiSdMy45557vlNVE6O0PSqDf82aNUxNTS31MCTpmJHkf4/a1qkeSWqMwS9JjTH4JakxBr8kNcbgl6TGzBn8SVYl+Z0kDya5P8k/HtImST6cZFeSe5O8tld3aZJvdrdLx/0EJEmHZ5TDOfcDP1tVX0tyKnBPklur6oFemwuBs7rb64CPAq9LchpwDTAJVNd3a1U9PtZnIUka2Zx7/FX17ar6Wnf/e8CDwOkzmm0APlkDdwEvTvIy4K3ArVW1rwv7W4H1Y30GkqTDclhz/EnWAOcBvzej6nTgkd7ydFc2W7kkaYmM/MvdJC8EPgf8k6r605nVQ7rUIcqHrX8jsBFg9erVow7rIGuu+uK8+x4PHv7Ajy6ov9vP7bcQbr+FWej2G9VIe/xJTmQQ+v+5qj4/pMk0sKq3vBLYc4jyg1TVpqqarKrJiYmRTjchSZqHUY7qCfBx4MGq+uAszbYC7+qO7nk98N2q+jawHViXZHmS5cC6rkyStERGmeo5H/gJ4L4kO7uyfwmsBqiqG4FtwI8Au4AngXd3dfuSXA/s6PpdV1X7xjd8SdLhmjP4q+oOhs/V99sU8L5Z6jYDm+c1OknS2PnLXUlqjMEvSY0x+CWpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWrMnFfgSrIZ+DHg0ar6wSH1Pwe8s7e+VwIT3WUXHwa+BzwL7K+qyXENXJI0P6Ps8d8ErJ+tsqp+uarOrapzgfcDvzvjurpv6uoNfUk6CswZ/FV1OzDqBdIvAW5e0IgkSYtqbHP8Sf4Kg/8ZfK5XXMAtSe5JsnFcjyVJmr855/gPw48D/3PGNM/5VbUnyfcDtyb5g+5/EAfpPhg2AqxevXqMw5Ik9Y3zqJ6LmTHNU1V7ur+PAl8A1s7Wuao2VdVkVU1OTEyMcViSpL6xBH+SFwFvBH6rV/aCJKceuA+sA74xjseTJM3fKIdz3gxcAKxIMg1cA5wIUFU3ds3+LnBLVf1Zr+tLgS8kOfA4v1lVvz2+oUuS5mPO4K+qS0ZocxODwz77ZbuB18x3YJKkxeEvdyWpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxcwZ/ks1JHk0y9Hq5SS5I8t0kO7vb1b269UkeSrIryVXjHLgkaX5G2eO/CVg/R5v/UVXndrfrAJIsA24ALgTOBi5JcvZCBitJWrg5g7+qbgf2zWPda4FdVbW7qp4BtgAb5rEeSdIYjWuO/w1Jfj/Jl5K8qis7HXik12a6KxsqycYkU0mm9u7dO6ZhSZJmGkfwfw04o6peA/wq8F+78gxpW7OtpKo2VdVkVU1OTEyMYViSpGEWHPxV9adV9UR3fxtwYpIVDPbwV/WargT2LPTxJEkLs+DgT/JXk6S7v7Zb52PADuCsJGcmOQm4GNi60MeTJC3MCXM1SHIzcAGwIsk0cA1wIkBV3Qi8A3hvkv3AU8DFVVXA/iRXANuBZcDmqrp/UZ6FJGlkcwZ/VV0yR/1HgI/MUrcN2Da/oUmSFoO/3JWkxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGzBn8STYneTTJN2apf2eSe7vbnUle06t7OMl9SXYmmRrnwCVJ8zPKHv9NwPpD1P8x8MaqOge4Htg0o/5NVXVuVU3Ob4iSpHEa5Zq7tydZc4j6O3uLdwErFz4sSdJiGfcc/3uAL/WWC7glyT1JNh6qY5KNSaaSTO3du3fMw5IkHTDnHv+okryJQfD/rV7x+VW1J8n3A7cm+YOqun1Y/6raRDdNNDk5WeMalyTpucayx5/kHOA/Ahuq6rED5VW1p/v7KPAFYO04Hk+SNH8LDv4kq4HPAz9RVX/YK39BklMP3AfWAUOPDJIkHTlzTvUkuRm4AFiRZBq4BjgRoKpuBK4GXgL8WhKA/d0RPC8FvtCVnQD8ZlX99iI8B0nSYRjlqJ5L5qi/HLh8SPlu4DUH95AkLSV/uStJjTH4JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmMMfklqjMEvSY0x+CWpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNGSn4k2xO8miSodfMzcCHk+xKcm+S1/bqLk3yze526bgGLkman1H3+G8C1h+i/kLgrO62EfgoQJLTGFyj93XAWuCaJMvnO1hJ0sKNFPxVdTuw7xBNNgCfrIG7gBcneRnwVuDWqtpXVY8Dt3LoDxBJ0iIb1xz/6cAjveXprmy28oMk2ZhkKsnU3r17xzQsSdJM4wr+DCmrQ5QfXFi1qaomq2pyYmJiTMOSJM00ruCfBlb1llcCew5RLklaIuMK/q3Au7qje14PfLeqvg1sB9YlWd59qbuuK5MkLZETRmmU5GbgAmBFkmkGR+qcCFBVNwLbgB8BdgFPAu/u6vYluR7Y0a3quqo61JfEkqRFNlLwV9Ulc9QX8L5Z6jYDmw9/aJKkxeAvdyWpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjRkp+JOsT/JQkl1JrhpS/6EkO7vbHyb5v726Z3t1W8c5eEnS4ZvzClxJlgE3AG9hcPH0HUm2VtUDB9pU1c/02v80cF5vFU9V1bnjG7IkaSFG2eNfC+yqqt1V9QywBdhwiPaXADePY3CSpPEbJfhPBx7pLU93ZQdJcgZwJnBbr/iUJFNJ7krytnmPVJI0FqNcbD1DymqWthcDn62qZ3tlq6tqT5KXA7clua+q/uigB0k2AhsBVq9ePcKwJEnzMcoe/zSwqre8EtgzS9uLmTHNU1V7ur+7ga/y3Pn/frtNVTVZVZMTExMjDEuSNB+jBP8O4KwkZyY5iUG4H3R0TpJXAMuB/9UrW57k5O7+CuB84IGZfSVJR86cUz1VtT/JFcB2YBmwuaruT3IdMFVVBz4ELgG2VFV/GuiVwMeS/DmDD5kP9I8GkiQdeaPM8VNV24BtM8qunrF87ZB+dwKvXsD4JElj5i93JakxBr8kNcbgl6TGGPyS1BiDX5IaY/BLUmMMfklqjMEvSY0x+CWpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEjBX+S9UkeSrIryVVD6i9LsjfJzu52ea/u0iTf7G6XjnPwkqTDN+elF5MsA24A3gJMAzuSbB1y7dxPV9UVM/qeBlwDTAIF3NP1fXwso5ckHbZR9vjXAruqandVPQNsATaMuP63ArdW1b4u7G8F1s9vqJKkcRgl+E8HHuktT3dlM709yb1JPptk1WH2lSQdIaMEf4aU1Yzl/wasqapzgC8DnziMvoOGycYkU0mm9u7dO8KwJEnzMUrwTwOressrgT39BlX1WFU93S3+OvBDo/btrWNTVU1W1eTExMQoY5ckzcMowb8DOCvJmUlOAi4GtvYbJHlZb/Ei4MHu/nZgXZLlSZYD67oySdISmfOonqran+QKBoG9DNhcVfcnuQ6YqqqtwJVJLgL2A/uAy7q++5Jcz+DDA+C6qtq3CM9DkjSiOYMfoKq2AdtmlF3du/9+4P2z9N0MbF7AGCVJY+QvdyWpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWqMwS9JjTH4JakxIwV/kvVJHkqyK8lVQ+r/aZIHktyb5CtJzujVPZtkZ3fbOrOvJOnImvPSi0mWATcAbwGmgR1JtlbVA71mXwcmq+rJJO8Ffgn4B13dU1V17pjHLUmap1H2+NcCu6pqd1U9A2wBNvQbVNXvVNWT3eJdwMrxDlOSNC6jBP/pwCO95emubDbvAb7UWz4lyVSSu5K8bR5jlCSN0ZxTPUCGlNXQhsk/BCaBN/aKV1fVniQvB25Lcl9V/dGQvhuBjQCrV68eYViSpPkYZY9/GljVW14J7JnZKMmbgZ8HLqqqpw+UV9We7u9u4KvAecMepKo2VdVkVU1OTEyM/AQkSYdnlODfAZyV5MwkJwEXA885OifJecDHGIT+o73y5UlO7u6vAM4H+l8KS5KOsDmneqpqf5IrgO3AMmBzVd2f5Dpgqqq2Ar8MvBD4TBKA/1NVFwGvBD6W5M8ZfMh8YMbRQJKkI2yUOX6qahuwbUbZ1b37b56l353AqxcyQEnSePnLXUlqjMEvSY0x+CWpMQa/JDXG4Jekxhj8ktQYg1+SGmPwS1JjDH5JaozBL0mNMfglqTEGvyQ1xuCXpMYY/JLUGINfkhpj8EtSYwx+SWrMSMGfZH2Sh5LsSnLVkPqTk3y6q/+9JGt6de/vyh9K8tbxDV2SNB9zBn+SZcANwIXA2cAlSc6e0ew9wONV9QPAh4Bf7PqezeDi7K8C1gO/1q1PkrRERtnjXwvsqqrdVfUMsAXYMKPNBuAT3f3PAn8ng6uubwC2VNXTVfXHwK5ufZKkJTJK8J8OPNJbnu7Khrapqv3Ad4GXjNhXknQEnTBCmwwpqxHbjNJ3sIJkI7CxW3wiyUMjjO1otAL4zlI9eH5xqR55bNx+C+P2W5hjefudMWrDUYJ/GljVW14J7JmlzXSSE4AXAftG7AtAVW0CNo027KNXkqmqmlzqcRyr3H4L4/ZbmFa23yhTPTuAs5KcmeQkBl/Wbp3RZitwaXf/HcBtVVVd+cXdUT9nAmcBd49n6JKk+Zhzj7+q9ie5AtgOLAM2V9X9Sa4DpqpqK/Bx4FNJdjHY07+463t/kv8CPADsB95XVc8u0nORJI0ggx1zjUuSjd20lebB7bcwbr+FaWX7GfyS1BhP2XCEJTkhyRVJTl7qsUhqk8Hfk+TKJA8meXzYqSnGsP4A/wG4t6qeHvf6l0qSlyTZ2d3+JMm3essnjbiO/5TkbUPKfyPJKw7R744k5y5k/MezJGuSfGOWuq8mOe6PYBkmyRMzli9L8pGlGs+RNsrhnC35R8CF3a+Mx6470umKxVj3Uqqqx4BzAZJcCzxRVf9u1P7dIcCzrfvdCx6gpOdwj7+T5Ebg5cDWJD9z4NM/yY93J577epIvJ3lpV35tks3dXtPuJFf21vWuJPcm+f0kn+rKJpJ8LsmO7nZ+V/6Cbj07useYeTqMY1aSH0iys7d8VZJ/1d2/I8kvJLmdGR+GSf5tko8ned6BPfpuiuxTSe5L8o3+9mZwyPDd3YkA/+aReXZHxszXUpIzknylK/tKktVdu5uSvKPX74kh63p+ki1d308Dzz+CT+WYMdt79XjiHn+nqn4qyXrgTcCP9aruAF5fVZXkcuCfAz/b1f2Nrv2pwENJPgr8deDngfOr6jtJTuva/grwoaq6o3uzbgde2bW9rap+MsmLgbuTfLmq/mxxn/FR4fuq6odhMNXT/f0gcDJwebfND7T9IWBFVb26a/fi3npSVWuTXARczeCEgMe8JK/i4NfSJ4BPVtUnkvwk8GHgoCmyWbwXeLKqzklyDvC1RRn4seH5/Z0S4DT+8vdJs71XjxsG/9xWAp9O8jLgJKA/DfTFbq7+6SSPAi8F/jbw2ar6DkBV7evavhk4uxdk35fkVGAdcFGSf9aVnwKsBh5cxOd0tNgyY/lfA3dW1XuHtN0FvCLJrwDbgFt6dZ/v/t4DrBn3IJfQQa+lJG8A/l5X/ynglw5jfT/M4IOCqro3yb3jHOwx5qmq+ovvhpJcBhz4vmPoe7Wqvndkh7h4DP65/SrwwaramuQC4NpeXf8L2mcZbM8w/HxEzwPeUFVP9Qu7L3zfXlXH6rmJDmU/z51OPKUrO2Dm/2ruBiaTLK+qx/sVVfVYt5d6IXAl8Hb+8txOB/4dDvwbHC9mey31Haj/i23dvaZm+1Ld47fnNvS9ejxxjn9uLwK+1d2/9FANO18B/n6SlwD0pnpuoTeX3TsSZTvw092blSTnjWPQR4k/Af5akuVJTgF+dI72XwT+PfDfk7ywX5FkgsGUzmeAa4DXLsaAjzLDXkt30v0yHngng6lIgIcZTIfB4HToJw5Z3+1dH5L8IHDOooz62Dfbe/W4cTztHS2Wa4HPJPkWcBdw5qEad6ep+AXgd5M8C3wduIzBXuoN3X+vT2DwJvwp4Hq6Qzy78H+Y537HcMyqqv+X5N8wON/Tbgan7pirz5ZuCuy3kvQ/KFYBH++2UQH/YjHGfDSZ5bV0JbA5yc8Be4EDRz39OoNtdjeDD4xh3xF9FPiN7jW4E8+bNZvZ3qvHDX+5K0mNcapHkhpj8EtSYwx+SWqMwS9JjTH4JakxBr8kNcbgl6TGGPyS1Jj/DySfuWUS7QhdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#plotting words with a frequency of more than 1\n",
    "plot_word_frequency(preprocess_with_dublicates(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'He': 2,\n",
       " 'first': 1,\n",
       " 'visited': 1,\n",
       " 'Saudi': 1,\n",
       " 'consulate': 1,\n",
       " 'Istanbul': 1,\n",
       " 'September': 1,\n",
       " 'obtain': 1,\n",
       " 'document': 1,\n",
       " 'certifying': 1,\n",
       " 'divorced': 1,\n",
       " 'ex': 1,\n",
       " 'wife': 1,\n",
       " 'could': 2,\n",
       " 'marry': 1,\n",
       " 'Turkish': 2,\n",
       " 'fiancée': 2,\n",
       " 'But': 1,\n",
       " 'told': 1,\n",
       " 'would': 1,\n",
       " 'return': 1,\n",
       " 'arranged': 1,\n",
       " 'come': 1,\n",
       " 'back': 1,\n",
       " 'October': 1,\n",
       " 'believe': 1,\n",
       " 'something': 1,\n",
       " 'bad': 1,\n",
       " 'happen': 1,\n",
       " 'soil': 1,\n",
       " 'Hatice': 1,\n",
       " 'Cengiz': 1,\n",
       " 'wrote': 1,\n",
       " 'Washington': 1,\n",
       " 'Post': 1,\n",
       " 'Jamal': 1,\n",
       " 'hardly': 1,\n",
       " 'concerned': 1,\n",
       " 'ahead': 1,\n",
       " 'second': 1,\n",
       " 'visit': 1}"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_count(preprocess_with_dublicates(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}