Text_Preprocessing_Cosine_Ngram_Merged.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "import os\n",
    "import re\n",
    "import codecs\n",
    "import pathlib\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import nltk\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from nltk.tokenize import word_tokenize\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import datetime\n",
    "from scipy.spatial.distance import cosine\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "# !pip install transformers\n",
    "# !pip install sentence_transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_of_folder_for_processing = 'sampled_raw'\n",
    "Country_to_process = 'IN'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PK, US, HK, GH, NG, KE, NZ, BD, IN, LK, CA, ZA, SG, PH, GB, MY, AU, IE, JM, TZ, "
     ]
    }
   ],
   "source": [
    "base_dir = os.path.join(os.getcwd(), 'data', name_of_folder_for_processing) \n",
    "countries_list = os.listdir(base_dir) # dir is your directory path\n",
    "summary = []\n",
    "\n",
    "for con in countries_list:\n",
    "    if(os.path.isdir(os.path.join(base_dir,con))):\n",
    "        print(con, end=\", \", flush=True)\n",
    "        publishers = os.listdir(os.path.join(base_dir,con))\n",
    "        for pub in publishers:\n",
    "            subdir = os.path.join(base_dir, con, pub)\n",
    "            length = sum([len(files) for r, d, files in os.walk(subdir)])\n",
    "            summary.append([con, pub, length])\n",
    "\n",
    "summary_df = pd.DataFrame(summary, columns = ['country','publisher','count'])     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country</th>\n",
       "      <th>publisher</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1332</th>\n",
       "      <td>IN</td>\n",
       "      <td>sportscafe</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1183</th>\n",
       "      <td>IN</td>\n",
       "      <td>ettech-com</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1264</th>\n",
       "      <td>IN</td>\n",
       "      <td>ettelecom-com</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1199</th>\n",
       "      <td>IN</td>\n",
       "      <td>investing-com</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1250</th>\n",
       "      <td>IN</td>\n",
       "      <td>defence-aviation-post</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1283</th>\n",
       "      <td>IN</td>\n",
       "      <td>livemint</td>\n",
       "      <td>3443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1359</th>\n",
       "      <td>IN</td>\n",
       "      <td>daily-news-analysis</td>\n",
       "      <td>3493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1253</th>\n",
       "      <td>IN</td>\n",
       "      <td>economic-times</td>\n",
       "      <td>4796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1231</th>\n",
       "      <td>IN</td>\n",
       "      <td>the-hindu</td>\n",
       "      <td>7776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1260</th>\n",
       "      <td>IN</td>\n",
       "      <td>times-of-india</td>\n",
       "      <td>21752</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>259 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     country              publisher  count\n",
       "1332      IN             sportscafe     29\n",
       "1183      IN             ettech-com     29\n",
       "1264      IN          ettelecom-com     30\n",
       "1199      IN          investing-com     30\n",
       "1250      IN  defence-aviation-post     30\n",
       "...      ...                    ...    ...\n",
       "1283      IN               livemint   3443\n",
       "1359      IN    daily-news-analysis   3493\n",
       "1253      IN         economic-times   4796\n",
       "1231      IN              the-hindu   7776\n",
       "1260      IN         times-of-india  21752\n",
       "\n",
       "[259 rows x 3 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary_df[summary_df.country=='IN'].sort_values(by=['count'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# idx = summary_df.groupby(['country'])['count'].transform(max) == summary_df['count']\n",
    "# max_table = summary_df[idx]\n",
    "# max_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, IN, "
     ]
    }
   ],
   "source": [
    "overall_dict = {}\n",
    "\n",
    "for index, row in summary_df.iterrows():\n",
    "    country, publisher = row[0], row[1]\n",
    "    if country != Country_to_process:\n",
    "        pass\n",
    "    else:\n",
    "        \n",
    "        print(country, end=\", \", flush=True)\n",
    "        directory = os.path.join(base_dir, country, publisher)\n",
    "        df_dict = {\"id\": [], \"title\": [], \"publisher\": [], \"article_text\":[],\"url\":[], \"path\":[]}\n",
    "        if not os.path.isdir(directory):\n",
    "            pass\n",
    "        else:\n",
    "            for entry in os.scandir(directory):\n",
    "                if(os.path.isdir(entry)):\n",
    "                    for entry_2 in os.scandir(entry.path):\n",
    "                        try:\n",
    "                            with open(entry_2, \"r\") as f:\n",
    "                                article_id = f.readline().strip()\n",
    "                                article_title = f.readline().strip()\n",
    "                                publisher = f.readline().strip()\n",
    "                                url = f.readline()\n",
    "                                f.readline()\n",
    "                                article_text = f.readline().strip()\n",
    "                                df_dict['url'].append(url)\n",
    "                                df_dict['id'].append(article_id)\n",
    "                                df_dict['title'].append(article_title)\n",
    "                                df_dict['publisher'].append(publisher)\n",
    "                                df_dict['article_text'].append(article_text)\n",
    "                                df_dict['path'].append(entry_2)\n",
    "                        except:\n",
    "                            pass\n",
    "            sample_df = pd.DataFrame.from_dict(df_dict)\n",
    "            #sample_df = sample_df.sample(n=5, random_state=1)\n",
    "\n",
    "            if country not in overall_dict.keys():\n",
    "                overall_dict[country] = {}\n",
    "            overall_dict[country][publisher] = sample_df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import a clean function data\n",
    "def clean_text(text):\n",
    "    return text.strip()\n",
    "\n",
    "def text_initial_preproecss(df):\n",
    "\n",
    "    df = df.copy()\n",
    "    df = df[~pd.to_numeric(df['id'], errors='coerce').isnull()]\n",
    "    df[\"id\"] = df[\"id\"].astype(int)\n",
    "    df['article_text'] = df.article_text.str.lower()\n",
    "    df[\"article_text\"] = df[\"article_text\"].apply(clean_text)\n",
    "    df['text'] = df['article_text']\n",
    "    reports = df\n",
    "    \n",
    "    # Puncutation preprocesing\n",
    "    reports['text'] = reports.text.str.replace('{', '')\n",
    "    reports['text'] = reports.text.str.replace('}', '')\n",
    "    reports['text'] = reports.text.str.replace(\"\\n\", '')\n",
    "    reports['text'] = reports.text.str.rstrip(\"\\n\") #remove empty lines\n",
    "    reports['text'] = reports.text.str.replace(\"@ @ @ @ @ @ @ @ @ @ \", '')\n",
    "    reports['text'] = reports.text.str.replace(\" @\", '')\n",
    "    reports['text'] = reports.text.str.replace(\" '\", \"'\")\n",
    "    reports['text'] = reports.text.str.replace(\"\\\"\", \"\")\n",
    "    reports['text'] = reports.text.str.replace(\",\", \"\")\n",
    "    reports['text'] = reports.text.str.replace(\"(\", \"\")\n",
    "    reports['text'] = reports.text.str.replace(\")\", \"\")\n",
    "    reports['text'] = reports.text.str.replace(\" <p>\", \".\")\n",
    "    reports['text'] = reports.text.str.replace(\" <h>\", \".\")\n",
    "    reports['text'] = reports.text.str.replace(\"<p>\", \"\")\n",
    "    reports['text'] = reports.text.str.replace(\"<h>\", \"\")\n",
    "    reports['text'] = reports.text.str.replace('<', '')\n",
    "    reports['text'] = reports.text.str.replace('>', '')\n",
    "    reports['text'] = reports.text.str.replace(\":\", \"\")\n",
    "    reports['text'] = reports.text.str.replace(\"?\", \".\")\n",
    "    reports['text'] = reports.text.str.replace(\"!\", \".\")\n",
    "    reports['text'] = reports.text.str.replace(r\"\\.\\s[\\.\\s]+\", \". \") #converting . . to .\n",
    "    reports['text'] = reports.text.str.replace(r\"\\.+\", \".\") #converting ... to .\n",
    "    reports['text'] = reports.text.str.replace(\"--\", \"\") \n",
    "    reports['text'] = reports.text.str.replace(\"-\", \" \")\n",
    "    reports['text'] = reports.text.str.replace(\" +\", \" \")\n",
    "    reports['text'] = reports.text.str.replace(\" n't\", \"n't\")\n",
    "    \n",
    "    return reports.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import platform\n",
    "os_name = platform.system()\n",
    "\n",
    "def save_file(report,base):\n",
    "#     print(report)\n",
    "#     print(type(report))\n",
    "    report = report.squeeze()\n",
    "        \n",
    "    full_path = report.publisher\n",
    "    file_name = str(report.id) + \".txt\"\n",
    "    \n",
    "    if os_name == 'Windows':\n",
    "        \n",
    "        base_dir = 'data\\preprocessed\\\\{}'.format(base)\n",
    "        con = report.path.path.split('{0}\\\\'.format(name_of_folder_for_processing))[1]\n",
    "        \n",
    "    else:\n",
    "        \n",
    "        base_dir = 'data/preprocessed/{}'.format(base)\n",
    "        con = report.path.path.split('/{0}/'.format(name_of_folder_for_processing))[1]\n",
    "    \n",
    "    path = os.path.join(base_dir,con)\n",
    "    \n",
    "    full_path = os.path.dirname(path)\n",
    "    file_name = os.path.basename(path)\n",
    "\n",
    "    pathlib.Path(full_path).mkdir(parents=True, exist_ok=True)\n",
    "    \n",
    "    with codecs.open(f\"{full_path}/{file_name}\", \"w\", encoding = \"utf-8\") as f:\n",
    "        id_number = str(report.id) if not pd.isna(report.id) else \"\"\n",
    "        title = report.title if not pd.isna(report.title) else \"\"\n",
    "        website = report.publisher if not pd.isna(report.publisher) else \"\"\n",
    "        url = report.url if not pd.isna(report.url) else \"\"\n",
    "        f.writelines([id_number, \"\\n\",\n",
    "                      title, \"\\n\",\n",
    "                      website, \"\\n\",\n",
    "                      url, \"\\n\\n\", report.text])\n",
    "        f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ngram_preprocess(df):\n",
    "    time_taken = []\n",
    "    \n",
    "#     reports_temp = df[df['publisher']==article][['id', 'text','text_vect']].copy()\n",
    "    if len(df) > 500:\n",
    "        reports_temp = df.sample(n=500, random_state=1).copy()\n",
    "    else:\n",
    "        reports_temp = df.copy()\n",
    "        \n",
    "    reports_temp['text_vect'] = reports_temp.text.apply(sent_tokenize)\n",
    "#     display(reports_temp.head(3))\n",
    "    word_vectorizer = CountVectorizer(ngram_range=(5,5), stop_words=[])\n",
    "    flatten = [item for sublist in reports_temp['text_vect'] for item in sublist]\n",
    "    sparse_matrix = word_vectorizer.fit_transform(flatten)\n",
    "\n",
    "    frequency = sum(sparse_matrix).toarray()[0]\n",
    "    frequency_df = pd.DataFrame(frequency, index=word_vectorizer.get_feature_names(),columns = ['frequency']).sort_values(by=['frequency'],ascending=False)\n",
    "\n",
    "    freq_above_10 = frequency_df[frequency_df['frequency'] >= max((len(reports_temp)/10),3)] #len(MAXSIZE)/2\n",
    "    phrase_list = list(freq_above_10.index)\n",
    "\n",
    "    sentences_to_remove = []\n",
    "\n",
    "    for phrase in phrase_list:\n",
    "#         removing_sent = set([sent for sent in flatten if phrase in sent])\n",
    "        removing_sent = set([sent for sent in flatten if ((phrase in sent) or (len(sent.split()) < 4))])\n",
    "\n",
    "        for sent in removing_sent:\n",
    "            sentences_to_remove.append(sent)\n",
    "\n",
    "    if(\".\" in sentences_to_remove):\n",
    "        while(\".\" in sentences_to_remove):\n",
    "            sentences_to_remove.remove(\".\")\n",
    "    if(\" .\" in sentences_to_remove):\n",
    "        while(\" .\" in sentences_to_remove):\n",
    "            sentences_to_remove.remove(\" .\")\n",
    "\n",
    "#    print(sentences_to_remove)\n",
    "    sentences_to_remove = list(set(sentences_to_remove))\n",
    "    sentences_to_remove.sort(key= len, reverse = True)  # sort starting by largest sentence, in case smaller sentence get chosen beforehand\n",
    "\n",
    "    for sent in sentences_to_remove:\n",
    "        df.text = df.text.apply(lambda x: str(x).replace(sent, \"\"))\n",
    "\n",
    "#     print(sentences_to_remove)\n",
    "    return df\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cosine_preprocess(df, model, threshold):\n",
    "\n",
    "    sent_removed = []\n",
    "    time_taken = []\n",
    "    new_txts = []\n",
    "    \n",
    "    for i in range(len(df)):\n",
    "        \n",
    "        idx_remove = []\n",
    "        orig_txt = df['text'].iloc[i]\n",
    "        sent_text = nltk.sent_tokenize(orig_txt) #Sentence Segmentation\n",
    "        docu_embeddings = model.encode(orig_txt) #Document Embedding\n",
    "        sentence_embeddings_ = model.encode(sent_text) #Sentence Embedding\n",
    "\n",
    "        for k in range(len(sentence_embeddings_)):\n",
    "            \n",
    "            sim = cosine(docu_embeddings, sentence_embeddings_[k])\n",
    "\n",
    "            if sim > threshold:\n",
    "                idx_remove.append(k)\n",
    "\n",
    "        idx_remove.sort()\n",
    "        idx_remove.reverse()\n",
    "\n",
    "        for j in idx_remove:\n",
    "            sent_removed.append(sent_text[j])\n",
    "            del sent_text[j]\n",
    "\n",
    "        new_txt = ' '.join(sent_text)\n",
    "        new_txts.append(new_txt)\n",
    "        \n",
    "    df['text'] = new_txts\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving original file, to do comparison later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "for keys1 in overall_dict.keys():\n",
    "    for keys2 in overall_dict[keys1].keys():\n",
    "        df = overall_dict[keys1][keys2]\n",
    "        df = text_initial_preproecss(df)\n",
    "\n",
    "        df.apply(save_file,args=('orig',),axis=1)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /Users/hojinlee/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /Users/hojinlee/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     /Users/hojinlee/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import stopwords, wordnet\n",
    "import string\n",
    "from textblob import TextBlob, Word\n",
    "\n",
    "nltk.download('stopwords')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('wordnet')\n",
    "\n",
    "stopwords.words('english')\n",
    "\n",
    "stop_words = set(stopwords.words('english'))\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "def remove_stopwords(text):\n",
    "    word_tokens = word_tokenize(text) \n",
    "    filtered_sentence = [w for w in word_tokens if not w in stop_words]\n",
    "    return ' '.join(filtered_sentence)\n",
    "\n",
    "def remove_digits(text):\n",
    "    text = re.sub(r'\\d+','', text)\n",
    "    return text\n",
    "\n",
    "def remove_punct(text):\n",
    "    word_tokens = word_tokenize(text) \n",
    "    filtered_sentence = [w for w in word_tokens if not w in string.punctuation]\n",
    "    return ' '.join(filtered_sentence)\n",
    "\n",
    "def get_wordnet_pos(word):\n",
    "    \"\"\"Map POS tag to first character lemmatize() accepts\"\"\"\n",
    "    tag = nltk.pos_tag([word])[0][1][0].upper()\n",
    "    tag_dict = {\"J\": wordnet.ADJ,\n",
    "                \"N\": wordnet.NOUN,\n",
    "                \"V\": wordnet.VERB,\n",
    "                \"R\": wordnet.ADV}\n",
    "\n",
    "    return tag_dict.get(tag, wordnet.NOUN)\n",
    "\n",
    "def nltk_lemmatizer(text):\n",
    "    '''\n",
    "    lemmatize each word in sentence if the word is not punctuation\n",
    "    Below version is just for ease of understanding\n",
    "    '''\n",
    "    words = word_tokenize(text)\n",
    "    lem_words = []\n",
    "    for word in words:\n",
    "        if word not in string.punctuation:\n",
    "            lem_words.append(lemmatizer.lemmatize(word, get_wordnet_pos(word)))\n",
    "\n",
    "    return ' '.join(lem_words)\n",
    "\n",
    "def lemmatize_with_postag(text):\n",
    "    sent = TextBlob(text)\n",
    "    tag_dict = {\"J\": 'a', \n",
    "                \"N\": 'n', \n",
    "                \"V\": 'v', \n",
    "                \"R\": 'r'}\n",
    "    words_and_tags = [(w, tag_dict.get(pos[0], 'n')) for w, pos in sent.tags]    \n",
    "    lemmatized_list = [wd.lemmatize(tag) for wd, tag in words_and_tags]\n",
    "    return \" \".join(lemmatized_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## N-Gram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Methodology: ngram\n",
      "Running time, country: IN, publisher: News Nation, punct_t: 0:00:00.157136, text_proc_t: 0:00:32.392307, total_t: 0:00:32.549443\n",
      "Running time, country: IN, publisher: Gulte, punct_t: 0:00:00.080560, text_proc_t: 0:00:04.756247, total_t: 0:00:04.836807\n",
      "Running time, country: IN, publisher: Moneycontrol.com, punct_t: 0:00:01.128391, text_proc_t: 0:02:38.011705, total_t: 0:02:39.140096\n",
      "Running time, country: IN, publisher: oneindia.com, punct_t: 0:00:00.065014, text_proc_t: 0:00:05.838386, total_t: 0:00:05.903400\n",
      "Running time, country: IN, publisher: Businessinsider India, punct_t: 0:00:00.048450, text_proc_t: 0:00:03.964102, total_t: 0:00:04.012552\n",
      "Running time, country: IN, publisher: Telegraph India, punct_t: 0:00:00.100234, text_proc_t: 0:00:16.385720, total_t: 0:00:16.485954\n",
      "Running time, country: IN, publisher: Reuters India, punct_t: 0:00:00.637056, text_proc_t: 0:01:50.688348, total_t: 0:01:51.325404\n",
      "Running time, country: IN, publisher: afaqs, punct_t: 0:00:00.073544, text_proc_t: 0:00:08.921688, total_t: 0:00:08.995232\n",
      "Running time, country: IN, publisher: GSMArena.com, punct_t: 0:00:00.150159, text_proc_t: 0:00:25.684602, total_t: 0:00:25.834761\n",
      "Running time, country: IN, publisher: NewsClick, punct_t: 0:00:00.058558, text_proc_t: 0:00:05.791579, total_t: 0:00:05.850137\n",
      "Running time, country: IN, publisher: outlookindia.com, punct_t: 0:00:00.064864, text_proc_t: 0:00:08.003376, total_t: 0:00:08.068240\n",
      "Running time, country: IN, publisher: koimoi, punct_t: 0:00:00.063942, text_proc_t: 0:00:07.885953, total_t: 0:00:07.949895\n",
      "Running time, country: IN, publisher: mysmartprice.com, punct_t: 0:00:00.077153, text_proc_t: 0:00:12.559100, total_t: 0:00:12.636253\n",
      "Running time, country: IN, publisher: dailyo.in, punct_t: 0:00:00.089199, text_proc_t: 0:00:13.783975, total_t: 0:00:13.873174\n",
      "Running time, country: IN, publisher: Doordarshan, punct_t: 0:00:00.024630, text_proc_t: 0:00:01.105347, total_t: 0:00:01.129977\n",
      "Running time, country: IN, publisher: Daily Excelsior, punct_t: 0:00:00.148416, text_proc_t: 0:00:30.213114, total_t: 0:00:30.361530\n",
      "Running time, country: IN, publisher: telegraphindia.com, punct_t: 0:00:00.103974, text_proc_t: 0:00:16.590745, total_t: 0:00:16.694719\n",
      "Running time, country: IN, publisher: Cogencis, punct_t: 0:00:00.042039, text_proc_t: 0:00:03.916126, total_t: 0:00:03.958165\n",
      "Running time, country: IN, publisher: in.reuters.com, punct_t: 0:00:00.061491, text_proc_t: 0:00:06.355333, total_t: 0:00:06.416824\n",
      "Running time, country: IN, publisher: WION, punct_t: 0:00:00.027318, text_proc_t: 0:00:01.544299, total_t: 0:00:01.571617\n",
      "Running time, country: IN, publisher: Washington Bangla Radio, punct_t: 0:00:00.116955, text_proc_t: 0:00:20.987015, total_t: 0:00:21.103970\n",
      "Running time, country: IN, publisher: newsclick.in, punct_t: 0:00:00.129758, text_proc_t: 0:00:29.962864, total_t: 0:00:30.092622\n",
      "Running time, country: IN, publisher: Kashmir Reader, punct_t: 0:00:00.033735, text_proc_t: 0:00:02.097305, total_t: 0:00:02.131040\n",
      "Running time, country: IN, publisher: businesswireindia.com, punct_t: 0:00:00.101935, text_proc_t: 0:00:20.270907, total_t: 0:00:20.372842\n",
      "Running time, country: IN, publisher: Indian Autos Blog, punct_t: 0:00:00.107317, text_proc_t: 0:00:12.609606, total_t: 0:00:12.716923\n",
      "Running time, country: IN, publisher: ESPNcricinfo.com, punct_t: 0:00:01.525326, text_proc_t: 1:32:37.224120, total_t: 1:32:38.749446\n",
      "Running time, country: IN, publisher: Jagran Josh, punct_t: 0:00:00.034621, text_proc_t: 0:00:02.819242, total_t: 0:00:02.853863\n",
      "Running time, country: IN, publisher: CounterCurrents.org, punct_t: 0:00:00.347786, text_proc_t: 0:04:21.062230, total_t: 0:04:21.410016\n",
      "Running time, country: IN, publisher: Oneindia, punct_t: 0:00:00.063243, text_proc_t: 0:00:06.547007, total_t: 0:00:06.610250\n",
      "Running time, country: IN, publisher: cnbctv18.com, punct_t: 0:00:00.071919, text_proc_t: 0:00:08.160500, total_t: 0:00:08.232419\n",
      "Running time, country: IN, publisher: theweek.in, punct_t: 0:00:00.091442, text_proc_t: 0:00:13.508261, total_t: 0:00:13.599703\n",
      "Running time, country: IN, publisher: Sify News, punct_t: 0:00:00.027259, text_proc_t: 0:00:01.844704, total_t: 0:00:01.871963\n",
      "Running time, country: IN, publisher: The Telegraph, punct_t: 0:00:00.039364, text_proc_t: 0:00:03.097970, total_t: 0:00:03.137334\n",
      "Running time, country: IN, publisher: freepressjournal.in, punct_t: 0:00:00.079908, text_proc_t: 0:00:11.638430, total_t: 0:00:11.718338\n",
      "Running time, country: IN, publisher: cartoq.com, punct_t: 0:00:00.071906, text_proc_t: 0:00:07.716030, total_t: 0:00:07.787936\n",
      "Running time, country: IN, publisher: Hindustan Times, punct_t: 0:00:00.311012, text_proc_t: 0:00:55.697165, total_t: 0:00:56.008177\n",
      "Running time, country: IN, publisher: Press Trust of India, punct_t: 0:00:00.049123, text_proc_t: 0:00:03.924231, total_t: 0:00:03.973354\n",
      "Running time, country: IN, publisher: The Hans India, punct_t: 0:00:00.126017, text_proc_t: 0:00:22.675869, total_t: 0:00:22.801886\n",
      "Running time, country: IN, publisher: indiatvnews.com, punct_t: 0:00:00.111490, text_proc_t: 0:00:17.695375, total_t: 0:00:17.806865\n",
      "Running time, country: IN, publisher: Live Law, punct_t: 0:00:00.070616, text_proc_t: 0:00:08.006559, total_t: 0:00:08.077175\n",
      "Running time, country: IN, publisher: ETtech.com, punct_t: 0:00:00.023420, text_proc_t: 0:00:00.996852, total_t: 0:00:01.020272\n",
      "Running time, country: IN, publisher: timesnownews.com, punct_t: 0:00:00.083655, text_proc_t: 0:00:12.374219, total_t: 0:00:12.457874\n",
      "Running time, country: IN, publisher: navhindtimes.in, punct_t: 0:00:00.055009, text_proc_t: 0:00:04.573156, total_t: 0:00:04.628165\n",
      "Running time, country: IN, publisher: Sportskeeda, punct_t: 0:00:00.094125, text_proc_t: 0:00:15.488928, total_t: 0:00:15.583053\n",
      "Running time, country: IN, publisher: Merinews, punct_t: 0:00:00.043668, text_proc_t: 0:00:03.587039, total_t: 0:00:03.630707\n",
      "Running time, country: IN, publisher: The Hitavada, punct_t: 0:00:00.020644, text_proc_t: 0:00:01.000889, total_t: 0:00:01.021533\n",
      "Running time, country: IN, publisher: India Times, punct_t: 0:00:00.033580, text_proc_t: 0:00:02.247704, total_t: 0:00:02.281284\n",
      "Running time, country: IN, publisher: http://www.newsgram.com/, punct_t: 0:00:00.027536, text_proc_t: 0:00:01.604005, total_t: 0:00:01.631541\n",
      "Running time, country: IN, publisher: thenorthlines.com, punct_t: 0:00:00.073183, text_proc_t: 0:00:12.359580, total_t: 0:00:12.432763\n",
      "Running time, country: IN, publisher: GlamSham, punct_t: 0:00:00.098025, text_proc_t: 0:00:14.453482, total_t: 0:00:14.551507\n",
      "Running time, country: IN, publisher: zeebiz.com, punct_t: 0:00:00.068332, text_proc_t: 0:00:06.005909, total_t: 0:00:06.074241\n",
      "Running time, country: IN, publisher: Mumbai Mirror, punct_t: 0:00:00.065621, text_proc_t: 0:00:07.328881, total_t: 0:00:07.394502\n",
      "Running time, country: IN, publisher: openpr.com, punct_t: 0:00:00.060741, text_proc_t: 0:00:09.154920, total_t: 0:00:09.215661\n",
      "Running time, country: IN, publisher: exchange4media.com, punct_t: 0:00:00.048587, text_proc_t: 0:00:03.986208, total_t: 0:00:04.034795\n",
      "Running time, country: IN, publisher: inUth.com, punct_t: 0:00:00.019526, text_proc_t: 0:00:00.988617, total_t: 0:00:01.008143\n",
      "Running time, country: IN, publisher: The Arunachal Times, punct_t: 0:00:00.027167, text_proc_t: 0:00:01.666636, total_t: 0:00:01.693803\n",
      "Running time, country: IN, publisher: Investing.com, punct_t: 0:00:00.029453, text_proc_t: 0:00:02.071730, total_t: 0:00:02.101183\n",
      "Running time, country: IN, publisher: The News Minute, punct_t: 0:00:00.021366, text_proc_t: 0:00:01.065330, total_t: 0:00:01.086696\n",
      "Running time, country: IN, publisher: Reuters, punct_t: 0:00:00.149685, text_proc_t: 0:00:36.224708, total_t: 0:00:36.374393\n",
      "Running time, country: IN, publisher: Newsx, punct_t: 0:00:00.033582, text_proc_t: 0:00:01.860221, total_t: 0:00:01.893803\n",
      "Running time, country: IN, publisher: India Today, punct_t: 0:00:00.729101, text_proc_t: 0:01:23.297114, total_t: 0:01:24.026215\n",
      "Running time, country: IN, publisher: indiablooms, punct_t: 0:00:00.033217, text_proc_t: 0:00:02.235794, total_t: 0:00:02.269011\n",
      "Running time, country: IN, publisher: digit.in, punct_t: 0:00:00.100529, text_proc_t: 0:00:17.668468, total_t: 0:00:17.768997\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running time, country: IN, publisher: thebetterindia.com, punct_t: 0:00:00.088195, text_proc_t: 0:00:12.042301, total_t: 0:00:12.130496\n",
      "Running time, country: IN, publisher: Forbes India, punct_t: 0:00:00.096277, text_proc_t: 0:00:12.987669, total_t: 0:00:13.083946\n",
      "Running time, country: IN, publisher: Indian Express, punct_t: 0:00:00.290865, text_proc_t: 0:00:33.499994, total_t: 0:00:33.790859\n",
      "Running time, country: IN, publisher: Fibre2fashion.com, punct_t: 0:00:00.107338, text_proc_t: 0:00:15.123542, total_t: 0:00:15.230880\n",
      "Running time, country: IN, publisher: economictimes.indiatimes.com, punct_t: 0:00:00.065262, text_proc_t: 0:00:07.560310, total_t: 0:00:07.625572\n",
      "Running time, country: IN, publisher: Scroll.in, punct_t: 0:00:00.463942, text_proc_t: 0:02:27.105599, total_t: 0:02:27.569541\n",
      "Running time, country: IN, publisher: firstpost.com, punct_t: 0:00:00.076818, text_proc_t: 0:00:08.283408, total_t: 0:00:08.360226\n",
      "Running time, country: IN, publisher: Assam Tribune, punct_t: 0:00:00.035108, text_proc_t: 0:00:02.001496, total_t: 0:00:02.036604\n",
      "Running time, country: IN, publisher: theprint.in, punct_t: 0:00:00.098698, text_proc_t: 0:00:18.076464, total_t: 0:00:18.175162\n",
      "Running time, country: IN, publisher: HITC, punct_t: 0:00:00.073920, text_proc_t: 0:00:08.170076, total_t: 0:00:08.243996\n",
      "Running time, country: IN, publisher: The Navhind Times, punct_t: 0:00:00.039711, text_proc_t: 0:00:03.220026, total_t: 0:00:03.259737\n",
      "Running time, country: IN, publisher: TopNews, punct_t: 0:00:00.294077, text_proc_t: 0:00:32.045084, total_t: 0:00:32.339161\n",
      "Running time, country: IN, publisher: Telangana Today, punct_t: 0:00:00.037814, text_proc_t: 0:00:02.929736, total_t: 0:00:02.967550\n",
      "Running time, country: IN, publisher: Indiatimes.com, punct_t: 0:00:00.066777, text_proc_t: 0:00:08.871389, total_t: 0:00:08.938166\n",
      "Running time, country: IN, publisher: risingkashmir.com, punct_t: 0:00:00.109637, text_proc_t: 0:00:12.509627, total_t: 0:00:12.619264\n",
      "Running time, country: IN, publisher: deccanchronicle.com, punct_t: 0:00:00.068842, text_proc_t: 0:00:07.726501, total_t: 0:00:07.795343\n",
      "Running time, country: IN, publisher: orfonline.org, punct_t: 0:00:00.091604, text_proc_t: 0:00:11.422154, total_t: 0:00:11.513758\n",
      "Running time, country: IN, publisher: devdiscourse.com, punct_t: 0:00:00.073586, text_proc_t: 0:00:09.185477, total_t: 0:00:09.259063\n",
      "Running time, country: IN, publisher: CNBCTV18, punct_t: 0:00:00.044494, text_proc_t: 0:00:04.112201, total_t: 0:00:04.156695\n",
      "Running time, country: IN, publisher: New Kerala, punct_t: 0:00:00.020307, text_proc_t: 0:00:00.775477, total_t: 0:00:00.795784\n",
      "Running time, country: IN, publisher: Millennium Post, punct_t: 0:00:00.033850, text_proc_t: 0:00:02.426798, total_t: 0:00:02.460648\n",
      "Running time, country: IN, publisher: The Wire, punct_t: 0:00:00.140402, text_proc_t: 0:00:36.204088, total_t: 0:00:36.344490\n",
      "Running time, country: IN, publisher: Kashmir Images, punct_t: 0:00:00.021823, text_proc_t: 0:00:01.188177, total_t: 0:00:01.210000\n",
      "Running time, country: IN, publisher: Super Good Movies, punct_t: 0:00:00.069200, text_proc_t: 0:00:10.155201, total_t: 0:00:10.224401\n",
      "Running time, country: IN, publisher: ANI News, punct_t: 0:00:00.069880, text_proc_t: 0:00:06.419583, total_t: 0:00:06.489463\n",
      "Running time, country: IN, publisher: The Hindu, punct_t: 0:00:02.517419, text_proc_t: 2:06:42.032095, total_t: 2:06:44.549514\n",
      "Running time, country: IN, publisher: kashmirtimes.com, punct_t: 0:00:00.133309, text_proc_t: 0:00:26.665205, total_t: 0:00:26.798514\n",
      "Running time, country: IN, publisher: The Indian Express, punct_t: 0:00:02.735286, text_proc_t: 1:04:03.495877, total_t: 1:04:06.231163\n",
      "Running time, country: IN, publisher: livemint.com, punct_t: 0:00:00.114931, text_proc_t: 0:22:47.349836, total_t: 0:22:47.464767\n",
      "Running time, country: IN, publisher: wallpaper.com, punct_t: 0:00:00.083183, text_proc_t: 0:00:11.114620, total_t: 0:00:11.197803\n",
      "Running time, country: IN, publisher: onmanorama, punct_t: 0:00:00.132778, text_proc_t: 0:00:08.089658, total_t: 0:00:08.222436\n",
      "Running time, country: IN, publisher: The Shillong Times, punct_t: 0:00:00.134314, text_proc_t: 0:00:09.425308, total_t: 0:00:09.559622\n",
      "Running time, country: IN, publisher: morungexpress.com, punct_t: 0:00:00.191326, text_proc_t: 0:00:15.909776, total_t: 0:00:16.101102\n",
      "Running time, country: IN, publisher: thewire.in, punct_t: 0:00:00.594667, text_proc_t: 0:32:24.995668, total_t: 0:32:25.590335\n",
      "Running time, country: IN, publisher: Wall Street Journal, punct_t: 0:00:00.709108, text_proc_t: 0:03:21.734894, total_t: 0:03:22.444002\n",
      "Running time, country: IN, publisher: in.ign.com, punct_t: 0:00:00.212500, text_proc_t: 0:38:19.019254, total_t: 0:38:19.231754\n",
      "Running time, country: IN, publisher: The New Indian Express, punct_t: 0:00:00.525713, text_proc_t: 0:04:13.828948, total_t: 0:04:14.354661\n",
      "Running time, country: IN, publisher: thesangaiexpress.com, punct_t: 0:00:00.130210, text_proc_t: 0:00:08.488139, total_t: 0:00:08.618349\n",
      "Running time, country: IN, publisher: Orissadiary.com, punct_t: 0:00:00.252395, text_proc_t: 0:00:31.634827, total_t: 0:00:31.887222\n",
      "Running time, country: IN, publisher: dailypioneer.com, punct_t: 0:00:00.366695, text_proc_t: 0:00:51.800373, total_t: 0:00:52.167068\n",
      "Running time, country: IN, publisher: mumbaimirror.indiatimes.com, punct_t: 0:00:00.281012, text_proc_t: 0:00:39.400674, total_t: 0:00:39.681686\n",
      "Running time, country: IN, publisher: Financial Express, punct_t: 0:00:01.025609, text_proc_t: 0:09:38.377782, total_t: 0:09:39.403391\n",
      "Running time, country: IN, publisher: Goal.com, punct_t: 0:00:00.266434, text_proc_t: 0:00:30.405732, total_t: 0:00:30.672166\n",
      "Running time, country: IN, publisher: fortuneindia.com, punct_t: 0:00:00.158008, text_proc_t: 0:34:45.504408, total_t: 0:34:45.662416\n",
      "Running time, country: IN, publisher: Defence Aviation Post, punct_t: 0:00:00.017343, text_proc_t: 0:00:00.561741, total_t: 0:00:00.579084\n",
      "Running time, country: IN, publisher: mid-day.com, punct_t: 0:00:00.082061, text_proc_t: 0:00:29.962050, total_t: 0:00:30.044111\n",
      "Running time, country: IN, publisher: Northeast Now, punct_t: 0:00:00.094922, text_proc_t: 0:00:03.515480, total_t: 0:00:03.610402\n",
      "Running time, country: IN, publisher: Economic Times, punct_t: 0:00:05.159085, text_proc_t: 0:14:59.742146, total_t: 0:15:04.901231\n",
      "Running time, country: IN, publisher: ahmedabadmirror.indiatimes.com, punct_t: 0:00:00.115810, text_proc_t: 0:03:09.737383, total_t: 0:03:09.853193\n",
      "Running time, country: IN, publisher: Morung Express, punct_t: 0:00:00.047968, text_proc_t: 0:00:05.993787, total_t: 0:00:06.041755\n",
      "Running time, country: IN, publisher: State Times, punct_t: 0:00:00.038425, text_proc_t: 0:00:11.720236, total_t: 0:00:11.758661\n",
      "Running time, country: IN, publisher: International Business Times, India Edition, punct_t: 0:00:00.655943, text_proc_t: 0:57:38.612338, total_t: 0:57:39.268281\n",
      "Running time, country: IN, publisher: THE WEEK, punct_t: 0:00:00.072277, text_proc_t: 0:00:21.649080, total_t: 0:00:21.721357\n",
      "Running time, country: IN, publisher: Rediff, punct_t: 0:00:04.275221, text_proc_t: 0:54:54.647190, total_t: 0:54:58.922411\n",
      "Running time, country: IN, publisher: Times of India, punct_t: 0:00:27.392984, text_proc_t: 1:30:52.741978, total_t: 1:31:20.134962\n",
      "Running time, country: IN, publisher: Kasmir Monitor, punct_t: 0:00:00.059482, text_proc_t: 0:00:01.299877, total_t: 0:00:01.359359\n",
      "Running time, country: IN, publisher: timesofindia.indiatimes.com, punct_t: 0:00:00.046026, text_proc_t: 0:00:04.647765, total_t: 0:00:04.693791\n",
      "Running time, country: IN, publisher: news18.com, punct_t: 0:00:00.065672, text_proc_t: 0:00:07.665641, total_t: 0:00:07.731313\n",
      "Running time, country: IN, publisher: ETTelecom.com, punct_t: 0:00:00.022742, text_proc_t: 0:00:01.158079, total_t: 0:00:01.180821\n",
      "Running time, country: IN, publisher: aninews.in, punct_t: 0:00:00.059062, text_proc_t: 0:00:05.477810, total_t: 0:00:05.536872\n",
      "Running time, country: IN, publisher: espn.in, punct_t: 0:00:00.120978, text_proc_t: 0:00:27.317387, total_t: 0:00:27.438365\n",
      "Running time, country: IN, publisher: DailyO, punct_t: 0:00:00.074393, text_proc_t: 0:00:10.821615, total_t: 0:00:10.896008\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running time, country: IN, publisher: indiatoday.in, punct_t: 0:00:00.076764, text_proc_t: 0:00:11.495538, total_t: 0:00:11.572302\n",
      "Running time, country: IN, publisher: The Asian Age, punct_t: 0:00:00.173444, text_proc_t: 0:00:39.687820, total_t: 0:00:39.861264\n",
      "Running time, country: IN, publisher: livelaw.in, punct_t: 0:00:00.097721, text_proc_t: 0:00:11.424211, total_t: 0:00:11.521932\n",
      "Running time, country: IN, publisher: BoldSky, punct_t: 0:00:00.088581, text_proc_t: 0:00:12.872550, total_t: 0:00:12.961131\n",
      "Running time, country: IN, publisher: Asian Correspondent, punct_t: 0:00:00.119783, text_proc_t: 0:00:17.292549, total_t: 0:00:17.412332\n",
      "Running time, country: IN, publisher: Bangalore Mirror, punct_t: 0:00:00.056055, text_proc_t: 0:00:05.030144, total_t: 0:00:05.086199\n",
      "Running time, country: IN, publisher: Sify, punct_t: 0:00:00.077317, text_proc_t: 0:00:08.389216, total_t: 0:00:08.466533\n",
      "Running time, country: IN, publisher: bgr.in, punct_t: 0:00:00.096770, text_proc_t: 0:00:13.884028, total_t: 0:00:13.980798\n",
      "Running time, country: IN, publisher: Greater Kashmir, punct_t: 0:00:00.105163, text_proc_t: 0:00:13.142597, total_t: 0:00:13.247760\n",
      "Running time, country: IN, publisher: peoplematters.in, punct_t: 0:00:00.076254, text_proc_t: 0:00:07.953173, total_t: 0:00:08.029427\n",
      "Running time, country: IN, publisher: Oherald, punct_t: 0:00:00.042827, text_proc_t: 0:00:02.761105, total_t: 0:00:02.803932\n",
      "Running time, country: IN, publisher: Firstpost, punct_t: 0:00:00.944709, text_proc_t: 0:02:01.272573, total_t: 0:02:02.217282\n",
      "Running time, country: IN, publisher: thehansindia.com, punct_t: 0:00:00.080318, text_proc_t: 0:00:09.257264, total_t: 0:00:09.337582\n",
      "Running time, country: IN, publisher: ABP Live, punct_t: 0:00:00.028092, text_proc_t: 0:00:01.912249, total_t: 0:00:01.940341\n",
      "Running time, country: IN, publisher: Pune Mirror, punct_t: 0:00:00.056967, text_proc_t: 0:00:05.537203, total_t: 0:00:05.594170\n",
      "Running time, country: IN, publisher: Livemint, punct_t: 0:00:01.390955, text_proc_t: 0:02:50.026194, total_t: 0:02:51.417149\n",
      "Running time, country: IN, publisher: news.statetimes.in, punct_t: 0:00:00.046637, text_proc_t: 0:00:03.672429, total_t: 0:00:03.719066\n",
      "Running time, country: IN, publisher: telanganatoday.com, punct_t: 0:00:00.070289, text_proc_t: 0:00:07.207808, total_t: 0:00:07.278097\n",
      "Running time, country: IN, publisher: Hindu Business Line, punct_t: 0:00:00.527898, text_proc_t: 0:01:15.432897, total_t: 0:01:15.960795\n",
      "Running time, country: IN, publisher: analyticsindiamag.com, punct_t: 0:00:00.086454, text_proc_t: 0:00:12.441132, total_t: 0:00:12.527586\n",
      "Running time, country: IN, publisher: Daily Pioneer, punct_t: 0:00:00.097967, text_proc_t: 0:00:14.786412, total_t: 0:00:14.884379\n",
      "Running time, country: IN, publisher: Business Insider India, punct_t: 0:00:00.079440, text_proc_t: 0:00:09.660439, total_t: 0:00:09.739879\n",
      "Running time, country: IN, publisher: punemirror.indiatimes.com, punct_t: 0:00:00.086903, text_proc_t: 0:00:12.331268, total_t: 0:00:12.418171\n",
      "Running time, country: IN, publisher: Mid-Day, punct_t: 0:00:00.061218, text_proc_t: 0:00:05.873516, total_t: 0:00:05.934734\n",
      "Running time, country: IN, publisher: nenow.in, punct_t: 0:00:00.074397, text_proc_t: 0:00:07.382801, total_t: 0:00:07.457198\n",
      "Running time, country: IN, publisher: DirectionsMag.com, punct_t: 0:00:00.095856, text_proc_t: 0:00:12.535123, total_t: 0:00:12.630979\n",
      "Running time, country: IN, publisher: afaqs.com, punct_t: 0:00:00.085789, text_proc_t: 0:00:10.943646, total_t: 0:00:11.029435\n",
      "Running time, country: IN, publisher: Bollywood Mantra, punct_t: 0:00:00.079560, text_proc_t: 0:00:09.094584, total_t: 0:00:09.174144\n",
      "Running time, country: IN, publisher: Cricket Country, punct_t: 0:00:00.349108, text_proc_t: 0:02:02.638933, total_t: 0:02:02.988041\n",
      "Running time, country: IN, publisher: From Software, punct_t: 0:00:00.029278, text_proc_t: 0:00:01.452936, total_t: 0:00:01.482214\n",
      "Running time, country: IN, publisher: Yahoo News, punct_t: 0:00:00.030837, text_proc_t: 0:00:01.503129, total_t: 0:00:01.533966\n",
      "Running time, country: IN, publisher: ibtimes.co.in, punct_t: 0:00:00.089546, text_proc_t: 0:00:13.572652, total_t: 0:00:13.662198\n",
      "Running time, country: IN, publisher: TwoCircles.net, punct_t: 0:00:00.145251, text_proc_t: 0:00:29.095956, total_t: 0:00:29.241207\n",
      "Running time, country: IN, publisher: Gizbot, punct_t: 0:00:00.027110, text_proc_t: 0:00:01.050217, total_t: 0:00:01.077327\n",
      "Running time, country: IN, publisher: businesstoday.in, punct_t: 0:00:00.073984, text_proc_t: 0:00:09.955078, total_t: 0:00:10.029062\n",
      "Running time, country: IN, publisher: Equity Bulls, punct_t: 0:00:00.027677, text_proc_t: 0:00:00.952535, total_t: 0:00:00.980212\n",
      "Running time, country: IN, publisher: BW Businessworld, punct_t: 0:00:00.041004, text_proc_t: 0:00:03.213287, total_t: 0:00:03.254291\n",
      "Running time, country: IN, publisher: Bollywood Life, punct_t: 0:00:00.192795, text_proc_t: 0:00:28.057822, total_t: 0:00:28.250617\n",
      "Running time, country: IN, publisher: nyoooz.com, punct_t: 0:00:00.057522, text_proc_t: 0:00:04.295038, total_t: 0:00:04.352560\n",
      "Running time, country: IN, publisher: Eastern Mirror, punct_t: 0:00:00.035439, text_proc_t: 0:00:01.557523, total_t: 0:00:01.592962\n",
      "Running time, country: IN, publisher: saipantribune.com, punct_t: 0:00:00.112092, text_proc_t: 0:00:13.289098, total_t: 0:00:13.401190\n",
      "Running time, country: IN, publisher: Devdiscourse, punct_t: 0:00:00.080600, text_proc_t: 0:00:08.860640, total_t: 0:00:08.941240\n",
      "Running time, country: IN, publisher: ESPN, punct_t: 0:00:00.060642, text_proc_t: 0:00:06.678108, total_t: 0:00:06.738750\n",
      "Running time, country: IN, publisher: Huffington Post India, punct_t: 0:00:00.034476, text_proc_t: 0:00:02.560522, total_t: 0:00:02.594998\n",
      "Running time, country: IN, publisher: republicworld.com, punct_t: 0:00:00.091058, text_proc_t: 0:00:12.481522, total_t: 0:00:12.572580\n",
      "Running time, country: IN, publisher: India.com, punct_t: 0:00:00.233633, text_proc_t: 0:00:31.230692, total_t: 0:00:31.464325\n",
      "Running time, country: IN, publisher: Times of Indi, punct_t: 0:00:00.040097, text_proc_t: 0:00:02.953769, total_t: 0:00:02.993866\n",
      "Running time, country: IN, publisher: Business Today, punct_t: 0:00:00.064755, text_proc_t: 0:00:07.120112, total_t: 0:00:07.184867\n",
      "Running time, country: IN, publisher: siasat.com, punct_t: 0:00:00.094720, text_proc_t: 0:00:11.888752, total_t: 0:00:11.983472\n",
      "Running time, country: IN, publisher: Free Press Journal, punct_t: 0:00:00.164807, text_proc_t: 0:00:33.262360, total_t: 0:00:33.427167\n",
      "Running time, country: IN, publisher: Republic World, punct_t: 0:00:00.026997, text_proc_t: 0:00:01.005738, total_t: 0:00:01.032735\n",
      "Running time, country: IN, publisher: IndiaGlitz, punct_t: 0:00:00.087703, text_proc_t: 0:00:13.666173, total_t: 0:00:13.753876\n",
      "Running time, country: IN, publisher: bloombergquint.com, punct_t: 0:00:00.040004, text_proc_t: 0:00:02.476810, total_t: 0:00:02.516814\n",
      "Running time, country: IN, publisher: News18, punct_t: 0:00:00.256958, text_proc_t: 0:00:46.526691, total_t: 0:00:46.783649\n",
      "Running time, country: IN, publisher: E-Pao.net, punct_t: 0:00:00.086146, text_proc_t: 0:00:10.489317, total_t: 0:00:10.575463\n",
      "Running time, country: IN, publisher: Nagpur Today, punct_t: 0:00:00.030115, text_proc_t: 0:00:01.715821, total_t: 0:00:01.745936\n",
      "Running time, country: IN, publisher: bangaloremirror.indiatimes.com, punct_t: 0:00:00.081137, text_proc_t: 0:00:09.941886, total_t: 0:00:10.023023\n",
      "Running time, country: IN, publisher: Cricbuzz, punct_t: 0:00:00.084797, text_proc_t: 0:00:11.711696, total_t: 0:00:11.796493\n",
      "Running time, country: IN, publisher: IBNLive, punct_t: 0:00:00.053976, text_proc_t: 0:00:04.792615, total_t: 0:00:04.846591\n",
      "Running time, country: IN, publisher: All India Radio, punct_t: 0:00:00.021575, text_proc_t: 0:00:00.985261, total_t: 0:00:01.006836\n",
      "Running time, country: IN, publisher: Times Now, punct_t: 0:00:00.150622, text_proc_t: 0:00:35.929314, total_t: 0:00:36.079936\n",
      "Running time, country: IN, publisher: thequint.com, punct_t: 0:00:00.056400, text_proc_t: 0:00:04.907966, total_t: 0:00:04.964366\n",
      "Running time, country: IN, publisher: PINKVILLA, punct_t: 0:00:00.049323, text_proc_t: 0:00:03.948792, total_t: 0:00:03.998115\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running time, country: IN, publisher: newindianexpress.com, punct_t: 0:00:00.086572, text_proc_t: 0:00:12.028800, total_t: 0:00:12.115372\n",
      "Running time, country: IN, publisher: SportsCafe, punct_t: 0:00:00.027709, text_proc_t: 0:00:00.966913, total_t: 0:00:00.994622\n",
      "Running time, country: IN, publisher: ETAuto.com, punct_t: 0:00:00.024824, text_proc_t: 0:00:01.316605, total_t: 0:00:01.341429\n",
      "Running time, country: IN, publisher: VC Circle, punct_t: 0:00:00.131123, text_proc_t: 0:00:25.146091, total_t: 0:00:25.277214\n",
      "Running time, country: IN, publisher: millenniumpost.in, punct_t: 0:00:00.026422, text_proc_t: 0:00:00.877913, total_t: 0:00:00.904335\n",
      "Running time, country: IN, publisher: YourStory.com, punct_t: 0:00:00.062594, text_proc_t: 0:00:08.158825, total_t: 0:00:08.221419\n",
      "Running time, country: IN, publisher: The Echo of India, punct_t: 0:00:00.045188, text_proc_t: 0:00:03.516558, total_t: 0:00:03.561746\n",
      "Running time, country: IN, publisher: huffingtonpost.in, punct_t: 0:00:00.098328, text_proc_t: 0:00:15.791015, total_t: 0:00:15.889343\n",
      "Running time, country: IN, publisher: NDTVSports.com, punct_t: 0:00:00.089157, text_proc_t: 0:00:11.547338, total_t: 0:00:11.636495\n",
      "Running time, country: IN, publisher: Indiainfoline, punct_t: 0:00:00.030006, text_proc_t: 0:00:02.165321, total_t: 0:00:02.195327\n",
      "Running time, country: IN, publisher: in.news.yahoo.com, punct_t: 0:00:00.111100, text_proc_t: 0:00:18.770737, total_t: 0:00:18.881837\n",
      "Running time, country: IN, publisher: Indian Cars Bikes, punct_t: 0:00:00.111508, text_proc_t: 0:00:17.399234, total_t: 0:00:17.510742\n",
      "Running time, country: IN, publisher: Times of India (blog), punct_t: 0:00:00.042039, text_proc_t: 0:00:03.880990, total_t: 0:00:03.923029\n",
      "Running time, country: IN, publisher: NDTV, punct_t: 0:00:00.701102, text_proc_t: 0:01:20.819751, total_t: 0:01:21.520853\n",
      "Running time, country: IN, publisher: asianage.com, punct_t: 0:00:00.084119, text_proc_t: 0:00:10.097907, total_t: 0:00:10.182026\n",
      "Running time, country: IN, publisher: India TV, punct_t: 0:00:00.052897, text_proc_t: 0:00:04.541742, total_t: 0:00:04.594639\n",
      "Running time, country: IN, publisher: businessinsider.in, punct_t: 0:00:00.100286, text_proc_t: 0:00:14.188943, total_t: 0:00:14.289229\n",
      "Running time, country: IN, publisher: BGR India, punct_t: 0:00:00.051742, text_proc_t: 0:00:04.580138, total_t: 0:00:04.631880\n",
      "Running time, country: IN, publisher: Goal India, punct_t: 0:00:00.025321, text_proc_t: 0:00:01.045529, total_t: 0:00:01.070850\n",
      "Running time, country: IN, publisher: IGN India, punct_t: 0:00:00.029373, text_proc_t: 0:00:02.173753, total_t: 0:00:02.203126\n",
      "Running time, country: IN, publisher: Zigwheels.com, punct_t: 0:00:00.054446, text_proc_t: 0:00:05.407279, total_t: 0:00:05.461725\n",
      "Running time, country: IN, publisher: MediaNama.com, punct_t: 0:00:00.111768, text_proc_t: 0:00:18.943088, total_t: 0:00:19.054856\n",
      "Running time, country: IN, publisher: Outlook India, punct_t: 0:00:00.105831, text_proc_t: 0:00:14.591608, total_t: 0:00:14.697439\n",
      "Running time, country: IN, publisher: Game Guru, punct_t: 0:00:00.095511, text_proc_t: 0:00:13.580140, total_t: 0:00:13.675651\n",
      "Running time, country: IN, publisher: thestatesman.com, punct_t: 0:00:00.089627, text_proc_t: 0:00:11.264479, total_t: 0:00:11.354106\n",
      "Running time, country: IN, publisher: Kashmir Life, punct_t: 0:00:00.021311, text_proc_t: 0:00:01.081601, total_t: 0:00:01.102912\n",
      "Running time, country: IN, publisher: The Sangai Express, punct_t: 0:00:00.036805, text_proc_t: 0:00:02.606272, total_t: 0:00:02.643077\n",
      "Running time, country: IN, publisher: The Siasat Daily, punct_t: 0:00:00.027061, text_proc_t: 0:00:01.804221, total_t: 0:00:01.831282\n",
      "Running time, country: IN, publisher: Daily News &amp; Analysis, punct_t: 0:00:00.854062, text_proc_t: 0:01:35.430442, total_t: 0:01:36.284504\n",
      "Running time, country: IN, publisher: moneylife.in, punct_t: 0:00:00.135740, text_proc_t: 0:00:26.486496, total_t: 0:00:26.622236\n",
      "Running time, country: IN, publisher: Nagaland Post, punct_t: 0:00:00.021874, text_proc_t: 0:00:01.065559, total_t: 0:00:01.087433\n",
      "Running time, country: IN, publisher: Business Standard, punct_t: 0:00:00.337958, text_proc_t: 0:00:41.240762, total_t: 0:00:41.578720\n",
      "Running time, country: IN, publisher: The Telegraph India, punct_t: 0:00:00.038796, text_proc_t: 0:00:02.839792, total_t: 0:00:02.878588\n",
      "Running time, country: IN, publisher: thecitizen.in, punct_t: 0:00:00.054866, text_proc_t: 0:00:06.898254, total_t: 0:00:06.953120\n",
      "Running time, country: IN, publisher: uniindia.com, punct_t: 0:00:00.042690, text_proc_t: 0:00:03.501960, total_t: 0:00:03.544650\n",
      "Running time, country: IN, publisher: Mathrubhumi, punct_t: 0:00:00.035274, text_proc_t: 0:00:02.544799, total_t: 0:00:02.580073\n",
      "Running time, country: IN, publisher: downtoearth.org.in, punct_t: 0:00:00.110410, text_proc_t: 0:00:23.125999, total_t: 0:00:23.236409\n",
      "Running time, country: IN, publisher: cogencis.com, punct_t: 0:00:00.087639, text_proc_t: 0:00:14.209208, total_t: 0:00:14.296847\n",
      "Running time, country: IN, publisher: NDTV.com, punct_t: 0:00:00.294746, text_proc_t: 0:00:45.998749, total_t: 0:00:46.293495\n",
      "Running time, country: IN, publisher: BloombergQuint, punct_t: 0:00:00.205718, text_proc_t: 0:00:51.856889, total_t: 0:00:52.062607\n",
      "Running time, country: IN, publisher: indianexpress.com, punct_t: 0:00:00.043386, text_proc_t: 0:00:03.112498, total_t: 0:00:03.155884\n",
      "Running time, country: IN, publisher: The Hindu BusinessLine, punct_t: 0:00:00.042404, text_proc_t: 0:00:03.131426, total_t: 0:00:03.173830\n",
      "Running time, country: IN, publisher: Web India 123, punct_t: 0:00:00.045993, text_proc_t: 0:00:03.272468, total_t: 0:00:03.318461\n",
      "Running time, country: IN, publisher: The Quint, punct_t: 0:00:00.129317, text_proc_t: 0:00:26.471393, total_t: 0:00:26.600710\n",
      "Running time, country: IN, publisher: Ahmedabad Mirror, punct_t: 0:00:00.039265, text_proc_t: 0:00:02.580840, total_t: 0:00:02.620105\n",
      "Running time, country: IN, publisher: Deccan Herald, punct_t: 0:00:00.454092, text_proc_t: 0:01:04.811848, total_t: 0:01:05.265940\n",
      "Running time, country: IN, publisher: dailyexcelsior.com, punct_t: 0:00:00.063676, text_proc_t: 0:00:06.106467, total_t: 0:00:06.170143\n",
      "Running time, country: IN, publisher: thehindubusinessline.com, punct_t: 0:00:00.081655, text_proc_t: 0:00:09.384177, total_t: 0:00:09.465832\n",
      "Running time, country: IN, publisher: NYOOOZ, punct_t: 0:00:00.065884, text_proc_t: 0:00:06.345185, total_t: 0:00:06.411069\n",
      "Running time, country: IN, publisher: United News of India, punct_t: 0:00:00.062290, text_proc_t: 0:00:05.882353, total_t: 0:00:05.944643\n",
      "Running time, country: IN, publisher: India PRwire, punct_t: 0:00:00.133351, text_proc_t: 0:00:23.330412, total_t: 0:00:23.463763\n",
      "Running time, country: IN, publisher: Speaking Times, punct_t: 0:00:00.032217, text_proc_t: 0:00:01.609967, total_t: 0:00:01.642184\n",
      "Running time, country: IN, publisher: The Better India, punct_t: 0:00:00.031608, text_proc_t: 0:00:02.301378, total_t: 0:00:02.332986\n",
      "Running time, country: IN, publisher: Zee Business, punct_t: 0:00:00.027883, text_proc_t: 0:00:01.639275, total_t: 0:00:01.667158\n",
      "Running time, country: IN, publisher: Odisha Television Ltd., punct_t: 0:00:00.021621, text_proc_t: 0:00:01.230100, total_t: 0:00:01.251721\n",
      "Running time, country: IN, publisher: mydigitalfc.com, punct_t: 0:00:00.069584, text_proc_t: 0:00:07.762847, total_t: 0:00:07.832431\n",
      "Running time, country: IN, publisher: Star of Mysore, punct_t: 0:00:00.029925, text_proc_t: 0:00:01.743082, total_t: 0:00:01.773007\n",
      "Running time, country: IN, publisher: Calcutta Telegraph, punct_t: 0:00:00.062307, text_proc_t: 0:00:07.008766, total_t: 0:00:07.071073\n",
      "Running time, country: IN, publisher: newsnation.in, punct_t: 0:00:00.044723, text_proc_t: 0:00:04.110973, total_t: 0:00:04.155696\n",
      "Running time, country: IN, publisher: The Northlines, punct_t: 0:00:00.025455, text_proc_t: 0:00:01.337488, total_t: 0:00:01.362943\n",
      "Running time, country: IN, publisher: Deccan Chronicle, punct_t: 0:00:00.187060, text_proc_t: 0:00:36.677833, total_t: 0:00:36.864893\n",
      "Running time, country: IN, publisher: financialexpress.com, punct_t: 0:00:00.048853, text_proc_t: 0:00:04.123367, total_t: 0:00:04.172220\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running time, country: IN, publisher: ETBrandEquity.com, punct_t: 0:00:00.023344, text_proc_t: 0:00:01.239863, total_t: 0:00:01.263207\n",
      "Running time, country: IN, publisher: Jagran Post, punct_t: 0:00:00.041641, text_proc_t: 0:00:03.024904, total_t: 0:00:03.066545\n",
      "Running time, country: IN, publisher: business-standard.com, punct_t: 0:00:00.051967, text_proc_t: 0:00:04.796564, total_t: 0:00:04.848531\n",
      "Running time, country: IN, publisher: MensXP.com, punct_t: 0:00:00.047707, text_proc_t: 0:00:04.553377, total_t: 0:00:04.601084\n",
      "Running time, country: IN, publisher: The Statesman, punct_t: 0:00:00.043201, text_proc_t: 0:00:03.454006, total_t: 0:00:03.497207\n",
      "Running time, country: IN, publisher: Siliconindia.com, punct_t: 0:00:00.053465, text_proc_t: 0:00:05.602958, total_t: 0:00:05.656423\n",
      "Running time, country: IN, publisher: Digit, punct_t: 0:00:00.030322, text_proc_t: 0:00:01.658209, total_t: 0:00:01.688531\n",
      "Running time, country: IN, publisher: ETEnergyworld.com, punct_t: 0:00:00.023030, text_proc_t: 0:00:00.866348, total_t: 0:00:00.889378\n",
      "Running time, country: IN, publisher: Avenue Mail, punct_t: 0:00:00.022493, text_proc_t: 0:00:01.303015, total_t: 0:00:01.325508\n",
      "0:02:46.458591\n",
      "11:58:32.775088\n"
     ]
    }
   ],
   "source": [
    "cosine_process = False\n",
    "N_gram_process = True\n",
    "merged_process = True if cosine_process and N_gram_process else False\n",
    "cosine_threshold=0.95\n",
    "\n",
    "time_taken = []\n",
    "text_list = []\n",
    "\n",
    "if merged_process:\n",
    "    base='merged'\n",
    "elif cosine_process:\n",
    "    base='cosine'\n",
    "else:\n",
    "    base='ngram'\n",
    "\n",
    "print(\"Methodology: {}\".format(base))\n",
    "  \n",
    "for keys1 in overall_dict.keys():\n",
    "    if keys1 != Country_to_process:\n",
    "        pass\n",
    "    else:\n",
    "        for keys2 in overall_dict[keys1].keys(): # i.e. 'US', \"CA\"\n",
    "\n",
    "            a = datetime.datetime.now()\n",
    "\n",
    "            df = overall_dict[keys1][keys2]\n",
    "            df = text_initial_preproecss(df)\n",
    "\n",
    "            b = datetime.datetime.now()\n",
    "\n",
    "            if merged_process:\n",
    "                df = ngram_preprocess(df)\n",
    "                df = cosine_preprocess(df,model,cosine_threshold)\n",
    "\n",
    "            elif cosine_process:\n",
    "                df = cosine_preprocess(df,model,cosine_threshold)\n",
    "\n",
    "            elif N_gram_process:\n",
    "                df = ngram_preprocess(df)\n",
    "\n",
    "\n",
    "            df.apply(save_file,args = (base,),axis=1)\n",
    "\n",
    "            df.text = df.text.apply(remove_stopwords)\n",
    "            df.text = df.text.apply(remove_punct)\n",
    "            df.text = df.text.apply(remove_digits)\n",
    "            df.text = df.text.apply(lemmatize_with_postag)\n",
    "\n",
    "            #saving files\n",
    "            df.apply(save_file,args = ('lemmatized',),axis=1)\n",
    "\n",
    "            c = datetime.datetime.now()\n",
    "\n",
    "            print(\"Running time, country: {}, publisher: {}, punct_t: {}, text_proc_t: {}, total_t: {}\".format(keys1,keys2, b-a, c-b, c-a))\n",
    "            time_taken.append(c-a)\n",
    "\n",
    "print(np.mean(time_taken))\n",
    "print(np.sum(time_taken))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cosine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cosine_process = True\n",
    "# N_gram_process = False\n",
    "# merged_process = True if cosine_process and N_gram_process else False\n",
    "# cosine_threshold=0.95\n",
    "\n",
    "# model = SentenceTransformer('sentence-transformers/bert-base-nli-stsb-mean-tokens')\n",
    "\n",
    "# time_taken = []\n",
    "# text_list = []\n",
    "\n",
    "# if merged_process:\n",
    "#     base='merged'\n",
    "# elif cosine_process:\n",
    "#     base='cosine'\n",
    "# else:\n",
    "#     base='ngram'\n",
    "\n",
    "# print(\"Methodology: {}\".format(base))\n",
    "  \n",
    "# for keys in overall_dict.keys(): # i.e. 'US', \"CA\"\n",
    "    \n",
    "#     a = datetime.datetime.now()\n",
    "    \n",
    "#     df = overall_dict[keys]   \n",
    "#     df = text_initial_preproecss(df)\n",
    "    \n",
    "#     b = datetime.datetime.now()\n",
    "    \n",
    "# #     df.apply(save_file_initial,args = (base,),axis=1)\n",
    "\n",
    "# #     text_list.append(df.iloc[1])\n",
    "#     if merged_process:\n",
    "#         df = ngram_preprocess(df)\n",
    "#         df = cosine_preprocess(df,model,cosine_threshold)\n",
    "\n",
    "#     elif cosine_process:\n",
    "#         df = cosine_preprocess(df,model,cosine_threshold)\n",
    "\n",
    "#     elif N_gram_process:\n",
    "#         df = ngram_preprocess(df)\n",
    "\n",
    "    \n",
    "#     #saving files\n",
    "#     df.apply(save_file,args = (base,),axis=1)\n",
    "\n",
    "#     c = datetime.datetime.now()\n",
    "    \n",
    "#     print(\"Running time, country: {}, punct_t: {}, text_proc_t: {}, total_t: {}\".format(keys, b-a, c-b, c-a))\n",
    "#     time_taken.append(c-a)\n",
    "\n",
    "# print(np.mean(time_taken))\n",
    "# print(np.sum(time_taken))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cosine_process = True\n",
    "# N_gram_process = True\n",
    "# merged_process = True if cosine_process and N_gram_process else False\n",
    "# cosine_threshold=0.95\n",
    "\n",
    "# model = SentenceTransformer('sentence-transformers/bert-base-nli-stsb-mean-tokens')\n",
    "\n",
    "# time_taken = []\n",
    "# text_list = []\n",
    "\n",
    "# if merged_process:\n",
    "#     base='merged'\n",
    "# elif cosine_process:\n",
    "#     base='cosine'\n",
    "# else:\n",
    "#     base='ngram'\n",
    "\n",
    "# print(\"Methodology: {}\".format(base))\n",
    "  \n",
    "# for keys in overall_dict.keys(): # i.e. 'US', \"CA\"\n",
    "    \n",
    "#     a = datetime.datetime.now()\n",
    "    \n",
    "#     df = overall_dict[keys]   \n",
    "#     df = text_initial_preproecss(df)\n",
    "    \n",
    "#     b = datetime.datetime.now()\n",
    "    \n",
    "# #     df.apply(save_file_initial,args = (base,),axis=1)\n",
    "\n",
    "# #     text_list.append(df.iloc[1])\n",
    "#     if merged_process:\n",
    "#         df = ngram_preprocess(df)\n",
    "#         df = cosine_preprocess(df,model,cosine_threshold)\n",
    "\n",
    "#     elif cosine_process:\n",
    "#         df = cosine_preprocess(df,model,cosine_threshold)\n",
    "\n",
    "#     elif N_gram_process:\n",
    "#         df = ngram_preprocess(df)\n",
    "\n",
    "    \n",
    "#     #saving files\n",
    "#     df.apply(save_file,args = (base,),axis=1)\n",
    "\n",
    "#     c = datetime.datetime.now()\n",
    "    \n",
    "#     print(\"Running time, country: {}, punct_t: {}, text_proc_t: {}, total_t: {}\".format(keys, b-a, c-b, c-a))\n",
    "#     time_taken.append(c-a)\n",
    "\n",
    "# print(np.mean(time_taken))\n",
    "# print(np.sum(time_taken))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}