Preprocess_final_merged.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "import os\n",
    "import re\n",
    "import codecs\n",
    "import pathlib\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import nltk\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from nltk.tokenize import word_tokenize\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import datetime\n",
    "from scipy.spatial.distance import cosine\n",
    "\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "# !pip install transformers\n",
    "# !pip install sentence_transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PK, US, HK, GH, NG, KE, NZ, BD, IN, LK, CA, ZA, SG, PH, GB, MY, AU, IE, JM, TZ, "
     ]
    }
   ],
   "source": [
    "base_dir = os.path.join(os.getcwd(), 'data', 'clean') \n",
    "countries_list = os.listdir(base_dir) # dir is your directory path\n",
    "summary = []\n",
    "\n",
    "for con in countries_list:\n",
    "    if(os.path.isdir(os.path.join(base_dir,con))):\n",
    "        print(con, end=\", \", flush=True)\n",
    "        publishers = os.listdir(os.path.join(base_dir,con))\n",
    "        for pub in publishers:\n",
    "            subdir = os.path.join(base_dir, con, pub)\n",
    "            length = sum([len(files) for r, d, files in os.walk(subdir)])\n",
    "            summary.append([con, pub, length])\n",
    "\n",
    "summary_df = pd.DataFrame(summary, columns = ['country','publisher','count'])     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# summary_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country</th>\n",
       "      <th>publisher</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>PK</td>\n",
       "      <td>the-express-tribune</td>\n",
       "      <td>6555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6085</th>\n",
       "      <td>US</td>\n",
       "      <td>npr</td>\n",
       "      <td>6879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13877</th>\n",
       "      <td>HK</td>\n",
       "      <td>ej-insight</td>\n",
       "      <td>2607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14167</th>\n",
       "      <td>GH</td>\n",
       "      <td>ghanaweb</td>\n",
       "      <td>4299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14370</th>\n",
       "      <td>NG</td>\n",
       "      <td>vanguard</td>\n",
       "      <td>3989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15048</th>\n",
       "      <td>KE</td>\n",
       "      <td>freshplaza</td>\n",
       "      <td>4137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15502</th>\n",
       "      <td>NZ</td>\n",
       "      <td>stuff-co-nz</td>\n",
       "      <td>7273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15591</th>\n",
       "      <td>BD</td>\n",
       "      <td>bangladesh-news-24-hours</td>\n",
       "      <td>2786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16411</th>\n",
       "      <td>IN</td>\n",
       "      <td>times-of-india</td>\n",
       "      <td>5557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17300</th>\n",
       "      <td>LK</td>\n",
       "      <td>hiru-news</td>\n",
       "      <td>2627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18777</th>\n",
       "      <td>CA</td>\n",
       "      <td>national-post</td>\n",
       "      <td>6518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19950</th>\n",
       "      <td>ZA</td>\n",
       "      <td>independent-online</td>\n",
       "      <td>5906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20907</th>\n",
       "      <td>SG</td>\n",
       "      <td>asiaone</td>\n",
       "      <td>5198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21330</th>\n",
       "      <td>PH</td>\n",
       "      <td>philippine-star</td>\n",
       "      <td>4867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26504</th>\n",
       "      <td>GB</td>\n",
       "      <td>telegraph-co-uk</td>\n",
       "      <td>8584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26986</th>\n",
       "      <td>MY</td>\n",
       "      <td>the-sun-daily</td>\n",
       "      <td>4010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27185</th>\n",
       "      <td>AU</td>\n",
       "      <td>news-com-au</td>\n",
       "      <td>6130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29661</th>\n",
       "      <td>IE</td>\n",
       "      <td>rte-ie</td>\n",
       "      <td>6412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29762</th>\n",
       "      <td>JM</td>\n",
       "      <td>jamaica-gleaner</td>\n",
       "      <td>4802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29783</th>\n",
       "      <td>TZ</td>\n",
       "      <td>ippmedia</td>\n",
       "      <td>3749</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      country                 publisher  count\n",
       "202        PK       the-express-tribune   6555\n",
       "6085       US                       npr   6879\n",
       "13877      HK                ej-insight   2607\n",
       "14167      GH                  ghanaweb   4299\n",
       "14370      NG                  vanguard   3989\n",
       "15048      KE                freshplaza   4137\n",
       "15502      NZ               stuff-co-nz   7273\n",
       "15591      BD  bangladesh-news-24-hours   2786\n",
       "16411      IN            times-of-india   5557\n",
       "17300      LK                 hiru-news   2627\n",
       "18777      CA             national-post   6518\n",
       "19950      ZA        independent-online   5906\n",
       "20907      SG                   asiaone   5198\n",
       "21330      PH           philippine-star   4867\n",
       "26504      GB           telegraph-co-uk   8584\n",
       "26986      MY             the-sun-daily   4010\n",
       "27185      AU               news-com-au   6130\n",
       "29661      IE                    rte-ie   6412\n",
       "29762      JM           jamaica-gleaner   4802\n",
       "29783      TZ                  ippmedia   3749"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx = summary_df.groupby(['country'])['count'].transform(max) == summary_df['count']\n",
    "max_table = summary_df[idx]\n",
    "max_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PK, US, HK, GH, NG, KE, NZ, BD, IN, LK, CA, ZA, SG, PH, GB, MY, AU, IE, JM, TZ, "
     ]
    }
   ],
   "source": [
    "overall_dict = {}\n",
    "\n",
    "for index, row in max_table.iterrows():\n",
    "    country, publisher = row[0], row[1]\n",
    "    print(country, end=\", \", flush=True)\n",
    "    directory = os.path.join(base_dir, country, publisher)\n",
    "    df_dict = {\"id\": [], \"title\": [], \"publisher\": [], \"article_text\":[],\"url\":[], \"path\":[]}\n",
    "\n",
    "    for entry in os.scandir(directory):\n",
    "        if(os.path.isdir(entry)):\n",
    "            for entry_2 in os.scandir(entry.path):\n",
    "                try:\n",
    "                    with open(entry_2, \"r\") as f:\n",
    "                        article_id = f.readline().strip()\n",
    "                        article_title = f.readline().strip()\n",
    "                        publisher = f.readline().strip()\n",
    "                        url = f.readline()\n",
    "                        f.readline()\n",
    "                        article_text = f.readline().strip()\n",
    "                        df_dict['url'].append(url)\n",
    "                        df_dict['id'].append(article_id)\n",
    "                        df_dict['title'].append(article_title)\n",
    "                        df_dict['publisher'].append(publisher)\n",
    "                        df_dict['article_text'].append(article_text)\n",
    "                        df_dict['path'].append(entry_2)\n",
    "                except:\n",
    "                    pass\n",
    "    sample_df = pd.DataFrame.from_dict(df_dict)\n",
    "    sample_df = sample_df.sample(n=250, random_state=1)\n",
    "    overall_dict[country] = sample_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import a clean function data\n",
    "def clean_text(text):\n",
    "    return text.strip()\n",
    "\n",
    "def text_initial_preproecss(df):\n",
    "    df = df.copy()\n",
    "    df = df[~pd.to_numeric(df['id'], errors='coerce').isnull()]\n",
    "    df[\"id\"] = df[\"id\"].astype(int)\n",
    "    df['article_text'] = df.article_text.str.lower()\n",
    "    df[\"article_text\"] = df[\"article_text\"].apply(clean_text)\n",
    "    df['text'] = df['article_text']\n",
    "    reports = df\n",
    "    \n",
    "    # Puncutation preprocesing\n",
    "    reports['text'] = reports.text.str.replace('{', '')\n",
    "    reports['text'] = reports.text.str.replace('}', '')\n",
    "    reports['text'] = reports.text.str.replace(\"\\n\", '')\n",
    "    reports['text'] = reports.text.str.rstrip(\"\\n\") #remove empty lines\n",
    "    reports['text'] = reports.text.str.replace(\"@ @ @ @ @ @ @ @ @ @ \", '')\n",
    "    reports['text'] = reports.text.str.replace(\" @\", '')\n",
    "    reports['text'] = reports.text.str.replace(\" '\", \"'\")\n",
    "    reports['text'] = reports.text.str.replace(\"\\\"\", \"\")\n",
    "    reports['text'] = reports.text.str.replace(\",\", \"\")\n",
    "    reports['text'] = reports.text.str.replace(\"(\", \"\")\n",
    "    reports['text'] = reports.text.str.replace(\")\", \"\")\n",
    "    reports['text'] = reports.text.str.replace(\" <p>\", \".\")\n",
    "    reports['text'] = reports.text.str.replace(\" <h>\", \".\")\n",
    "    reports['text'] = reports.text.str.replace(\"<p>\", \"\")\n",
    "    reports['text'] = reports.text.str.replace(\"<h>\", \"\")\n",
    "    reports['text'] = reports.text.str.replace('<', '')\n",
    "    reports['text'] = reports.text.str.replace('>', '')\n",
    "    reports['text'] = reports.text.str.replace(\":\", \"\")\n",
    "    reports['text'] = reports.text.str.replace(\"?\", \".\")\n",
    "    reports['text'] = reports.text.str.replace(\"!\", \".\")\n",
    "    reports['text'] = reports.text.str.replace(r\"\\.\\s[\\.\\s]+\", \". \") #converting . . to .\n",
    "    reports['text'] = reports.text.str.replace(r\"\\.+\", \".\") #converting ... to .\n",
    "    reports['text'] = reports.text.str.replace(\"--\", \"\") \n",
    "    reports['text'] = reports.text.str.replace(\"-\", \" \")\n",
    "    reports['text'] = reports.text.str.replace(\" +\", \" \")\n",
    "    reports['text'] = reports.text.str.replace(\" n't\", \"n't\")\n",
    "    \n",
    "    return reports.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import platform\n",
    "os_name = platform.system()\n",
    "\n",
    "def save_file(report,base):\n",
    "#     print(report)\n",
    "#     print(type(report))\n",
    "    report = report.squeeze()\n",
    "        \n",
    "    full_path = report.publisher\n",
    "    file_name = str(report.id) + \".txt\"\n",
    "    \n",
    "    if os_name == 'Windows':\n",
    "        base_dir = 'Process2\\\\{}'.format(base)\n",
    "        con = report.path.path.split('clean\\\\')[1]\n",
    "        \n",
    "    else:\n",
    "        \n",
    "        base_dir = 'Process2/{}'.format(base)\n",
    "        con = report.path.path.split('/clean/')[1]\n",
    "        \n",
    "    path = os.path.join(base_dir,con)\n",
    "        \n",
    "    full_path = os.path.dirname(path)\n",
    "    file_name = os.path.basename(path)\n",
    "    \n",
    "    pathlib.Path(full_path).mkdir(parents=True, exist_ok=True)\n",
    "    with codecs.open(f\"{full_path}/{file_name}\", \"w\", encoding = \"utf-8\") as f:\n",
    "        id_number = str(report.id) if not pd.isna(report.id) else \"\"\n",
    "        title = report.title if not pd.isna(report.title) else \"\"\n",
    "        website = report.publisher if not pd.isna(report.publisher) else \"\"\n",
    "        url = report.url if not pd.isna(report.url) else \"\"\n",
    "        f.writelines([id_number, \"\\n\",\n",
    "                      title, \"\\n\",\n",
    "                      website, \"\\n\",\n",
    "                      url, \"\\n\\n\", report.text])\n",
    "        f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ngram_preprocess(df):\n",
    "    time_taken = []\n",
    "    \n",
    "#     reports_temp = df[df['publisher']==article][['id', 'text','text_vect']].copy()\n",
    "    reports_temp = df.copy()\n",
    "    \n",
    "\n",
    "    reports_temp['text_vect'] = reports_temp.text.apply(sent_tokenize)\n",
    "#     display(reports_temp.head(3))\n",
    "    word_vectorizer = CountVectorizer(ngram_range=(5,5), stop_words=[])\n",
    "    flatten = [item for sublist in reports_temp['text_vect'] for item in sublist]\n",
    "    sparse_matrix = word_vectorizer.fit_transform(flatten)\n",
    "\n",
    "    frequency = sum(sparse_matrix).toarray()[0]\n",
    "    frequency_df = pd.DataFrame(frequency, index=word_vectorizer.get_feature_names(),columns = ['frequency']).sort_values(by=['frequency'],ascending=False)\n",
    "\n",
    "    freq_above_10 = frequency_df[frequency_df['frequency'] >=len(reports_temp)/4]\n",
    "    phrase_list = list(freq_above_10.index)\n",
    "\n",
    "\n",
    "    sentences_to_remove = []\n",
    "\n",
    "    for phrase in phrase_list:\n",
    "        removing_sent = set([sent for sent in flatten if ((phrase in sent) or (len(sent.split()) < 4))])\n",
    "\n",
    "        for sent in removing_sent:\n",
    "            sentences_to_remove.append(sent)\n",
    "\n",
    "    if(\".\" in sentences_to_remove):\n",
    "        while(\".\" in sentences_to_remove):\n",
    "            sentences_to_remove.remove(\".\")\n",
    "    if(\" .\" in sentences_to_remove):\n",
    "        while(\" .\" in sentences_to_remove):\n",
    "            sentences_to_remove.remove(\" .\")\n",
    "\n",
    "#    print(sentences_to_remove)\n",
    "    sentences_to_remove = list(set(sentences_to_remove))\n",
    "    sentences_to_remove.sort(key= len, reverse = True)  # sort starting by largest sentence, in case smaller sentence get chosen beforehand\n",
    "\n",
    "    for sent in sentences_to_remove:\n",
    "        reports_temp.text = reports_temp.text.apply(lambda x: str(x).replace(sent, \"\"))\n",
    "\n",
    "    \n",
    "    return reports_temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cosine_preprocess(df, model, threshold):\n",
    "\n",
    "    sent_removed = []\n",
    "    time_taken = []\n",
    "    new_txts = []\n",
    "    \n",
    "    for i in range(len(df)):\n",
    "        \n",
    "        idx_remove = []\n",
    "        orig_txt = df['text'].iloc[i]\n",
    "        sent_text = nltk.sent_tokenize(orig_txt) #Sentence Segmentation\n",
    "        docu_embeddings = model.encode(orig_txt) #Document Embedding\n",
    "        sentence_embeddings_ = model.encode(sent_text) #Sentence Embedding\n",
    "\n",
    "        for k in range(len(sentence_embeddings_)):\n",
    "            \n",
    "            sim = cosine(docu_embeddings, sentence_embeddings_[k])\n",
    "\n",
    "            if sim > threshold:\n",
    "                idx_remove.append(k)\n",
    "\n",
    "        idx_remove.sort()\n",
    "        idx_remove.reverse()\n",
    "\n",
    "        for j in idx_remove:\n",
    "            sent_removed.append(sent_text[j])\n",
    "            del sent_text[j]\n",
    "\n",
    "        new_txt = ' '.join(sent_text)\n",
    "        new_txts.append(new_txt)\n",
    "        \n",
    "    df['text'] = new_txts\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving original file, to do comparison later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "for keys in overall_dict.keys():\n",
    "    df = overall_dict[keys]\n",
    "    df = text_initial_preproecss(df)\n",
    "    df.apply(save_file,args=('orig',),axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## N-Gram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Methodology: ngram\n",
      "Running time, country: PK, punct_t: 0:00:00.028252, text_proc_t: 0:00:00.161585, total_t: 0:00:00.189837\n",
      "Running time, country: US, punct_t: 0:00:00.018529, text_proc_t: 0:00:00.129167, total_t: 0:00:00.147696\n",
      "Running time, country: HK, punct_t: 0:00:00.020740, text_proc_t: 0:00:00.090714, total_t: 0:00:00.111454\n",
      "Running time, country: GH, punct_t: 0:00:00.020756, text_proc_t: 0:00:00.139624, total_t: 0:00:00.160380\n",
      "Running time, country: NG, punct_t: 0:00:00.030246, text_proc_t: 0:00:00.182570, total_t: 0:00:00.212816\n",
      "Running time, country: KE, punct_t: 0:00:00.032731, text_proc_t: 0:00:00.074514, total_t: 0:00:00.107245\n",
      "Running time, country: NZ, punct_t: 0:00:00.028170, text_proc_t: 0:00:00.103434, total_t: 0:00:00.131604\n",
      "Running time, country: BD, punct_t: 0:00:00.095937, text_proc_t: 0:00:00.163049, total_t: 0:00:00.258986\n",
      "Running time, country: IN, punct_t: 0:00:00.047109, text_proc_t: 0:00:00.338201, total_t: 0:00:00.385310\n",
      "Running time, country: LK, punct_t: 0:00:00.022645, text_proc_t: 0:00:00.029185, total_t: 0:00:00.051830\n",
      "Running time, country: CA, punct_t: 0:00:00.024383, text_proc_t: 0:00:00.109859, total_t: 0:00:00.134242\n",
      "Running time, country: ZA, punct_t: 0:00:00.024149, text_proc_t: 0:00:00.170774, total_t: 0:00:00.194923\n",
      "Running time, country: SG, punct_t: 0:00:00.020228, text_proc_t: 0:00:00.104195, total_t: 0:00:00.124423\n",
      "Running time, country: PH, punct_t: 0:00:00.063024, text_proc_t: 0:00:00.142991, total_t: 0:00:00.206015\n",
      "Running time, country: GB, punct_t: 0:00:00.041752, text_proc_t: 0:00:00.132763, total_t: 0:00:00.174515\n",
      "Running time, country: MY, punct_t: 0:00:00.036697, text_proc_t: 0:00:00.102957, total_t: 0:00:00.139654\n",
      "Running time, country: AU, punct_t: 0:00:00.032547, text_proc_t: 0:00:00.139154, total_t: 0:00:00.171701\n",
      "Running time, country: IE, punct_t: 0:00:00.034504, text_proc_t: 0:00:00.151741, total_t: 0:00:00.186245\n",
      "Running time, country: JM, punct_t: 0:00:00.024218, text_proc_t: 0:00:00.108981, total_t: 0:00:00.133199\n",
      "Running time, country: TZ, punct_t: 0:00:00.035424, text_proc_t: 0:00:00.151910, total_t: 0:00:00.187334\n",
      "0:00:00.170470\n",
      "0:00:03.409409\n"
     ]
    }
   ],
   "source": [
    "cosine_process = False\n",
    "N_gram_process = True\n",
    "merged_process = True if cosine_process and N_gram_process else False\n",
    "cosine_threshold=0.95\n",
    "\n",
    "time_taken = []\n",
    "text_list = []\n",
    "\n",
    "if merged_process:\n",
    "    base='merged'\n",
    "elif cosine_process:\n",
    "    base='cosine'\n",
    "else:\n",
    "    base='ngram'\n",
    "\n",
    "print(\"Methodology: {}\".format(base))\n",
    "  \n",
    "for keys in overall_dict.keys(): # i.e. 'US', \"CA\"\n",
    "    \n",
    "    a = datetime.datetime.now()\n",
    "    \n",
    "    df = overall_dict[keys]   \n",
    "    df = text_initial_preproecss(df)\n",
    "    \n",
    "    b = datetime.datetime.now()\n",
    "    \n",
    "#     df.apply(save_file_initial,args = (base,),axis=1)\n",
    "\n",
    "#     text_list.append(df.iloc[1])\n",
    "    if merged_process:\n",
    "        df = ngram_preprocess(df)\n",
    "        df = cosine_preprocess(df,model,cosine_threshold)\n",
    "\n",
    "    elif cosine_process:\n",
    "        df = cosine_preprocess(df,model,cosine_threshold)\n",
    "\n",
    "    elif N_gram_process:\n",
    "        df = ngram_preprocess(df)\n",
    "\n",
    "    \n",
    "    #saving files\n",
    "    df.apply(save_file,args = (base,),axis=1)\n",
    "\n",
    "    c = datetime.datetime.now()\n",
    "    \n",
    "    print(\"Running time, country: {}, punct_t: {}, text_proc_t: {}, total_t: {}\".format(keys, b-a, c-b, c-a))\n",
    "    time_taken.append(c-a)\n",
    "\n",
    "print(np.mean(time_taken))\n",
    "print(np.sum(time_taken))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cosine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception when trying to download https://sbert.net/models/sentence-transformers/bert-base-nli-stsb-mean-tokens.zip. Response 404\n",
      "WARNING:root:SentenceTransformer-Model https://sbert.net/models/sentence-transformers/bert-base-nli-stsb-mean-tokens.zip not found. Try to create it from scratch\n",
      "WARNING:root:Try to create Transformer Model sentence-transformers/bert-base-nli-stsb-mean-tokens with mean pooling\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Methodology: cosine\n",
      "Running time, country: PK, punct_t: 0:00:00.025535, text_proc_t: 0:00:29.980857, total_t: 0:00:30.006392\n",
      "Running time, country: US, punct_t: 0:00:00.033244, text_proc_t: 0:00:21.887256, total_t: 0:00:21.920500\n",
      "Running time, country: HK, punct_t: 0:00:00.042379, text_proc_t: 0:00:20.013417, total_t: 0:00:20.055796\n",
      "Running time, country: GH, punct_t: 0:00:00.056402, text_proc_t: 0:00:18.373036, total_t: 0:00:18.429438\n",
      "Running time, country: NG, punct_t: 0:00:00.038770, text_proc_t: 0:00:19.092112, total_t: 0:00:19.130882\n",
      "Running time, country: KE, punct_t: 0:00:00.043322, text_proc_t: 0:00:12.386265, total_t: 0:00:12.429587\n",
      "Running time, country: NZ, punct_t: 0:00:00.033503, text_proc_t: 0:00:14.561457, total_t: 0:00:14.594960\n",
      "Running time, country: BD, punct_t: 0:00:00.031834, text_proc_t: 0:00:09.279376, total_t: 0:00:09.311210\n",
      "Running time, country: IN, punct_t: 0:00:00.047133, text_proc_t: 0:00:26.028671, total_t: 0:00:26.075804\n",
      "Running time, country: LK, punct_t: 0:00:00.036595, text_proc_t: 0:00:05.555069, total_t: 0:00:05.591664\n",
      "Running time, country: CA, punct_t: 0:00:00.045484, text_proc_t: 0:00:15.966879, total_t: 0:00:16.012363\n",
      "Running time, country: ZA, punct_t: 0:00:00.043593, text_proc_t: 0:00:22.000077, total_t: 0:00:22.043670\n",
      "Running time, country: SG, punct_t: 0:00:00.041888, text_proc_t: 0:00:16.663181, total_t: 0:00:16.705069\n",
      "Running time, country: PH, punct_t: 0:00:00.041395, text_proc_t: 0:00:18.703596, total_t: 0:00:18.744991\n",
      "Running time, country: GB, punct_t: 0:00:00.037592, text_proc_t: 0:00:16.610173, total_t: 0:00:16.647765\n",
      "Running time, country: MY, punct_t: 0:00:00.045314, text_proc_t: 0:00:12.781915, total_t: 0:00:12.827229\n",
      "Running time, country: AU, punct_t: 0:00:00.031422, text_proc_t: 0:00:21.325338, total_t: 0:00:21.356760\n",
      "Running time, country: IE, punct_t: 0:00:00.058231, text_proc_t: 0:00:20.398317, total_t: 0:00:20.456548\n",
      "Running time, country: JM, punct_t: 0:00:00.038201, text_proc_t: 0:00:17.959495, total_t: 0:00:17.997696\n",
      "Running time, country: TZ, punct_t: 0:00:00.040279, text_proc_t: 0:00:27.131511, total_t: 0:00:27.171790\n",
      "0:00:18.375506\n",
      "0:06:07.510114\n"
     ]
    }
   ],
   "source": [
    "cosine_process = True\n",
    "N_gram_process = False\n",
    "merged_process = True if cosine_process and N_gram_process else False\n",
    "cosine_threshold=0.95\n",
    "\n",
    "model = SentenceTransformer('sentence-transformers/bert-base-nli-stsb-mean-tokens')\n",
    "\n",
    "time_taken = []\n",
    "text_list = []\n",
    "\n",
    "if merged_process:\n",
    "    base='merged'\n",
    "elif cosine_process:\n",
    "    base='cosine'\n",
    "else:\n",
    "    base='ngram'\n",
    "\n",
    "print(\"Methodology: {}\".format(base))\n",
    "  \n",
    "for keys in overall_dict.keys(): # i.e. 'US', \"CA\"\n",
    "    \n",
    "    a = datetime.datetime.now()\n",
    "    \n",
    "    df = overall_dict[keys]   \n",
    "    df = text_initial_preproecss(df)\n",
    "    \n",
    "    b = datetime.datetime.now()\n",
    "    \n",
    "#     df.apply(save_file_initial,args = (base,),axis=1)\n",
    "\n",
    "#     text_list.append(df.iloc[1])\n",
    "    if merged_process:\n",
    "        df = ngram_preprocess(df)\n",
    "        df = cosine_preprocess(df,model,cosine_threshold)\n",
    "\n",
    "    elif cosine_process:\n",
    "        df = cosine_preprocess(df,model,cosine_threshold)\n",
    "\n",
    "    elif N_gram_process:\n",
    "        df = ngram_preprocess(df)\n",
    "\n",
    "    \n",
    "    #saving files\n",
    "    df.apply(save_file,args = (base,),axis=1)\n",
    "\n",
    "    c = datetime.datetime.now()\n",
    "    \n",
    "    print(\"Running time, country: {}, punct_t: {}, text_proc_t: {}, total_t: {}\".format(keys, b-a, c-b, c-a))\n",
    "    time_taken.append(c-a)\n",
    "\n",
    "print(np.mean(time_taken))\n",
    "print(np.sum(time_taken))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception when trying to download https://sbert.net/models/sentence-transformers/bert-base-nli-stsb-mean-tokens.zip. Response 404\n",
      "WARNING:root:SentenceTransformer-Model https://sbert.net/models/sentence-transformers/bert-base-nli-stsb-mean-tokens.zip not found. Try to create it from scratch\n",
      "WARNING:root:Try to create Transformer Model sentence-transformers/bert-base-nli-stsb-mean-tokens with mean pooling\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Methodology: merged\n",
      "Running time, country: PK, punct_t: 0:00:00.043334, text_proc_t: 0:00:51.024576, total_t: 0:00:51.067910\n",
      "Running time, country: US, punct_t: 0:00:00.050589, text_proc_t: 0:00:34.143380, total_t: 0:00:34.193969\n",
      "Running time, country: HK, punct_t: 0:00:00.042564, text_proc_t: 0:00:30.855597, total_t: 0:00:30.898161\n",
      "Running time, country: GH, punct_t: 0:00:00.037785, text_proc_t: 0:00:22.952209, total_t: 0:00:22.989994\n",
      "Running time, country: NG, punct_t: 0:00:00.060841, text_proc_t: 0:00:22.611507, total_t: 0:00:22.672348\n",
      "Running time, country: KE, punct_t: 0:00:00.049623, text_proc_t: 0:00:17.964988, total_t: 0:00:18.014611\n",
      "Running time, country: NZ, punct_t: 0:00:00.060705, text_proc_t: 0:00:18.633917, total_t: 0:00:18.694622\n"
     ]
    }
   ],
   "source": [
    "cosine_process = True\n",
    "N_gram_process = True\n",
    "merged_process = True if cosine_process and N_gram_process else False\n",
    "cosine_threshold=0.95\n",
    "\n",
    "model = SentenceTransformer('sentence-transformers/bert-base-nli-stsb-mean-tokens')\n",
    "\n",
    "time_taken = []\n",
    "text_list = []\n",
    "\n",
    "if merged_process:\n",
    "    base='merged'\n",
    "elif cosine_process:\n",
    "    base='cosine'\n",
    "else:\n",
    "    base='ngram'\n",
    "\n",
    "print(\"Methodology: {}\".format(base))\n",
    "  \n",
    "for keys in overall_dict.keys(): # i.e. 'US', \"CA\"\n",
    "    \n",
    "    a = datetime.datetime.now()\n",
    "    \n",
    "    df = overall_dict[keys]   \n",
    "    df = text_initial_preproecss(df)\n",
    "    \n",
    "    b = datetime.datetime.now()\n",
    "    \n",
    "#     df.apply(save_file_initial,args = (base,),axis=1)\n",
    "\n",
    "#     text_list.append(df.iloc[1])\n",
    "    if merged_process:\n",
    "        df = ngram_preprocess(df)\n",
    "        df = cosine_preprocess(df,model,cosine_threshold)\n",
    "\n",
    "    elif cosine_process:\n",
    "        df = cosine_preprocess(df,model,cosine_threshold)\n",
    "\n",
    "    elif N_gram_process:\n",
    "        df = ngram_preprocess(df)\n",
    "\n",
    "    \n",
    "    #saving files\n",
    "    df.apply(save_file,args = (base,),axis=1)\n",
    "\n",
    "    c = datetime.datetime.now()\n",
    "    \n",
    "    print(\"Running time, country: {}, punct_t: {}, text_proc_t: {}, total_t: {}\".format(keys, b-a, c-b, c-a))\n",
    "    time_taken.append(c-a)\n",
    "\n",
    "print(np.mean(time_taken))\n",
    "print(np.sum(time_taken))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}