Get Lexicon Term Counts.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\mattb\\\\Documents\\\\GitHub\\\\peace-speech-project\\\\data\\\\clean_sample'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import pathlib\n",
    "import warnings\n",
    "warnings.filterwarnings(action=\"ignore\")\n",
    "\n",
    "import pprint\n",
    "import itertools\n",
    "from tqdm import tqdm\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "from utils.reading import read_all_files, get_text\n",
    "from utils.common import get_country_labels, get_society_label\n",
    "\n",
    "# CLEAN_DATA_FOLDER = os.path.join(\"/Users\", \"mmackenzie\", \"Data\", \"peace-speech-project\", \"clean_sample\")\n",
    "CLEAN_DATA_FOLDER = r\"C:\\Users\\mattb\\Documents\\GitHub\\peace-speech-project\\data\\clean_sample\"\n",
    "CLEAN_DATA_FOLDER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "articles = read_all_files(path=CLEAN_DATA_FOLDER)\n",
    "articles_with_text = get_text(articles, path=CLEAN_DATA_FOLDER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 24.2 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "US    110197\n",
       "IN    109777\n",
       "AU    109475\n",
       "CA    109119\n",
       "GB    107366\n",
       "NZ    107278\n",
       "ZA    105460\n",
       "IE    102635\n",
       "PH     95348\n",
       "NG     79354\n",
       "MY     59589\n",
       "SG     56211\n",
       "KE     56132\n",
       "PK     55157\n",
       "JM     52523\n",
       "GH     35686\n",
       "BD     23400\n",
       "TZ     16861\n",
       "LK     14676\n",
       "HK      5069\n",
       "Name: country, dtype: int64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "countries = list(itertools.chain(*get_country_labels().values()))\n",
    "articles_with_text = pd.read_csv(\"data/clean_sample/all_data.csv\")\n",
    "articles_with_text = articles_with_text[~articles_with_text.text.isna()]\n",
    "articles_with_text.country.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Peaceful       592084\n",
       "Other          588325\n",
       "Nonpeaceful    230904\n",
       "Name: society, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "country_labels = get_country_labels()\n",
    "\n",
    "articles_with_text[\"society\"] = articles_with_text.country.apply(get_society_label, country_labels=country_labels)\n",
    "# articles_with_text = articles_with_text[articles_with_text.society != \"Other\"]\n",
    "articles = articles_with_text.drop(\"text\", axis=1)\n",
    "\n",
    "articles.society.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>term</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>ngati</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>iona</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>vg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>mcstay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>kaipara</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7075</th>\n",
       "      <td>7075</td>\n",
       "      <td>inishmaan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7076</th>\n",
       "      <td>7076</td>\n",
       "      <td>peka</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7077</th>\n",
       "      <td>7077</td>\n",
       "      <td>leck</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7078</th>\n",
       "      <td>7078</td>\n",
       "      <td>eth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7079</th>\n",
       "      <td>7079</td>\n",
       "      <td>mayo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7080 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index       term\n",
       "0         0      ngati\n",
       "1         1       iona\n",
       "2         2         vg\n",
       "3         3     mcstay\n",
       "4         4    kaipara\n",
       "...     ...        ...\n",
       "7075   7075  inishmaan\n",
       "7076   7076       peka\n",
       "7077   7077       leck\n",
       "7078   7078        eth\n",
       "7079   7079       mayo\n",
       "\n",
       "[7080 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(\"lexicons/peace_vocab_rationale.csv\", names=[\"index\", \"term\"], skiprows=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>term</th>\n",
       "      <th>lexicon</th>\n",
       "      <th>version</th>\n",
       "      <th>num_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>academic freedom</td>\n",
       "      <td>peace</td>\n",
       "      <td>Original</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>acceptance</td>\n",
       "      <td>peace</td>\n",
       "      <td>Original</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>accepting</td>\n",
       "      <td>peace</td>\n",
       "      <td>Original</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>active listening</td>\n",
       "      <td>peace</td>\n",
       "      <td>Original</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>activism</td>\n",
       "      <td>peace</td>\n",
       "      <td>Original</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2827</th>\n",
       "      <td>podcast</td>\n",
       "      <td>conflict</td>\n",
       "      <td>Attention Layer</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2828</th>\n",
       "      <td>menace</td>\n",
       "      <td>conflict</td>\n",
       "      <td>Attention Layer</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2829</th>\n",
       "      <td>pdp</td>\n",
       "      <td>conflict</td>\n",
       "      <td>Attention Layer</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2830</th>\n",
       "      <td>aspirant</td>\n",
       "      <td>conflict</td>\n",
       "      <td>Attention Layer</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2831</th>\n",
       "      <td>vanguard</td>\n",
       "      <td>conflict</td>\n",
       "      <td>Attention Layer</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2832 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  term   lexicon          version  num_words\n",
       "0     academic freedom     peace         Original          2\n",
       "1           acceptance     peace         Original          1\n",
       "2            accepting     peace         Original          1\n",
       "3     active listening     peace         Original          2\n",
       "4             activism     peace         Original          1\n",
       "...                ...       ...              ...        ...\n",
       "2827           podcast  conflict  Attention Layer          1\n",
       "2828            menace  conflict  Attention Layer          1\n",
       "2829               pdp  conflict  Attention Layer          1\n",
       "2830          aspirant  conflict  Attention Layer          1\n",
       "2831          vanguard  conflict  Attention Layer          1\n",
       "\n",
       "[2832 rows x 4 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def read_lexicon(lexicon):\n",
    "    path = os.path.join(\"lexicons\", f\"enh_{lexicon}_lexicon.xlsx\")\n",
    "    df = pd.read_excel(path, names=[\"term\"])\n",
    "    df[\"lexicon\"] = lexicon\n",
    "    \n",
    "    return df\n",
    "\n",
    "old_lexicon = pd.concat([read_lexicon(l) for l in [\"peace\", \"conflict\", \"resilience\"]])\n",
    "old_lexicon[\"version\"] = \"Original\"\n",
    "\n",
    "tf_lexicon = pd.read_csv(\"lexicons/top_words_lexicon.csv\")\n",
    "tf_lexicon[\"version\"] = \"Term Frequency\"\n",
    "\n",
    "attention_lexicon = pd.read_csv(\"lexicons/attention_lexicon.csv\")\n",
    "attention_lexicon = attention_lexicon.iloc[:, 1:].stack().reset_index().iloc[:, 1:]\n",
    "attention_lexicon.columns = [\"lexicon\", \"term\"]\n",
    "attention_lexicon[\"lexicon\"] = attention_lexicon.lexicon.apply(lambda x: {\"Peaceful\": \"peace\", \"Non Peaceful\": \"conflict\"}[x])\n",
    "attention_lexicon[\"term\"] = attention_lexicon.term.str.strip('][').str.split(', ')\n",
    "attention_lexicon = attention_lexicon.explode(\"term\")\n",
    "attention_lexicon[\"term\"] = attention_lexicon.term.str.strip('\\'')\n",
    "\n",
    "attention_lexicon[\"version\"] = \"Attention Layer\"\n",
    "\n",
    "r\n",
    "\n",
    "lexicon = pd.concat([\n",
    "    old_lexicon, \n",
    "    tf_lexicon,\n",
    "    attention_lexicon\n",
    "]).reset_index(drop=True)\n",
    "\n",
    "lexicon[\"term\"] = lexicon.term.str.lower()\n",
    "lexicon = lexicon.drop_duplicates([\"lexicon\", \"version\", \"term\"]).reset_index(drop=True)\n",
    "\n",
    "lexicon[\"num_words\"] = lexicon.term.str.split(\" \").str.len()\n",
    "lexicon.to_csv(\"lexicons/lexicon.csv\", index=False) \n",
    "\n",
    "lexicon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2389,)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique_terms = lexicon[lexicon.num_words <= 2].term.unique()\n",
    "unique_terms.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "cv = CountVectorizer(ngram_range=(1, 2), vocabulary=unique_terms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 6min 51s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<1411313x2389 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 48877502 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "word_counts = cv.fit_transform(articles_with_text.text)\n",
    "word_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "del articles_with_text\n",
    "# del counts_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 7.76 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "counts_df =  pd.DataFrame(word_counts.todense(), columns=cv.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "counts_df[\"society\"] = articles[\"society\"].values\n",
    "counts_df[\"country\"] = articles[\"country\"].values\n",
    "counts_df[\"year\"] = articles[\"year\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "del articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "counts_df.to_csv(\"data/word_counts_by_article_ALL.csv\", index=False)\n",
    "counts_df.loc[counts_df.society != \"Other\", :].to_csv(\"data/word_counts_by_article_NO_OTHER.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "counts_df = pd.read_csv(\"data/word_counts_by_article_ALL.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = counts_df.columns[:-3]\n",
    "def make_count_summary(group, df, features=features, lexicon=lexicon):\n",
    "    summary = df[features] \\\n",
    "        .sum(axis=0) \\\n",
    "        .rename(\"n\") \\\n",
    "        .reset_index() \\\n",
    "        .rename(columns={\"index\": \"term\"})\n",
    "    \n",
    "    summary[\"society\"] = group[0]\n",
    "    summary[\"country\"] = group[1]\n",
    "    summary[\"year\"] = group[2]\n",
    "    \n",
    "    summary = summary.merge(\n",
    "        lexicon, \n",
    "            on = \"term\", \n",
    "            how = \"left\"\n",
    "        ).drop_duplicates([\"lexicon\", \"version\", \"term\"])\n",
    "    \n",
    "    summary.to_csv(f\"data/word_counts/tf_by_year__{'_'.join([str(g) for g in group[1:]]).upper()}.csv\", index=False)    \n",
    "    return summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 220/220 [03:21<00:00,  1.09it/s]\n"
     ]
    }
   ],
   "source": [
    "summaries = []\n",
    "for g, df in tqdm(counts_df.groupby([\"society\", \"country\", \"year\"])):\n",
    "    summaries.append(make_count_summary(g, df))\n",
    "    \n",
    "count_df_summary = pd.concat(summaries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_df_summary.to_csv(\"data/tf_by_country_by_year.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "del summaries\n",
    "del count_df_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 3min 54s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "corrcoef = np.corrcoef(word_counts.toarray().T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(corrcoef, columns=cv.get_feature_names(), index=cv.get_feature_names()) \\\n",
    "    .stack() \\\n",
    "    .rename(\"corr\") \\\n",
    "    .reset_index() \\\n",
    "    .rename(columns={\"level_0\": \"term1\",\n",
    "                     \"level_1\": \"term2\"}) \\\n",
    "    .query(\"term1 != term2\") \\\n",
    "    .to_csv(\"data/sample_term_corr.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1411313, 2389)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_counts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}