Data preparation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import collections\n",
    "import itertools\n",
    "import numpy as np\n",
    "import datetime\n",
    "import json\n",
    "from dateutil import parser\n",
    "\n",
    "tqdm.pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_date(date):\n",
    "    json_acceptable_string = date.replace(\"'\", \"\\\"\")\n",
    "    d = json.loads(json_acceptable_string)\n",
    "    return(parser.parse(d['$date']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def person_in_headline(headline, persons):\n",
    "    if headline:\n",
    "        headline_dict = {}\n",
    "        for i in persons:\n",
    "            if i in headline:\n",
    "                if i in headline_dict:\n",
    "                    headline_dict[i] += 1\n",
    "                else:\n",
    "                    headline_dict[i] = 1\n",
    "        return headline_dict\n",
    "    else:\n",
    "        return {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def person_in_text(text, persons):\n",
    "    text_dict = {}\n",
    "    for person in persons:\n",
    "        text_dict[person] = text.count(person)\n",
    "    return text_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dhaka Tribune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_pickle('Data/Processed Data/Networks/DT-network.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['entity0'] = df['entity0'].apply(lambda x: \"Sheikh Hasina\" if x == \"Sheikh Hasina,\" else x)\n",
    "df['entity1'] = df['entity1'].apply(lambda x: \"Sheikh Hasina\" if x == \"Sheikh Hasina,\" else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 247369/247369 [00:13<00:00, 17926.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24925\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "G = nx.Graph()\n",
    "network_type = 'person'\n",
    "count = 0\n",
    "for _, row in tqdm(df.iterrows(), total=df.shape[0]):\n",
    "    \n",
    "    if row['type0'] in network_type and row['type1'] in network_type:\n",
    "        # print(row['entity0'], row['entity1'])\n",
    "        count += 1\n",
    "        G.add_edge(row['entity0'], row['entity1'])\n",
    "\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10132\n",
      "16867\n",
      "0.00032863994112966957\n"
     ]
    }
   ],
   "source": [
    "print(len(G.nodes))\n",
    "print(len(G.edges))\n",
    "print(nx.density(G))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Sheikh Hasina', 0.042740104629355444),\n",
       " ('Khaleda Zia', 0.03405389398874741),\n",
       " ('Ershad', 0.017471128220313888),\n",
       " ('Hossain', 0.011252591057151318),\n",
       " ('Rahman', 0.010759056361662226),\n",
       " ('Monirul', 0.010166814727075313),\n",
       " ('Tarique Rahman', 0.009475866153390584),\n",
       " ('Khokon', 0.009475866153390584),\n",
       " ('Nur Hossain', 0.009278452275194946),\n",
       " ('Babul', 0.008192675945118941),\n",
       " ('Ziaur Rahman', 0.007896555127825486),\n",
       " ('Kamal', 0.007797848188727668),\n",
       " ('Latif', 0.007008192675945119),\n",
       " ('Shamim', 0.007008192675945119),\n",
       " ('Masud', 0.0069094857368473),\n",
       " ('Mizanur Rahman', 0.0069094857368473),\n",
       " ('Hannan', 0.006613364919553845),\n",
       " ('Bangabandhu Sheikh Mujibur Rahman', 0.006514657980456026),\n",
       " ('Mamun', 0.006218537163162571),\n",
       " ('Kabir', 0.0060211232849669335)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "degree_centrality_G = nx.degree_centrality(G)\n",
    "sorted(degree_centrality_G.items(), key = lambda x:x[1], reverse= True)[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Sheikh Hasina', 0.008714946541820276),\n",
       " ('Khaleda Zia', 0.0065276253347618515),\n",
       " ('Ershad', 0.0033629463372757845),\n",
       " ('Hossain', 0.0023554989532027773),\n",
       " ('Rahman', 0.0022351802397646008),\n",
       " ('Monirul', 0.0020809714059857303),\n",
       " ('Khokon', 0.0020029827831596937),\n",
       " ('Nur Hossain', 0.001861798382496689),\n",
       " ('Babul', 0.0018555368742018648),\n",
       " ('Tarique Rahman', 0.0017628393955292435),\n",
       " ('Ziaur Rahman', 0.00155272881742216),\n",
       " ('Kamal', 0.0015279512046905848),\n",
       " ('Mizanur Rahman', 0.0014902815077797804),\n",
       " ('Hannan', 0.0013829262783952648),\n",
       " ('Masud', 0.0013666953228968159),\n",
       " ('Shamim', 0.0013530665774883657),\n",
       " ('Mamun', 0.0013335490807444776),\n",
       " ('Latif', 0.0013117671578834467),\n",
       " ('Kabir', 0.0012747192498525433),\n",
       " ('Bangabandhu Sheikh Mujibur Rahman', 0.0012723027199051877)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pr_G = nx.pagerank(G)\n",
    "sorted(pr_G.items(), key = lambda x:x[1], reverse= True)[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Sarker', 9),\n",
       " ('Hasanul Haque', 9),\n",
       " ('Shahriar Kabir', 9),\n",
       " ('Muhammad Zafar Iqbal', 9),\n",
       " ('Arefin Siddique', 9),\n",
       " ('Sangeeta Imam', 9),\n",
       " ('Ramendu Majumdar', 9),\n",
       " ('Nasiruddin Yusuf', 9),\n",
       " ('Mahmudul Haque Munshi Gonojagoron Moncho', 9),\n",
       " ('Syed Anwar Hossain', 9),\n",
       " ('Kamal', 8),\n",
       " ('Shamim Osman', 8),\n",
       " ('Sheikh Hasina', 8),\n",
       " ('Obaidul Quader', 8),\n",
       " ('Osman', 8),\n",
       " ('Mirza Fakhrul Islam Alamgir', 8),\n",
       " ('Kamal Hossain', 8),\n",
       " ('Nur Hossain', 8),\n",
       " ('Matia Chowdhury', 8),\n",
       " ('Ziaur Rahman', 8)]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "G.remove_edges_from(G.selfloop_edges())\n",
    "sorted(nx.core_number(G).items(), key= lambda x:x[1], reverse=True)[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# G2=nx.Graph()\n",
    "\n",
    "# df = df[((df['type0'] == 'person') & (df['type1'] == 'person'))]\n",
    "# G2.add_edge(row['entity0'], row['entity1'])\n",
    "# G2 = nx.from_pandas_edgelist(df, 'entity0', 'entity1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_pickle('Data/DT/DT.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = collections.Counter([item for sublist in df['persons_unique'] for item in sublist])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Sheikh Hasina', 4543),\n",
       " ('Khaleda Zia', 2616),\n",
       " ('Bangabandhu Sheikh Mujibur Rahman', 739),\n",
       " ('Abdul Hamid', 653),\n",
       " ('Mirza Fakhrul Islam Alamgir', 639),\n",
       " ('Mizanur Rahman', 551),\n",
       " ('Tarique Rahman', 477),\n",
       " ('Obaidul Quader', 433),\n",
       " ('Motiur Rahman Nizami', 395),\n",
       " ('Abul Kalam', 393),\n",
       " ('Ershad', 353),\n",
       " ('Abul Kalam Azad', 341),\n",
       " ('Ziaur Rahman', 334),\n",
       " ('Abul Maal Abdul Muhith', 334),\n",
       " ('Anwar Hossain', 304),\n",
       " ('Mahbubey Alam', 299),\n",
       " ('Habibur Rahman', 299),\n",
       " ('Nurul Islam Nahid', 288),\n",
       " ('Asaduzzaman Khan Kamal', 276),\n",
       " ('Shahidul Haque', 264)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count.most_common()[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_pickle('Data/DT/DT.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['person_in_headline'] = np.vectorize(person_in_headline)(df['news_headline'], df['persons_unique'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['person_in_text'] = np.vectorize(person_in_text)(df['news_text'], df['persons_unique'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 49055/49055 [12:10<00:00, 67.11it/s]\n"
     ]
    }
   ],
   "source": [
    "count_person_in_text = sum(map(collections.Counter, tqdm(df['person_in_text'].values.tolist())), collections.Counter())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 49055/49055 [00:22<00:00, 2134.80it/s]\n"
     ]
    }
   ],
   "source": [
    "count_person_in_headline = sum(map(collections.Counter, tqdm(df['person_in_headline'].values.tolist())), collections.Counter())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_text = pd.DataFrame.from_dict(count_person_in_text, orient='index')\n",
    "df_headline = pd.DataFrame.from_dict(count_person_in_headline, orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "pagerank = pd.DataFrame.from_dict(pr_G, orient='index')\n",
    "pagerank = pagerank.reset_index()\n",
    "pagerank = pagerank.sort_values(by='index')\n",
    "pagerank.columns = ['name', 'pagerank']\n",
    "\n",
    "degree_centrality = pd.DataFrame.from_dict(degree_centrality_G, orient='index')\n",
    "degree_centrality = degree_centrality.reset_index()\n",
    "degree_centrality = degree_centrality.sort_values(by='index')\n",
    "degree_centrality.columns = ['name', 'degree_centrality']\n",
    "\n",
    "G.remove_edges_from(G.selfloop_edges())\n",
    "core = nx.core_number(G)\n",
    "core_periphery = pd.DataFrame.from_dict(core, orient='index')\n",
    "core_periphery = core_periphery.reset_index()\n",
    "core_periphery = core_periphery.sort_values(by='index')\n",
    "core_periphery.columns = ['name', 'core_periphery']\n",
    "\n",
    "count = dict(count)\n",
    "person_count = pd.DataFrame.from_dict(count, orient='index')\n",
    "person_count = person_count.reset_index()\n",
    "person_count = person_count.sort_values(by='index')\n",
    "person_count.columns = ['name', 'count']\n",
    "\n",
    "df_headline = df_headline.reset_index()\n",
    "df_headline = df_headline.sort_values(by='index')\n",
    "df_headline.columns = ['name', 'headline_count']\n",
    "\n",
    "df_text = df_text.reset_index()\n",
    "df_text = df_text.sort_values(by='index')\n",
    "df_text.columns = ['name', 'text_count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "ml = pd.DataFrame()\n",
    "\n",
    "ml = pd.merge(pagerank, core_periphery, on=\"name\")\n",
    "ml = pd.merge(ml, degree_centrality, on=\"name\")\n",
    "ml = pd.merge(ml, person_count, on=\"name\", how=\"left\")\n",
    "ml = pd.merge(ml, df_headline, on=\"name\", how=\"left\")\n",
    "ml = pd.merge(ml, df_text, on=\"name\", how=\"left\")\n",
    "ml.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "ml.drop(ml.index[0:9], inplace=True)\n",
    "ml.reset_index(drop=True, inplace=True)\n",
    "ml.drop(ml.index[-7:], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ml.to_pickle('Data/Processed Data/ML/DT-ml.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Daily Star"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_pickle('Data/Processed Data/Networks/DS-network.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "G = nx.Graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914208/914208 [00:59<00:00, 15437.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "170661\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "network_type = 'person'\n",
    "count = 0\n",
    "for _, row in tqdm(df.iterrows(), total=df.shape[0]):\n",
    "    \n",
    "    if row['type0'] in network_type and row['type1'] in network_type:\n",
    "        # print(row['entity0'], row['entity1'])\n",
    "        count += 1\n",
    "        G.add_edge(row['entity0'], row['entity1'])\n",
    "        \n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "45684\n",
      "123931\n"
     ]
    }
   ],
   "source": [
    "print(len(G.nodes))\n",
    "print(len(G.edges))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Khan', 0.013133988573429942),\n",
       " ('Barack Obama', 0.012980758706739925),\n",
       " ('Rahman', 0.012192719392334128),\n",
       " ('Ali', 0.009872381411028173),\n",
       " ('Ahmed', 0.009872381411028173),\n",
       " ('Clinton', 0.009390801830002408),\n",
       " ('Nazrul', 0.008953002210888076),\n",
       " ('Sheikh Hasina', 0.00886544228706521),\n",
       " ('Rab', 0.008734102401330912),\n",
       " ('Hossain', 0.008405752686995163),\n",
       " ('Bush', 0.00746448350589935),\n",
       " ('Singh', 0.007136133791563601),\n",
       " ('Gaddafi', 0.006829674058183569),\n",
       " ('Khaleda Zia', 0.006216754591423505),\n",
       " ('Hillary Clinton', 0.0061510846485563556),\n",
       " ('Musharraf', 0.005778954972309174),\n",
       " ('Assad', 0.005275485410327693),\n",
       " ('Chowdhury', 0.005209815467460543),\n",
       " ('Abbas', 0.005056585600770528),\n",
       " ('Manmohan Singh', 0.004947135695991945)]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "degree_centrality_G = nx.degree_centrality(G)\n",
    "sorted(degree_centrality_G.items(), key = lambda x:x[1], reverse= True)[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Khan', 0.0020221291101969676),\n",
       " ('Barack Obama', 0.0018603863764148833),\n",
       " ('Rahman', 0.0016470698180094314),\n",
       " ('Ali', 0.00143925353838546),\n",
       " ('Ahmed', 0.0013892710589818492),\n",
       " ('Clinton', 0.001359444219986478),\n",
       " ('Hossain', 0.001194355049643284),\n",
       " ('Rab', 0.0011939810349725446),\n",
       " ('Gaddafi', 0.0011799060070162564),\n",
       " ('Sheikh Hasina', 0.0011410663011670868),\n",
       " ('Nazrul', 0.0010968293976068345),\n",
       " ('Bush', 0.0010825096553345776),\n",
       " ('Singh', 0.0010689226742943228),\n",
       " ('Musharraf', 0.0008674203692742141),\n",
       " ('Assad', 0.0008148937602715171),\n",
       " ('Hillary Clinton', 0.0008097963993054698),\n",
       " ('Khaleda Zia', 0.0007314108114951344),\n",
       " ('Chowdhury', 0.0007280814787891742),\n",
       " ('Abbas', 0.0007247929345799016),\n",
       " ('Kim', 0.0006925116346644017)]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pr_G = nx.pagerank(G)\n",
    "sorted(pr_G.items(), key = lambda x:x[1], reverse= True)[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Shakti Chattapadhaya', 39),\n",
       " ('Shaheed Subedar Mailkiat Singh', 39),\n",
       " ('Mrinmoyee Bose', 39),\n",
       " ('Shukharanjan Sengupta', 39),\n",
       " ('Samar Ranjan Sen', 39),\n",
       " ('Maj Gen Lachhman Singh Lehl', 39),\n",
       " ('Waheeda', 39),\n",
       " ('Waheeda Rehman', 39),\n",
       " ('Ashok Ray', 39),\n",
       " ('Zainal Abedin', 39),\n",
       " ('Shalil Ghosh', 39),\n",
       " ('Rabindranath Chowdhury', 39),\n",
       " ('Swaroop Krishna Kaul', 39),\n",
       " ('Pranabranjan Ray', 39),\n",
       " ('Kishore Parekh', 39),\n",
       " ('Biswajit R Chatterjee', 39),\n",
       " ('Subhash Mukhopadhaya', 39),\n",
       " ('Maulana Syed Asad Madni', 39),\n",
       " ('Asghar Ali', 39),\n",
       " ('Aniruddha Ray', 39)]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "G.remove_edges_from(G.selfloop_edges())\n",
    "sorted(nx.core_number(G).items(), key= lambda x:x[1], reverse=True)[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_pickle('Data/DS/DS.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = collections.Counter([item for sublist in df['ner_unique_person'] for item in sublist])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Barack Obama', 4951),\n",
       " ('Sheikh Hasina', 3881),\n",
       " ('Rabindranath Tagore', 2764),\n",
       " ('Khaleda Zia', 2283),\n",
       " ('Rahman', 1949),\n",
       " ('Bangabandhu Sheikh Mujibur Rahman', 1933),\n",
       " ('Rab', 1456),\n",
       " ('Manmohan Singh', 1375),\n",
       " ('Hossain', 1316),\n",
       " ('Nazrul', 1310),\n",
       " ('Hillary Clinton', 1257),\n",
       " ('Ahmed', 1179),\n",
       " ('Ali', 1110),\n",
       " ('Mizanur Rahman', 979),\n",
       " ('Bashar al-Assad', 943),\n",
       " ('Narendra Modi', 942),\n",
       " ('Abul Kalam Azad', 909),\n",
       " ('Rafiqul Islam', 899),\n",
       " ('Khan', 885),\n",
       " ('Hamid Karzai', 858)]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count.most_common()[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['person_in_headline'] = np.vectorize(person_in_headline)(df['title'], df['ner_unique_person'])\n",
    "df['person_in_text'] = np.vectorize(person_in_text)(df['content'], df['ner_unique_person'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 165236/165236 [08:55<00:00, 308.65it/s]\n"
     ]
    }
   ],
   "source": [
    "count_person_in_headline = sum(map(collections.Counter, tqdm(df['person_in_headline'].values.tolist())), collections.Counter())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_headline = pd.DataFrame.from_dict(count_person_in_headline, orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# count_person_in_text = sum(map(collections.Counter, tqdm(df['person_in_text'].values.tolist())), collections.Counter())\n",
    "# df_text = pd.DataFrame.from_dict(count_person_in_text, orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_text = pd.read_pickle('Data/Processed Data/Misc/DS-Text-Count.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "pagerank = pd.DataFrame.from_dict(pr_G, orient='index')\n",
    "pagerank = pagerank.reset_index()\n",
    "pagerank = pagerank.sort_values(by='index')\n",
    "pagerank.columns = ['name', 'pagerank']\n",
    "\n",
    "degree_centrality = pd.DataFrame.from_dict(degree_centrality_G, orient='index')\n",
    "degree_centrality = degree_centrality.reset_index()\n",
    "degree_centrality = degree_centrality.sort_values(by='index')\n",
    "degree_centrality.columns = ['name', 'degree_centrality']\n",
    "\n",
    "G.remove_edges_from(G.selfloop_edges())\n",
    "core = nx.core_number(G)\n",
    "\n",
    "core_periphery = pd.DataFrame.from_dict(core, orient='index')\n",
    "core_periphery = core_periphery.reset_index()\n",
    "core_periphery = core_periphery.sort_values(by='index')\n",
    "core_periphery.columns = ['name', 'core_periphery']\n",
    "\n",
    "count = dict(count)\n",
    "person_count = pd.DataFrame.from_dict(count, orient='index')\n",
    "person_count = person_count.reset_index()\n",
    "person_count = person_count.sort_values(by='index')\n",
    "person_count.columns = ['name', 'count']\n",
    "\n",
    "df_headline = df_headline.reset_index()\n",
    "df_headline = df_headline.sort_values(by='index')\n",
    "df_headline.columns = ['name', 'headline_count']\n",
    "\n",
    "df_text = df_text.reset_index()\n",
    "df_text = df_text.sort_values(by='index')\n",
    "df_text.columns = ['name', 'text_count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "ml = pd.DataFrame()\n",
    "ml = pd.merge(pagerank, core_periphery, on=\"name\")\n",
    "ml = pd.merge(ml, degree_centrality, on=\"name\")\n",
    "ml = pd.merge(ml, person_count, on=\"name\", how=\"left\")\n",
    "ml = pd.merge(ml, df_headline, on=\"name\", how=\"left\")\n",
    "ml = pd.merge(ml, df_text, on=\"name\", how=\"left\")\n",
    "ml.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "ml.drop(ml.index[0:86], inplace=True)\n",
    "ml.reset_index(drop=True, inplace=True)\n",
    "ml.drop(ml.index[-153:], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ml.to_pickle('Data/Processed Data/ML/DS-ml.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_pickle('Data/Processed Data/Networks/NA-network.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "G = nx.Graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 820506/820506 [00:50<00:00, 16353.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "79369\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "network_type = 'person'\n",
    "exclusions = ['Rakhine', 'Test', 'Dhaka', 'Bangla', 'Bangladesh', 'Narayanganj', 'AL', 'Gulshan', 'Dhanmondi', 'Bogra', 'Rongpur', 'Barisal', 'Rangpur', 'Comilla', 'Chelsea', 'Rajshahi', 'Mymensingh', 'Sadar', 'Sylhet', 'Cox', 'Twitter', 'Barca', 'Kolkata', 'Teknaf', 'Ukhiya', 'Bandarban', 'Tripura', 'Abahani']\n",
    "count = 0\n",
    "for _, row in tqdm(df.iterrows(), total=df.shape[0]):\n",
    "    \n",
    "    if row['type0'] in network_type and row['type1'] in network_type:\n",
    "        if row['entity0'] not in exclusions and row['entity1'] not in exclusions:\n",
    "            # print(row['entity0'], row['entity1'])\n",
    "            count += 1\n",
    "            G.add_edge(row['entity0'], row['entity1'])\n",
    "        \n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23874\n",
      "64591\n"
     ]
    }
   ],
   "source": [
    "print(len(G.nodes))\n",
    "print(len(G.edges))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Sheikh Hasina', 0.02530054873706698),\n",
       " ('Donald Trump', 0.02127926946759938),\n",
       " ('Khaleda Zia', 0.00959242659070917),\n",
       " ('Putin', 0.00908976668202572),\n",
       " ('Clinton', 0.00908976668202572),\n",
       " ('Rahman', 0.007875005236040716),\n",
       " ('Khan', 0.007707451933146233),\n",
       " ('Ali', 0.00758178695597537),\n",
       " ('Mustafizur Rahman', 0.006618355464332091),\n",
       " ('Quader', 0.006325137184266745),\n",
       " ('Tofail Ahmed', 0.006241360532819503),\n",
       " ('Suu Kyi', 0.006157583881372262),\n",
       " ('Abdul Hamid', 0.00607380722992502),\n",
       " ('Nasir', 0.005906253927030537),\n",
       " ('Barack Obama', 0.005696812298412433),\n",
       " ('Abdur Razzak', 0.0056549239726888115),\n",
       " ('Bush', 0.005529258995517949),\n",
       " ('Mujib', 0.005403594018347087),\n",
       " ('Mourinho', 0.0052779290411762245),\n",
       " ('Aung San Suu Kyi', 0.005194152389728983)]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "degree_centrality_G = nx.degree_centrality(G)\n",
    "sorted(degree_centrality_G.items(), key = lambda x:x[1], reverse= True)[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Sheikh Hasina', 0.00370014149136523),\n",
       " ('Donald Trump', 0.003154837393952081),\n",
       " ('Clinton', 0.0013093035154407673),\n",
       " ('Putin', 0.0012997899963753153),\n",
       " ('Khaleda Zia', 0.0011680900178250376),\n",
       " ('Rahman', 0.0010655812015762437),\n",
       " ('Suu Kyi', 0.0010376351665081166),\n",
       " ('Khan', 0.0010131492923911812),\n",
       " ('Ali', 0.000987753025942132),\n",
       " ('Quader', 0.0008565130683169668),\n",
       " ('Barack Obama', 0.0008293584893295223),\n",
       " ('Aung San Suu Kyi', 0.0008197711494204998),\n",
       " ('Tofail Ahmed', 0.000715494818308283),\n",
       " ('Bush', 0.0007082834379158772),\n",
       " ('Kim', 0.0007014877986340839),\n",
       " ('Abdul Hamid', 0.0006913300533997925),\n",
       " ('Jamaat', 0.0006796141348578668),\n",
       " ('Md', 0.0006772518321823963),\n",
       " ('Vladimir Putin', 0.0006442993711699444),\n",
       " ('Oscar', 0.0006373263175412666)]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pr_G = nx.pagerank(G)\n",
    "sorted(pr_G.items(), key = lambda x:x[1], reverse= True)[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Sarwar Jahan', 35),\n",
       " ('Uma Khan', 35),\n",
       " ('Mesbah Uddin Ahmed', 35),\n",
       " ('Mohiuddin Ahmed', 35),\n",
       " ('Gitosree Chowdhury', 35),\n",
       " ('Shafiur Rahman Dulu', 35),\n",
       " ('Abu Bakar Siddique', 35),\n",
       " ('Anu Islam', 35),\n",
       " ('Chittaranjan Bhuiyan', 35),\n",
       " ('Jahirul Haque', 35),\n",
       " ('Milon Bhattacharjee', 35),\n",
       " ('Firoj Chowdhury', 35),\n",
       " ('Sabuj Chakrabarty', 35),\n",
       " ('Sujit Roy', 35),\n",
       " ('Niranjan Adhikari', 35),\n",
       " ('Sajeda Khatun', 35),\n",
       " ('Md Nazrul Islam', 35),\n",
       " ('Mohammed Niaz Uddin', 35),\n",
       " ('M Mamun', 35),\n",
       " ('Manjusri Niyogi', 35)]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "G.remove_edges_from(G.selfloop_edges())\n",
    "sorted(nx.core_number(G).items(), key= lambda x:x[1], reverse=True)[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_pickle('Data/New Age/NewAge_ent.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = collections.Counter([item for sublist in df['person_entities'] for item in sublist])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Bangladesh', 3010),\n",
       " ('Sheikh Hasina', 2513),\n",
       " ('Donald Trump', 1871),\n",
       " ('Dhaka', 1516),\n",
       " ('Khaleda Zia', 995),\n",
       " ('Test', 982),\n",
       " ('Barack Obama', 668),\n",
       " ('Abul Maal Abdul Muhith', 646),\n",
       " ('Obaidul Quader', 622),\n",
       " ('Rakhine', 611),\n",
       " ('Bangabandhu Sheikh Mujibur Rahman', 570),\n",
       " ('Sylhet', 561),\n",
       " ('Abdul Hamid', 552),\n",
       " ('Mirza Fakhrul Islam Alamgir', 537),\n",
       " ('Narayanganj', 509),\n",
       " ('Narendra Modi', 495),\n",
       " ('AL', 433),\n",
       " ('Gulshan', 421),\n",
       " ('Sakib al Hasan', 413),\n",
       " ('Dhanmondi', 382)]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count.most_common()[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['person_in_headline'] = np.vectorize(person_in_headline)(df['title'], df['person_entities'])\n",
    "df['person_in_text'] = np.vectorize(person_in_text)(df['news_content'], df['person_entities'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# count_person_in_text = sum(map(collections.Counter, tqdm(df['person_in_text'].values.tolist())), collections.Counter())\n",
    "# df_text = pd.DataFrame.from_dict(count_person_in_text, orient='index')\n",
    "df_text = pd.read_pickle('Data/Processed Data/Misc/NA-Text-Count.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 32893/32893 [00:31<00:00, 1031.75it/s]\n"
     ]
    }
   ],
   "source": [
    "count_person_in_headline = sum(map(collections.Counter, tqdm(df['person_in_headline'].values.tolist())), collections.Counter())\n",
    "df_headline = pd.DataFrame.from_dict(count_person_in_headline, orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "pagerank = pd.DataFrame.from_dict(pr_G, orient='index')\n",
    "pagerank = pagerank.reset_index()\n",
    "pagerank = pagerank.sort_values(by='index')\n",
    "pagerank.columns = ['name', 'pagerank']\n",
    "\n",
    "degree_centrality = pd.DataFrame.from_dict(degree_centrality_G, orient='index')\n",
    "degree_centrality = degree_centrality.reset_index()\n",
    "degree_centrality = degree_centrality.sort_values(by='index')\n",
    "degree_centrality.columns = ['name', 'degree_centrality']\n",
    "\n",
    "G.remove_edges_from(G.selfloop_edges())\n",
    "core = nx.core_number(G)\n",
    "\n",
    "core_periphery = pd.DataFrame.from_dict(core, orient='index')\n",
    "core_periphery = core_periphery.reset_index()\n",
    "core_periphery = core_periphery.sort_values(by='index')\n",
    "core_periphery.columns = ['name', 'core_periphery']\n",
    "\n",
    "count = dict(count)\n",
    "person_count = pd.DataFrame.from_dict(count, orient='index')\n",
    "person_count = person_count.reset_index()\n",
    "person_count = person_count.sort_values(by='index')\n",
    "person_count.columns = ['name', 'count']\n",
    "\n",
    "df_headline = df_headline.reset_index()\n",
    "df_headline = df_headline.sort_values(by='index')\n",
    "df_headline.columns = ['name', 'headline_count']\n",
    "\n",
    "df_text = df_text.reset_index()\n",
    "df_text = df_text.sort_values(by='index')\n",
    "df_text.columns = ['name', 'text_count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "ml = pd.DataFrame()\n",
    "ml = pd.merge(pagerank, core_periphery, on=\"name\")\n",
    "ml = pd.merge(ml, degree_centrality, on=\"name\")\n",
    "ml = pd.merge(ml, person_count, on=\"name\", how=\"left\")\n",
    "ml = pd.merge(ml, df_headline, on=\"name\", how=\"left\")\n",
    "ml = pd.merge(ml, df_text, on=\"name\", how=\"left\")\n",
    "ml.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "ml.drop(ml.index[0:59], inplace=True)\n",
    "ml.reset_index(drop=True, inplace=True)\n",
    "ml.drop(ml.index[-151:], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ml.to_pickle('Data/Processed Data/ML/NA-ml.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Making ML ready datasets in time intervals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_pickle('Data/Processed Data/Networks/DT-network.pkl')\n",
    "\n",
    "df['timestamp'] = df['timestamp'].apply(lambda x: x['$date'])\n",
    "df['timestamp'] = pd.to_datetime(df['timestamp'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6, 3, 1 month interval (replaced cells using month because notebook was getting too long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n"
     ]
    }
   ],
   "source": [
    "i = df['timestamp'].min()\n",
    "max_date_counter = 0\n",
    "while i < df['timestamp'].max() + datetime.timedelta(weeks=24):\n",
    "    max_date_counter += 1\n",
    "    i = i + datetime.timedelta(weeks=24)\n",
    "\n",
    "print(max_date_counter)\n",
    "time = []\n",
    "time.append(df['timestamp'].min())\n",
    "i = time[0]\n",
    "for i in range(1,max_date_counter):\n",
    "    time.append(time[i-1] + datetime.timedelta(weeks=24))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 247369/247369 [00:17<00:00, 14542.14it/s]\n",
      "100%|██████████| 247369/247369 [00:16<00:00, 14773.22it/s]\n",
      "100%|██████████| 247369/247369 [00:16<00:00, 14690.48it/s]\n",
      "100%|██████████| 247369/247369 [00:16<00:00, 15248.82it/s]\n",
      "100%|██████████| 247369/247369 [00:16<00:00, 15353.58it/s]\n",
      "100%|██████████| 247369/247369 [00:15<00:00, 16028.52it/s]\n",
      "100%|██████████| 247369/247369 [00:15<00:00, 15973.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24925\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "Graph_list = []\n",
    "\n",
    "network_type = 'person'\n",
    "count = 0\n",
    "for i in range(len(time)-1):\n",
    "    G = nx.Graph()\n",
    "    for _, row in tqdm(df.iterrows(), total=df.shape[0]):\n",
    "        if row['timestamp'] > time[i] and row['timestamp'] <= time[i+1]:\n",
    "            if row['type0'] in network_type and row['type1'] in network_type:\n",
    "                count += 1\n",
    "                G.add_edge(row['entity0'], row['entity1'])\n",
    "    Graph_list.append(G)\n",
    "\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "degree_centrality_list = ()\n",
    "pagerank_list = ()\n",
    "core_list = ()\n",
    "\n",
    "for G in Graph_list:\n",
    "    degree_centrality_G = nx.degree_centrality(G)\n",
    "    pr_G = nx.pagerank(G)\n",
    "    G.remove_edges_from(G.selfloop_edges())\n",
    "    core = nx.core_number(G)\n",
    "    \n",
    "    degree_centrality_list += (degree_centrality_G,)\n",
    "    pagerank_list += (pr_G,)\n",
    "    core_list += (core,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt = pd.read_pickle('Data/DT/DT.pkl')\n",
    "dt['news_publish_date'] = dt['news_publish_date'].apply(lambda x: x['$date'])\n",
    "dt['news_publish_date'] = pd.to_datetime(dt['news_publish_date'])\n",
    "dt = dt.set_index(dt['news_publish_date'])\n",
    "dt = dt.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt_list = []\n",
    "for i in range(len(time)-1):\n",
    "    temp = dt[time[i]:time[i+1]]\n",
    "    dt_list.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5379/5379 [00:11<00:00, 485.61it/s]\n",
      "100%|██████████| 5379/5379 [00:00<00:00, 11580.47it/s]\n",
      "100%|██████████| 8855/8855 [00:25<00:00, 350.45it/s]\n",
      "100%|██████████| 8855/8855 [00:00<00:00, 9158.09it/s]\n",
      "100%|██████████| 8675/8675 [00:26<00:00, 332.33it/s]\n",
      "100%|██████████| 8675/8675 [00:00<00:00, 9995.88it/s] \n",
      "100%|██████████| 8187/8187 [00:25<00:00, 317.48it/s]\n",
      "100%|██████████| 8187/8187 [00:01<00:00, 7183.28it/s]\n",
      "100%|██████████| 6137/6137 [00:14<00:00, 430.39it/s]\n",
      "100%|██████████| 6137/6137 [00:00<00:00, 11486.43it/s]\n",
      "100%|██████████| 5900/5900 [00:13<00:00, 446.12it/s]\n",
      "100%|██████████| 5900/5900 [00:00<00:00, 9134.47it/s]\n",
      "100%|██████████| 5922/5922 [00:13<00:00, 454.81it/s]\n",
      "100%|██████████| 5922/5922 [00:00<00:00, 11054.63it/s]\n"
     ]
    }
   ],
   "source": [
    "count_list = [[], [], []] # List structure is: count, count_in_text, count_in_headline\n",
    "\n",
    "for dt in dt_list:\n",
    "    dt['person_in_headline'] = np.vectorize(person_in_headline)(dt['news_headline'], dt['persons_unique'])\n",
    "    dt['person_in_text'] = np.vectorize(person_in_text)(dt['news_text'], dt['persons_unique'])\n",
    "    count_person_in_text = sum(map(collections.Counter, tqdm(dt['person_in_text'].values.tolist())), collections.Counter())\n",
    "    count_person_in_headline = sum(map(collections.Counter, tqdm(dt['person_in_headline'].values.tolist())), collections.Counter())\n",
    "    count = collections.Counter([item for sublist in dt['persons_unique'] for item in sublist])\n",
    "    \n",
    "    count_list[0].append(count)\n",
    "    count_list[1].append(count_person_in_headline)\n",
    "    count_list[2].append(count_person_in_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 7/7 [00:00<00:00, 19.77it/s]\n"
     ]
    }
   ],
   "source": [
    "for i in tqdm(range(len(dt_list))):\n",
    "    pagerank = pd.DataFrame.from_dict(pagerank_list[i], orient='index')\n",
    "    pagerank = pagerank.reset_index()\n",
    "    pagerank = pagerank.sort_values(by='index')\n",
    "    pagerank.columns = ['name', 'pagerank']\n",
    "    \n",
    "    degree_centrality = pd.DataFrame.from_dict(degree_centrality_list[i], orient='index')\n",
    "    degree_centrality = degree_centrality.reset_index()\n",
    "    degree_centrality = degree_centrality.sort_values(by='index')\n",
    "    degree_centrality.columns = ['name', 'degree_centrality']\n",
    "    \n",
    "    core_periphery = pd.DataFrame.from_dict(core_list[i], orient='index')\n",
    "    core_periphery = core_periphery.reset_index()\n",
    "    core_periphery = core_periphery.sort_values(by='index')\n",
    "    core_periphery.columns = ['name', 'core_periphery']\n",
    "    \n",
    "    count = dict(count_list[0][i])\n",
    "    person_count = pd.DataFrame.from_dict(count, orient='index')\n",
    "    person_count = person_count.reset_index()\n",
    "    person_count = person_count.sort_values(by='index')\n",
    "    person_count.columns = ['name', 'count']\n",
    "    \n",
    "    df_text = pd.DataFrame.from_dict(count_list[1][i], orient='index')\n",
    "    df_text = df_text.reset_index()\n",
    "    df_text = df_text.sort_values(by='index')\n",
    "    df_text.columns = ['name', 'text_count']\n",
    "    \n",
    "    df_headline = pd.DataFrame.from_dict(count_list[2][i], orient='index')\n",
    "    df_headline = df_headline.reset_index()\n",
    "    df_headline = df_headline.sort_values(by='index')\n",
    "    df_headline.columns = ['name', 'headline_count']\n",
    "    \n",
    "    ml = pd.DataFrame()\n",
    "    ml = pd.merge(pagerank, core_periphery, on=\"name\")\n",
    "    ml = pd.merge(ml, degree_centrality, on=\"name\")\n",
    "    ml = pd.merge(ml, person_count, on=\"name\", how=\"left\")\n",
    "    ml = pd.merge(ml, df_headline, on=\"name\", how=\"left\")\n",
    "    ml = pd.merge(ml, df_text, on=\"name\", how=\"left\")\n",
    "    \n",
    "    # ml.to_pickle('Data/Processed Data/Intervals/DT/6 month/ml_'+str(i+1)+'.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 914208/914208 [01:40<00:00, 9141.41it/s]\n"
     ]
    }
   ],
   "source": [
    "ds = pd.read_pickle('Data/Processed Data/Networks/DS-network.pkl')\n",
    "ds['timestamp'] = ds['timestamp'].progress_apply(fix_date)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6, 3, 1 month interval (replaced cells using month because notebook was getting too long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23\n"
     ]
    }
   ],
   "source": [
    "i = ds['timestamp'].min()\n",
    "max_date_counter = 0\n",
    "while i < ds['timestamp'].max() + datetime.timedelta(weeks=24):\n",
    "    max_date_counter += 1\n",
    "    i = i + datetime.timedelta(weeks=24)\n",
    "\n",
    "print(max_date_counter)\n",
    "time = []\n",
    "time.append(ds['timestamp'].min())\n",
    "i = time[0]\n",
    "for i in range(1,max_date_counter):\n",
    "    time.append(time[i-1] + datetime.timedelta(weeks=24))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = ds.set_index(ds['timestamp'])\n",
    "ds = ds.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds_list = []\n",
    "for i in range(len(time)-1):\n",
    "    temp = ds[time[i]:time[i+1]]\n",
    "    ds_list.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 22/22 [08:31<00:00, 23.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "171470\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "Graph_list = []\n",
    "\n",
    "network_type = 'person'\n",
    "count = 0\n",
    "\n",
    "for df in tqdm(ds_list):\n",
    "    G = nx.Graph()\n",
    "    for _, row in df.iterrows():\n",
    "        if row['type0'] in network_type and row['type1'] in network_type:\n",
    "            count += 1\n",
    "            G.add_edge(row['entity0'], row['entity1'])\n",
    "    Graph_list.append(G)\n",
    "\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "degree_centrality_list = ()\n",
    "pagerank_list = ()\n",
    "core_list = ()\n",
    "\n",
    "for G in Graph_list:\n",
    "    degree_centrality_G = nx.degree_centrality(G)\n",
    "    pr_G = nx.pagerank(G)\n",
    "    G.remove_edges_from(G.selfloop_edges())\n",
    "    core = nx.core_number(G)\n",
    "    \n",
    "    degree_centrality_list += (degree_centrality_G,)\n",
    "    pagerank_list += (pr_G,)\n",
    "    core_list += (core,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = pd.read_pickle('Data/DS/DS.pkl')\n",
    "ds['date_published'] = ds['date_published'].apply(lambda x: x['$date'])\n",
    "ds['date_published'] = pd.to_datetime(ds['date_published'])\n",
    "ds = ds.set_index(ds['date_published'])\n",
    "ds = ds.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds_list = []\n",
    "for i in range(len(time)-1):\n",
    "    temp = ds[time[i]:time[i+1]]\n",
    "    ds_list.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 22/22 [21:30<00:00, 58.65s/it]\n"
     ]
    }
   ],
   "source": [
    "count_list = [[], [], []] # List structure is: count, count_in_text, count_in_headline\n",
    "\n",
    "for ds in tqdm(ds_list):\n",
    "    ds['person_in_headline'] = np.vectorize(person_in_headline)(ds['title'], ds['ner_unique_person'])\n",
    "    count_person_in_text = sum(map(collections.Counter, ds['ner_person'].values.tolist()), collections.Counter())\n",
    "    count_person_in_headline = sum(map(collections.Counter, ds['person_in_headline'].values.tolist()), collections.Counter())\n",
    "    count = collections.Counter([item for sublist in ds['ner_unique_person'] for item in sublist])\n",
    "    \n",
    "    count_list[0].append(count)\n",
    "    count_list[1].append(count_person_in_headline)\n",
    "    count_list[2].append(count_person_in_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 22/22 [00:02<00:00,  9.04it/s]\n"
     ]
    }
   ],
   "source": [
    "for i in tqdm(range(len(ds_list))):\n",
    "    pagerank = pd.DataFrame.from_dict(pagerank_list[i], orient='index')\n",
    "    pagerank = pagerank.reset_index()\n",
    "    pagerank = pagerank.sort_values(by='index')\n",
    "    pagerank.columns = ['name', 'pagerank']\n",
    "    \n",
    "    degree_centrality = pd.DataFrame.from_dict(degree_centrality_list[i], orient='index')\n",
    "    degree_centrality = degree_centrality.reset_index()\n",
    "    degree_centrality = degree_centrality.sort_values(by='index')\n",
    "    degree_centrality.columns = ['name', 'degree_centrality']\n",
    "    \n",
    "    core_periphery = pd.DataFrame.from_dict(core_list[i], orient='index')\n",
    "    core_periphery = core_periphery.reset_index()\n",
    "    core_periphery = core_periphery.sort_values(by='index')\n",
    "    core_periphery.columns = ['name', 'core_periphery']\n",
    "    \n",
    "    count = dict(count_list[0][i])\n",
    "    person_count = pd.DataFrame.from_dict(count, orient='index')\n",
    "    person_count = person_count.reset_index()\n",
    "    person_count = person_count.sort_values(by='index')\n",
    "    person_count.columns = ['name', 'count']\n",
    "    \n",
    "    df_text = pd.DataFrame.from_dict(count_list[1][i], orient='index')\n",
    "    df_text = df_text.reset_index()\n",
    "    df_text = df_text.sort_values(by='index')\n",
    "    df_text.columns = ['name', 'text_count']\n",
    "    \n",
    "    df_headline = pd.DataFrame.from_dict(count_list[2][i], orient='index')\n",
    "    df_headline = df_headline.reset_index()\n",
    "    df_headline = df_headline.sort_values(by='index')\n",
    "    df_headline.columns = ['name', 'headline_count']\n",
    "    \n",
    "    ml = pd.DataFrame()\n",
    "    ml = pd.merge(pagerank, core_periphery, on=\"name\")\n",
    "    ml = pd.merge(ml, degree_centrality, on=\"name\")\n",
    "    ml = pd.merge(ml, person_count, on=\"name\", how=\"left\")\n",
    "    ml = pd.merge(ml, df_headline, on=\"name\", how=\"left\")\n",
    "    ml = pd.merge(ml, df_text, on=\"name\", how=\"left\")\n",
    "    \n",
    "    # ml.to_pickle('Data/Processed Data/Intervals/DS/6 month/ml_'+str(i+1)+'.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "na = pd.read_pickle('Data/Processed Data/Networks/NA-network.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "na['timestamp'] = pd.to_datetime(na['timestamp'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6, 3, 1 month interval (replaced cells using month because notebook was getting too long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n"
     ]
    }
   ],
   "source": [
    "i = na['timestamp'].min()\n",
    "max_date_counter = 0\n",
    "while i < na['timestamp'].max() + datetime.timedelta(weeks=24):\n",
    "    max_date_counter += 1\n",
    "    i = i + datetime.timedelta(weeks=24)\n",
    "\n",
    "print(max_date_counter)\n",
    "time = []\n",
    "time.append(na['timestamp'].min())\n",
    "i = time[0]\n",
    "for i in range(1,max_date_counter):\n",
    "    time.append(time[i-1] + datetime.timedelta(weeks=24))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "na = na.set_index(na['timestamp'])\n",
    "na = na.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "na_list = []\n",
    "for i in range(len(time)-1):\n",
    "    temp = na[time[i]:time[i+1]]\n",
    "    na_list.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4/4 [00:51<00:00, 12.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "86532\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "Graph_list = []\n",
    "\n",
    "network_type = 'person'\n",
    "count = 0\n",
    "\n",
    "for df in tqdm(na_list):\n",
    "    G = nx.Graph()\n",
    "    for _, row in df.iterrows():\n",
    "        if row['type0'] in network_type and row['type1'] in network_type:\n",
    "            count += 1\n",
    "            G.add_edge(row['entity0'], row['entity1'])\n",
    "    Graph_list.append(G)\n",
    "\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "degree_centrality_list = ()\n",
    "pagerank_list = ()\n",
    "core_list = ()\n",
    "\n",
    "for G in Graph_list:\n",
    "    degree_centrality_G = nx.degree_centrality(G)\n",
    "    pr_G = nx.pagerank(G)\n",
    "    G.remove_edges_from(G.selfloop_edges())\n",
    "    core = nx.core_number(G)\n",
    "    \n",
    "    degree_centrality_list += (degree_centrality_G,)\n",
    "    pagerank_list += (pr_G,)\n",
    "    core_list += (core,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "na = pd.read_pickle('Data/New Age/NewAge_ent.pkl')\n",
    "na['date_published'] = pd.to_datetime(na['date_published'])\n",
    "na = na.set_index(na['date_published'])\n",
    "na = na.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "na_list = []\n",
    "for i in range(len(time)-1):\n",
    "    temp = na[time[i]:time[i+1]]\n",
    "    na_list.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4/4 [05:34<00:00, 83.72s/it] \n"
     ]
    }
   ],
   "source": [
    "count_list = [[], [], []] # List structure is: count, count_in_text, count_in_headline\n",
    "\n",
    "for na in tqdm(na_list):\n",
    "    na['person_in_headline'] = np.vectorize(person_in_headline)(na['title'], na['person_entities'])\n",
    "    na['person_in_text'] = np.vectorize(person_in_text)(na['news_content'], na['person_entities'])\n",
    "    count_person_in_text = sum(map(collections.Counter, na['person_in_text'].values.tolist()), collections.Counter())\n",
    "    count_person_in_headline = sum(map(collections.Counter, na['person_in_headline'].values.tolist()), collections.Counter())\n",
    "    count = collections.Counter([item for sublist in na['person_entities'] for item in sublist])\n",
    "    \n",
    "    count_list[0].append(count)\n",
    "    count_list[1].append(count_person_in_headline)\n",
    "    count_list[2].append(count_person_in_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4/4 [00:00<00:00,  6.79it/s]\n"
     ]
    }
   ],
   "source": [
    "for i in tqdm(range(len(na_list))):\n",
    "    pagerank = pd.DataFrame.from_dict(pagerank_list[i], orient='index')\n",
    "    pagerank = pagerank.reset_index()\n",
    "    pagerank = pagerank.sort_values(by='index')\n",
    "    pagerank.columns = ['name', 'pagerank']\n",
    "    \n",
    "    degree_centrality = pd.DataFrame.from_dict(degree_centrality_list[i], orient='index')\n",
    "    degree_centrality = degree_centrality.reset_index()\n",
    "    degree_centrality = degree_centrality.sort_values(by='index')\n",
    "    degree_centrality.columns = ['name', 'degree_centrality']\n",
    "    \n",
    "    core_periphery = pd.DataFrame.from_dict(core_list[i], orient='index')\n",
    "    core_periphery = core_periphery.reset_index()\n",
    "    core_periphery = core_periphery.sort_values(by='index')\n",
    "    core_periphery.columns = ['name', 'core_periphery']\n",
    "    \n",
    "    count = dict(count_list[0][i])\n",
    "    person_count = pd.DataFrame.from_dict(count, orient='index')\n",
    "    person_count = person_count.reset_index()\n",
    "    person_count = person_count.sort_values(by='index')\n",
    "    person_count.columns = ['name', 'count']\n",
    "    \n",
    "    df_text = pd.DataFrame.from_dict(count_list[1][i], orient='index')\n",
    "    df_text = df_text.reset_index()\n",
    "    df_text = df_text.sort_values(by='index')\n",
    "    df_text.columns = ['name', 'text_count']\n",
    "    \n",
    "    df_headline = pd.DataFrame.from_dict(count_list[2][i], orient='index')\n",
    "    df_headline = df_headline.reset_index()\n",
    "    df_headline = df_headline.sort_values(by='index')\n",
    "    df_headline.columns = ['name', 'headline_count']\n",
    "    \n",
    "    ml = pd.DataFrame()\n",
    "    ml = pd.merge(pagerank, core_periphery, on=\"name\")\n",
    "    ml = pd.merge(ml, degree_centrality, on=\"name\")\n",
    "    ml = pd.merge(ml, person_count, on=\"name\", how=\"left\")\n",
    "    ml = pd.merge(ml, df_headline, on=\"name\", how=\"left\")\n",
    "    ml = pd.merge(ml, df_text, on=\"name\", how=\"left\")\n",
    "    \n",
    "    # ml.to_pickle('Data/Processed Data/Intervals/NA/6 month/ml_'+str(i+1)+'.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}