merge_subfiles_by_publisher.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merge all the individual files inside of each publisher into one text file\n",
    "\n",
    "####  So output file: in each line, seperated by tab has id, year, title, publisher, article content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import codecs\n",
    "import pathlib\n",
    "import glob\n",
    "import datetime\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "n_jobs = -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_dir = os.path.join(os.getcwd(),'data','sampled_raw') # directory where you have initial subsampled data\n",
    "output_dir = os.path.join(os.getcwd(),'data','sampled_raw_publisher') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "country of AU time:  0:09:38.779096\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-9-8cbacc7d24fc>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     22\u001b[0m                     \u001b[0minput_f_path\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0myear_path\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mfile\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 24\u001b[1;33m                     \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_f_path\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"r\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"ISO-8859-1\"\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mf_in\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     25\u001b[0m                         \u001b[0mindex\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;31m#i.e. 0: id, 1:title, 2:publisher, 3:  url , 5: news article content\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     26\u001b[0m                         \u001b[0mtext_list\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf_in\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplitlines\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\codecs.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, errors)\u001b[0m\n\u001b[0;32m    258\u001b[0m     \u001b[0mremembers\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mstate\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mdecoding\u001b[0m \u001b[0mprocess\u001b[0m \u001b[0mbetween\u001b[0m \u001b[0mcalls\u001b[0m \u001b[0mto\u001b[0m \u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    259\u001b[0m     \"\"\"\n\u001b[1;32m--> 260\u001b[1;33m     \u001b[1;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'strict'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    261\u001b[0m         \"\"\"\n\u001b[0;32m    262\u001b[0m         \u001b[0mCreate\u001b[0m \u001b[0man\u001b[0m \u001b[0mIncrementalDecoder\u001b[0m \u001b[0minstance\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "time_start = datetime.datetime.now()\n",
    "for country in os.listdir(base_dir):\n",
    "    t_country_start = datetime.datetime.now()\n",
    "\n",
    "    country_dir = os.path.join(base_dir,country)\n",
    "    os.mkdir(country_dir.replace('sampled_raw','sampled_raw_publisher'))\n",
    "\n",
    "    for publisher in os.listdir(country_dir):\n",
    "#         t_publisher_start = datetime.datetime.now()\n",
    "        publisher_path = os.path.join(country_dir,publisher)\n",
    "\n",
    "        output_file_path = publisher_path.replace('sampled_raw','sampled_raw_publisher')\n",
    "#         print(publisher_path)\n",
    "#         print(output_file_path)\n",
    "\n",
    "        with open(output_file_path +'.txt', \"w\", encoding = \"utf8\") as f_out:\n",
    "\n",
    "            for year in os.listdir(publisher_path):\n",
    "                year_path = os.path.join(publisher_path,year)\n",
    "\n",
    "                for file in os.listdir(year_path):\n",
    "                    input_f_path = os.path.join(year_path,file)\n",
    "\n",
    "                    with open(input_f_path, \"r\", encoding=\"ISO-8859-1\") as f_in:\n",
    "                        index = [0,1,3,5] #i.e. 0: id, 1:title, 2:publisher, 3:  url , 5: news article content\n",
    "                        text_list = f_in.read().splitlines() \n",
    "                        info_list = [text_list[i] for i in index]\n",
    "                        info_list.append(year)\n",
    "\n",
    "                        f_out.write('\\t'.join([str(x) for x in info_list])+'\\n')\n",
    "                    f_in.close()\n",
    "                    \n",
    "        f_out.close()\n",
    "        \n",
    "#         t_publisher_end = datetime.datetime.now()\n",
    "#         print(\"Publisher of\", publihser, \"time: \", t_publisher_end-t_publisher_start)\n",
    "    \n",
    "    t_country_end = datetime.datetime.now()\n",
    "    print(\"country of\", country, \"time: \", t_country_end-t_country_start)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}