Create Sampled Data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import shutil\n",
    "import random\n",
    "import pathlib\n",
    "from datetime import datetime\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "from tqdm import tqdm\n",
    "import slugify\n",
    "\n",
    "import utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>country</th>\n",
       "      <th>publisher</th>\n",
       "      <th>num_articles</th>\n",
       "      <th>total_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>AU</td>\n",
       "      <td>3AW</td>\n",
       "      <td>1</td>\n",
       "      <td>1505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>AU</td>\n",
       "      <td>ABC Message Stick</td>\n",
       "      <td>2</td>\n",
       "      <td>7284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>AU</td>\n",
       "      <td>ABC Online</td>\n",
       "      <td>143</td>\n",
       "      <td>78050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>AU</td>\n",
       "      <td>ABC Regional Online</td>\n",
       "      <td>55</td>\n",
       "      <td>21602</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>AU</td>\n",
       "      <td>Architecture and Design</td>\n",
       "      <td>14</td>\n",
       "      <td>4981</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   year  month country                publisher  num_articles  total_words\n",
       "0  2010      1      AU                      3AW             1         1505\n",
       "1  2010      1      AU        ABC Message Stick             2         7284\n",
       "2  2010      1      AU               ABC Online           143        78050\n",
       "3  2010      1      AU      ABC Regional Online            55        21602\n",
       "4  2010      1      AU  Architecture and Design            14         4981"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary_stats_df = pd.read_csv(\"data/summary_stats.csv\")\n",
    "\n",
    "summary_stats_df[\"country\"] = summary_stats_df[\"country\"].apply(lambda x: \"GB\" if x == \"UK\" else x)\n",
    "summary_stats_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country</th>\n",
       "      <th>publisher</th>\n",
       "      <th>publisher_tot_articles</th>\n",
       "      <th>publisher_tot_words</th>\n",
       "      <th>is_big_publisher</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AU</td>\n",
       "      <td>'Bourne this way</td>\n",
       "      <td>48</td>\n",
       "      <td>14834</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AU</td>\n",
       "      <td>10 News First</td>\n",
       "      <td>37</td>\n",
       "      <td>6240</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AU</td>\n",
       "      <td>10 daily</td>\n",
       "      <td>221</td>\n",
       "      <td>103872</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AU</td>\n",
       "      <td>123Jump.com</td>\n",
       "      <td>3</td>\n",
       "      <td>778</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AU</td>\n",
       "      <td>2DayFM</td>\n",
       "      <td>15</td>\n",
       "      <td>3771</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  country         publisher  publisher_tot_articles  publisher_tot_words  \\\n",
       "0      AU  'Bourne this way                      48                14834   \n",
       "1      AU     10 News First                      37                 6240   \n",
       "2      AU          10 daily                     221               103872   \n",
       "3      AU       123Jump.com                       3                  778   \n",
       "4      AU            2DayFM                      15                 3771   \n",
       "\n",
       "   is_big_publisher  \n",
       "0             False  \n",
       "1             False  \n",
       "2             False  \n",
       "3             False  \n",
       "4             False  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "by_publisher = summary_stats_df.groupby([\"country\", \"publisher\"])\n",
    "by_publisher = by_publisher[[\"num_articles\", \"total_words\"]]\n",
    "by_publisher = by_publisher.sum().reset_index().rename(columns={\n",
    "    \"num_articles\": \"publisher_tot_articles\", \n",
    "    \"total_words\": \"publisher_tot_words\"\n",
    "})\n",
    "\n",
    "by_publisher[\"is_big_publisher\"] = by_publisher.publisher_tot_articles.apply(lambda x: x > 1000)\n",
    "by_publisher.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>country</th>\n",
       "      <th>publisher</th>\n",
       "      <th>num_articles</th>\n",
       "      <th>total_words</th>\n",
       "      <th>publisher_tot_articles</th>\n",
       "      <th>publisher_tot_words</th>\n",
       "      <th>is_big_publisher</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2016</td>\n",
       "      <td>11</td>\n",
       "      <td>AU</td>\n",
       "      <td>9Honey</td>\n",
       "      <td>14</td>\n",
       "      <td>5941</td>\n",
       "      <td>3798</td>\n",
       "      <td>1839552</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2016</td>\n",
       "      <td>12</td>\n",
       "      <td>AU</td>\n",
       "      <td>9Honey</td>\n",
       "      <td>52</td>\n",
       "      <td>18680</td>\n",
       "      <td>3798</td>\n",
       "      <td>1839552</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016</td>\n",
       "      <td>11</td>\n",
       "      <td>AU</td>\n",
       "      <td>9Honey</td>\n",
       "      <td>14</td>\n",
       "      <td>5941</td>\n",
       "      <td>3798</td>\n",
       "      <td>1839552</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2016</td>\n",
       "      <td>12</td>\n",
       "      <td>AU</td>\n",
       "      <td>9Honey</td>\n",
       "      <td>56</td>\n",
       "      <td>20555</td>\n",
       "      <td>3798</td>\n",
       "      <td>1839552</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2017</td>\n",
       "      <td>1</td>\n",
       "      <td>AU</td>\n",
       "      <td>9Honey</td>\n",
       "      <td>75</td>\n",
       "      <td>31747</td>\n",
       "      <td>3798</td>\n",
       "      <td>1839552</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   year  month country publisher  num_articles  total_words  \\\n",
       "0  2016     11      AU    9Honey            14         5941   \n",
       "1  2016     12      AU    9Honey            52        18680   \n",
       "2  2016     11      AU    9Honey            14         5941   \n",
       "3  2016     12      AU    9Honey            56        20555   \n",
       "4  2017      1      AU    9Honey            75        31747   \n",
       "\n",
       "   publisher_tot_articles  publisher_tot_words  is_big_publisher  \n",
       "0                    3798              1839552              True  \n",
       "1                    3798              1839552              True  \n",
       "2                    3798              1839552              True  \n",
       "3                    3798              1839552              True  \n",
       "4                    3798              1839552              True  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "big_publishers = summary_stats_df.merge(\n",
    "    by_publisher.query(\"is_big_publisher\"), \n",
    "    how=\"right\", on=[\"country\", \"publisher\"]\n",
    ")\n",
    "\n",
    "big_publishers.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country</th>\n",
       "      <th>year</th>\n",
       "      <th>publisher</th>\n",
       "      <th>total_articles</th>\n",
       "      <th>sampled_articles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AU</td>\n",
       "      <td>2010</td>\n",
       "      <td>ABC Local</td>\n",
       "      <td>338</td>\n",
       "      <td>248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AU</td>\n",
       "      <td>2010</td>\n",
       "      <td>ABC Online</td>\n",
       "      <td>2743</td>\n",
       "      <td>2010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AU</td>\n",
       "      <td>2010</td>\n",
       "      <td>AdNews</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AU</td>\n",
       "      <td>2010</td>\n",
       "      <td>Architecture and Design</td>\n",
       "      <td>147</td>\n",
       "      <td>108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AU</td>\n",
       "      <td>2010</td>\n",
       "      <td>Australian Personal Computer</td>\n",
       "      <td>263</td>\n",
       "      <td>193</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  country  year                     publisher  total_articles  \\\n",
       "0      AU  2010                     ABC Local             338   \n",
       "1      AU  2010                    ABC Online            2743   \n",
       "2      AU  2010                        AdNews               7   \n",
       "3      AU  2010       Architecture and Design             147   \n",
       "4      AU  2010  Australian Personal Computer             263   \n",
       "\n",
       "   sampled_articles  \n",
       "0               248  \n",
       "1              2010  \n",
       "2                 5  \n",
       "3               108  \n",
       "4               193  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CAP = 10000\n",
    "\n",
    "def get_num_samples(x, num_samples):\n",
    "    pct, max_ = x[\"pct\"], x[\"total_articles\"]\n",
    "    n = int(np.around(num_samples * pct))\n",
    "    if n > max_:\n",
    "        n = max_\n",
    "        \n",
    "    return n if n > 0 else 1\n",
    "\n",
    "sampled_counts = []\n",
    "for country, country_df in big_publishers.groupby(\"country\"):\n",
    "    \n",
    "    # Get rid of month\n",
    "    # For each contry group by years. Articles per year. \n",
    "    articles_by_year = country_df.groupby(\"year\")[\"num_articles\"].sum()\n",
    "    \n",
    "    # Median from the number of articles that are of the year 2015 and below. \n",
    "    num_to_sample = articles_by_year[articles_by_year.index < 2016].median()\n",
    "    num_to_sample = np.min([CAP, num_to_sample])\n",
    "    \n",
    "    # Going through each year at a time per country. \n",
    "    for year, year_df in country_df.groupby(\"year\"):\n",
    "        publishers = year_df.groupby(\"publisher\")[\"num_articles\"].sum().reset_index()\n",
    "        publishers[\"country\"] = country\n",
    "        publishers[\"year\"] = year\n",
    "        \n",
    "        publishers[\"pct\"] = publishers.num_articles / publishers.num_articles.sum()\n",
    "        publishers[\"total_articles\"] = publishers.num_articles\n",
    "        \n",
    "        publishers[\"sampled_articles\"] = publishers.apply(\n",
    "            get_num_samples, axis=1, num_samples=num_to_sample\n",
    "        )\n",
    "    \n",
    "        sampled_counts.append(publishers[[\n",
    "            \"country\", \"year\", \"publisher\", \"total_articles\", \"sampled_articles\"\n",
    "         ]].copy())\n",
    "        \n",
    "big_publisher_sampling = pd.concat(sampled_counts)\n",
    "big_publisher_sampling.to_csv(\"misc/sampling_amounts.csv\", index=False)\n",
    "big_publisher_sampling.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_articles      17,336,805\n",
      "sampled_articles     1,465,631\n",
      "dtype: object\n",
      "\n",
      "By Year:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>total_articles</th>\n",
       "      <th>sampled_articles</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>year</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2010</th>\n",
       "      <td>258826</td>\n",
       "      <td>84177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2011</th>\n",
       "      <td>347212</td>\n",
       "      <td>109529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2012</th>\n",
       "      <td>448073</td>\n",
       "      <td>133451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013</th>\n",
       "      <td>510685</td>\n",
       "      <td>141856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014</th>\n",
       "      <td>576154</td>\n",
       "      <td>142436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015</th>\n",
       "      <td>734857</td>\n",
       "      <td>142400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016</th>\n",
       "      <td>3006000</td>\n",
       "      <td>142405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2017</th>\n",
       "      <td>3064616</td>\n",
       "      <td>142362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018</th>\n",
       "      <td>2575893</td>\n",
       "      <td>142433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019</th>\n",
       "      <td>2956593</td>\n",
       "      <td>142388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020</th>\n",
       "      <td>2857896</td>\n",
       "      <td>142194</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      total_articles  sampled_articles\n",
       "year                                  \n",
       "2010          258826             84177\n",
       "2011          347212            109529\n",
       "2012          448073            133451\n",
       "2013          510685            141856\n",
       "2014          576154            142436\n",
       "2015          734857            142400\n",
       "2016         3006000            142405\n",
       "2017         3064616            142362\n",
       "2018         2575893            142433\n",
       "2019         2956593            142388\n",
       "2020         2857896            142194"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(big_publisher_sampling[[\"total_articles\", \"sampled_articles\"]].sum().apply(lambda x: f\"{x:,d}\"))\n",
    "\n",
    "print(\"\\nBy Year:\")\n",
    "big_publisher_sampling.groupby(\"year\")[[\"total_articles\", \"sampled_articles\"]].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "CLEAN_PATH = r\"D:\\Data\\peace-speech-project\\clean_full\"\n",
    "\n",
    "def article_is_empty(text_file):\n",
    "    return os.stat(text_file).st_size == 0\n",
    "\n",
    "def simplify_path(text_file):\n",
    "    return os.path.join(*pathlib.Path(text_file).parts[-4:])\n",
    "\n",
    "def split_valid_and_empty_files(text_files):\n",
    "    valid, empty = [], []\n",
    "    for text_file in text_files:\n",
    "        (valid, empty)[article_is_empty(text_file)].append(simplify_path(text_file))\n",
    "        \n",
    "    return valid, empty\n",
    "\n",
    "def get_all_file_paths(country: str, publisher: str, year: int, slug=True):\n",
    "    if slug:\n",
    "        publisher = slugify.slugify(publisher)\n",
    "        \n",
    "    folder_path = os.path.join(CLEAN_PATH, country, publisher, str(year))\n",
    "    if not os.path.exists(folder_path):\n",
    "        print(folder_path)\n",
    "        return None\n",
    "    \n",
    "    file_path = os.path.join(folder_path, \"*.txt\")\n",
    "    text_files = glob.glob(file_path)\n",
    "    return split_valid_and_empty_files(text_files)\n",
    "\n",
    "def sample_files(text_files, num_articles):\n",
    "    if num_articles > len(text_files):\n",
    "        return text_files\n",
    "    \n",
    "    return np.random.choice(text_files, size=num_articles, replace=False)\n",
    "\n",
    "def get_articles_to_sample(to_sample):\n",
    "    valid_files, empty_files = get_all_file_paths(to_sample.country, to_sample.publisher, to_sample.year, slug=True)\n",
    "    \n",
    "    if not valid_files:\n",
    "        sampled_text_file_paths = []\n",
    "    else:\n",
    "        sampled_text_file_paths = sample_files(valid_files, to_sample.sampled_articles)\n",
    "    \n",
    "    return (to_sample.country, to_sample.publisher, to_sample.year, len(valid_files), len(empty_files), sampled_text_file_paths)\n",
    "\n",
    "def copy_sampled_file(file_path):\n",
    "    from_path = pathlib.Path(os.path.join(CLEAN_PATH, file_path))\n",
    "    to_path = pathlib.Path(os.path.join(utils.CLEAN_DATA_FOLDER, file_path))\n",
    "\n",
    "    if not to_path.parent.exists():\n",
    "        to_path.parent.mkdir(parents=True)\n",
    "\n",
    "    shutil.copyfile(from_path, to_path)\n",
    "\n",
    "def get_id_from_path(file_path):\n",
    "    if not pd.isna(file_path):\n",
    "        file_name = pathlib.Path(file_path).parts[-1]\n",
    "        return int(file_name.split(\"_\")[0])\n",
    "\n",
    "    return np.nan\n",
    "\n",
    "def get_month_from_path(file_path):\n",
    "    if not pd.isna(file_path):\n",
    "        file_name = pathlib.Path(file_path).parts[-1]\n",
    "        date_str = file_name.split(\"_\")[-1][:-4]\n",
    "        return datetime.strptime(date_str, \"%d-%m-%y\").month\n",
    "\n",
    "    return np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\mattb\\Miniconda3\\envs\\peace\\lib\\site-packages\\tqdm\\std.py:697: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
      "  from pandas import Panel\n",
      "100%|████████████████████████████████████████████████████████████████████████████| 14529/14529 [51:35<00:00,  4.69it/s]\n"
     ]
    }
   ],
   "source": [
    "bp_sample = big_publisher_sampling.copy() #big_publisher_sampling[big_publisher_sampling.country == \"AU\"].copy()\n",
    "\n",
    "tqdm.pandas()\n",
    "to_sample_df = pd.DataFrame(\n",
    "    bp_sample.progress_apply(get_articles_to_sample, axis=1).to_list(), \n",
    "    columns=[\"country\", \"publisher\", \"year\", \"valid_articles_found\", \"empty_articles_found\", \"to_sample_file_path\"]\n",
    ").explode(\"to_sample_file_path\")\n",
    "\n",
    "to_sample_df[\"id\"] = to_sample_df.to_sample_file_path.apply(get_id_from_path)\n",
    "to_sample_df[\"month\"] = to_sample_df.to_sample_file_path.apply(get_month_from_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country</th>\n",
       "      <th>year</th>\n",
       "      <th>publisher</th>\n",
       "      <th>total_articles</th>\n",
       "      <th>num_articles_to_sample</th>\n",
       "      <th>valid_articles_found</th>\n",
       "      <th>empty_articles_found</th>\n",
       "      <th>to_sample_file_path</th>\n",
       "      <th>id</th>\n",
       "      <th>month</th>\n",
       "      <th>total_articles_found</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AU</td>\n",
       "      <td>2010</td>\n",
       "      <td>ABC Local</td>\n",
       "      <td>338</td>\n",
       "      <td>248</td>\n",
       "      <td>338</td>\n",
       "      <td>0</td>\n",
       "      <td>AU\\abc-local\\2010\\1706268_AU_09-11-10.txt</td>\n",
       "      <td>1706268.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AU</td>\n",
       "      <td>2010</td>\n",
       "      <td>ABC Local</td>\n",
       "      <td>338</td>\n",
       "      <td>248</td>\n",
       "      <td>338</td>\n",
       "      <td>0</td>\n",
       "      <td>AU\\abc-local\\2010\\1520721_AU_14-06-10.txt</td>\n",
       "      <td>1520721.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AU</td>\n",
       "      <td>2010</td>\n",
       "      <td>ABC Local</td>\n",
       "      <td>338</td>\n",
       "      <td>248</td>\n",
       "      <td>338</td>\n",
       "      <td>0</td>\n",
       "      <td>AU\\abc-local\\2010\\1492348_AU_20-05-10.txt</td>\n",
       "      <td>1492348.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AU</td>\n",
       "      <td>2010</td>\n",
       "      <td>ABC Local</td>\n",
       "      <td>338</td>\n",
       "      <td>248</td>\n",
       "      <td>338</td>\n",
       "      <td>0</td>\n",
       "      <td>AU\\abc-local\\2010\\1617875_AU_01-09-10.txt</td>\n",
       "      <td>1617875.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AU</td>\n",
       "      <td>2010</td>\n",
       "      <td>ABC Local</td>\n",
       "      <td>338</td>\n",
       "      <td>248</td>\n",
       "      <td>338</td>\n",
       "      <td>0</td>\n",
       "      <td>AU\\abc-local\\2010\\1641111_AU_20-09-10.txt</td>\n",
       "      <td>1641111.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>338</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  country  year  publisher  total_articles  num_articles_to_sample  \\\n",
       "0      AU  2010  ABC Local             338                     248   \n",
       "1      AU  2010  ABC Local             338                     248   \n",
       "2      AU  2010  ABC Local             338                     248   \n",
       "3      AU  2010  ABC Local             338                     248   \n",
       "4      AU  2010  ABC Local             338                     248   \n",
       "\n",
       "   valid_articles_found  empty_articles_found  \\\n",
       "0                   338                     0   \n",
       "1                   338                     0   \n",
       "2                   338                     0   \n",
       "3                   338                     0   \n",
       "4                   338                     0   \n",
       "\n",
       "                         to_sample_file_path         id  month  \\\n",
       "0  AU\\abc-local\\2010\\1706268_AU_09-11-10.txt  1706268.0   11.0   \n",
       "1  AU\\abc-local\\2010\\1520721_AU_14-06-10.txt  1520721.0    6.0   \n",
       "2  AU\\abc-local\\2010\\1492348_AU_20-05-10.txt  1492348.0    5.0   \n",
       "3  AU\\abc-local\\2010\\1617875_AU_01-09-10.txt  1617875.0    9.0   \n",
       "4  AU\\abc-local\\2010\\1641111_AU_20-09-10.txt  1641111.0    9.0   \n",
       "\n",
       "   total_articles_found  \n",
       "0                   338  \n",
       "1                   338  \n",
       "2                   338  \n",
       "3                   338  \n",
       "4                   338  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bp_sample.rename(columns={\"sampled_articles\": \"num_articles_to_sample\"}, inplace=True)\n",
    "\n",
    "sampled_articles = bp_sample.merge(\n",
    "    to_sample_df, \n",
    "    on=[\"country\", \"publisher\", \"year\"], \n",
    "    how=\"left\"\n",
    ")\n",
    "\n",
    "sampled_articles[\"total_articles_found\"] = sampled_articles.valid_articles_found + sampled_articles.empty_articles_found\n",
    "\n",
    "sampled_articles.to_csv(\"data/sampled_articles.csv\", index=False)\n",
    "sampled_articles.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country</th>\n",
       "      <th>publisher</th>\n",
       "      <th>year</th>\n",
       "      <th>id</th>\n",
       "      <th>num_sampled</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [country, publisher, year, id, num_sampled]\n",
       "Index: []"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "repeat_ids_check = sampled_articles.groupby([\"country\", \"publisher\", \"year\", \"id\"]).size().rename(\"num_sampled\").reset_index()\n",
    "repeat_ids_check[repeat_ids_check.num_sampled > 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "key = [\"country\", \"publisher\", \"year\"]\n",
    "num_articles_sampled = sampled_articles.groupby(key).to_sample_file_path.count().to_frame().reset_index()\n",
    "num_articles_sampled.rename(columns={\"to_sample_file_path\": \"num_articles_sampled\"}, inplace=True)\n",
    "\n",
    "sanity_check_df = sampled_articles.merge(num_articles_sampled, on=key, how=\"left\")\n",
    "sanity_check_df[\"articles_in_folder_check\"] = sanity_check_df.total_articles == sanity_check_df.total_articles_found\n",
    "sanity_check_df[\"num_articles_sampled_check\"] = sanity_check_df.num_articles_to_sample == sanity_check_df.num_articles_sampled\n",
    "sanity_check_df[\"sanity_check\"] = sanity_check_df.articles_in_folder_check & sanity_check_df.articles_in_folder_check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "num_articles_to_sample    1465631\n",
       "num_articles_sampled      1465240\n",
       "dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_articles_check = sanity_check_df.groupby(key + [\"num_articles_to_sample\"])\n",
    "total_articles_check = total_articles_check.size().rename(\"num_articles_sampled\").reset_index()\n",
    "total_articles_check[[\"num_articles_to_sample\", \"num_articles_sampled\"]].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country</th>\n",
       "      <th>year</th>\n",
       "      <th>publisher</th>\n",
       "      <th>total_articles</th>\n",
       "      <th>num_articles_to_sample</th>\n",
       "      <th>valid_articles_found</th>\n",
       "      <th>empty_articles_found</th>\n",
       "      <th>to_sample_file_path</th>\n",
       "      <th>id</th>\n",
       "      <th>month</th>\n",
       "      <th>total_articles_found</th>\n",
       "      <th>num_articles_sampled</th>\n",
       "      <th>articles_in_folder_check</th>\n",
       "      <th>num_articles_sampled_check</th>\n",
       "      <th>sanity_check</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>59999</th>\n",
       "      <td>AU</td>\n",
       "      <td>2016</td>\n",
       "      <td>9Honey</td>\n",
       "      <td>136</td>\n",
       "      <td>6</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>AU\\9honey\\2016\\15895978_AU_13-12-16.txt</td>\n",
       "      <td>15895978.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>70</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60000</th>\n",
       "      <td>AU</td>\n",
       "      <td>2016</td>\n",
       "      <td>9Honey</td>\n",
       "      <td>136</td>\n",
       "      <td>6</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>AU\\9honey\\2016\\15855997_AU_11-12-16.txt</td>\n",
       "      <td>15855997.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>70</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60001</th>\n",
       "      <td>AU</td>\n",
       "      <td>2016</td>\n",
       "      <td>9Honey</td>\n",
       "      <td>136</td>\n",
       "      <td>6</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>AU\\9honey\\2016\\15849634_AU_11-12-16.txt</td>\n",
       "      <td>15849634.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>70</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60002</th>\n",
       "      <td>AU</td>\n",
       "      <td>2016</td>\n",
       "      <td>9Honey</td>\n",
       "      <td>136</td>\n",
       "      <td>6</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>AU\\9honey\\2016\\16008320_AU_20-12-16.txt</td>\n",
       "      <td>16008320.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>70</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60003</th>\n",
       "      <td>AU</td>\n",
       "      <td>2016</td>\n",
       "      <td>9Honey</td>\n",
       "      <td>136</td>\n",
       "      <td>6</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>AU\\9honey\\2016\\15732244_AU_04-12-16.txt</td>\n",
       "      <td>15732244.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>70</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1454255</th>\n",
       "      <td>ZA</td>\n",
       "      <td>2019</td>\n",
       "      <td>moneyweb.co.za</td>\n",
       "      <td>671</td>\n",
       "      <td>37</td>\n",
       "      <td>2095</td>\n",
       "      <td>304</td>\n",
       "      <td>ZA\\moneyweb-co-za\\2019\\30726246_ZA_01-04-19.txt</td>\n",
       "      <td>30726246.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2399</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1454256</th>\n",
       "      <td>ZA</td>\n",
       "      <td>2019</td>\n",
       "      <td>moneyweb.co.za</td>\n",
       "      <td>671</td>\n",
       "      <td>37</td>\n",
       "      <td>2095</td>\n",
       "      <td>304</td>\n",
       "      <td>ZA\\moneyweb-co-za\\2019\\30456195_ZA_15-01-19.txt</td>\n",
       "      <td>30456195.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2399</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1454257</th>\n",
       "      <td>ZA</td>\n",
       "      <td>2019</td>\n",
       "      <td>moneyweb.co.za</td>\n",
       "      <td>671</td>\n",
       "      <td>37</td>\n",
       "      <td>2095</td>\n",
       "      <td>304</td>\n",
       "      <td>ZA\\moneyweb-co-za\\2019\\30938411_ZA_03-06-19.txt</td>\n",
       "      <td>30938411.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2399</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1454258</th>\n",
       "      <td>ZA</td>\n",
       "      <td>2019</td>\n",
       "      <td>moneyweb.co.za</td>\n",
       "      <td>671</td>\n",
       "      <td>37</td>\n",
       "      <td>2095</td>\n",
       "      <td>304</td>\n",
       "      <td>ZA\\moneyweb-co-za\\2019\\30961580_ZA_10-06-19.txt</td>\n",
       "      <td>30961580.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2399</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1454259</th>\n",
       "      <td>ZA</td>\n",
       "      <td>2019</td>\n",
       "      <td>moneyweb.co.za</td>\n",
       "      <td>671</td>\n",
       "      <td>37</td>\n",
       "      <td>2095</td>\n",
       "      <td>304</td>\n",
       "      <td>ZA\\moneyweb-co-za\\2019\\30503858_ZA_28-01-19.txt</td>\n",
       "      <td>30503858.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2399</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>297092 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        country  year       publisher  total_articles  num_articles_to_sample  \\\n",
       "59999        AU  2016          9Honey             136                       6   \n",
       "60000        AU  2016          9Honey             136                       6   \n",
       "60001        AU  2016          9Honey             136                       6   \n",
       "60002        AU  2016          9Honey             136                       6   \n",
       "60003        AU  2016          9Honey             136                       6   \n",
       "...         ...   ...             ...             ...                     ...   \n",
       "1454255      ZA  2019  moneyweb.co.za             671                      37   \n",
       "1454256      ZA  2019  moneyweb.co.za             671                      37   \n",
       "1454257      ZA  2019  moneyweb.co.za             671                      37   \n",
       "1454258      ZA  2019  moneyweb.co.za             671                      37   \n",
       "1454259      ZA  2019  moneyweb.co.za             671                      37   \n",
       "\n",
       "         valid_articles_found  empty_articles_found  \\\n",
       "59999                      70                     0   \n",
       "60000                      70                     0   \n",
       "60001                      70                     0   \n",
       "60002                      70                     0   \n",
       "60003                      70                     0   \n",
       "...                       ...                   ...   \n",
       "1454255                  2095                   304   \n",
       "1454256                  2095                   304   \n",
       "1454257                  2095                   304   \n",
       "1454258                  2095                   304   \n",
       "1454259                  2095                   304   \n",
       "\n",
       "                                     to_sample_file_path          id  month  \\\n",
       "59999            AU\\9honey\\2016\\15895978_AU_13-12-16.txt  15895978.0   12.0   \n",
       "60000            AU\\9honey\\2016\\15855997_AU_11-12-16.txt  15855997.0   12.0   \n",
       "60001            AU\\9honey\\2016\\15849634_AU_11-12-16.txt  15849634.0   12.0   \n",
       "60002            AU\\9honey\\2016\\16008320_AU_20-12-16.txt  16008320.0   12.0   \n",
       "60003            AU\\9honey\\2016\\15732244_AU_04-12-16.txt  15732244.0   12.0   \n",
       "...                                                  ...         ...    ...   \n",
       "1454255  ZA\\moneyweb-co-za\\2019\\30726246_ZA_01-04-19.txt  30726246.0    4.0   \n",
       "1454256  ZA\\moneyweb-co-za\\2019\\30456195_ZA_15-01-19.txt  30456195.0    1.0   \n",
       "1454257  ZA\\moneyweb-co-za\\2019\\30938411_ZA_03-06-19.txt  30938411.0    6.0   \n",
       "1454258  ZA\\moneyweb-co-za\\2019\\30961580_ZA_10-06-19.txt  30961580.0    6.0   \n",
       "1454259  ZA\\moneyweb-co-za\\2019\\30503858_ZA_28-01-19.txt  30503858.0    1.0   \n",
       "\n",
       "         total_articles_found  num_articles_sampled  articles_in_folder_check  \\\n",
       "59999                      70                     6                     False   \n",
       "60000                      70                     6                     False   \n",
       "60001                      70                     6                     False   \n",
       "60002                      70                     6                     False   \n",
       "60003                      70                     6                     False   \n",
       "...                       ...                   ...                       ...   \n",
       "1454255                  2399                    37                     False   \n",
       "1454256                  2399                    37                     False   \n",
       "1454257                  2399                    37                     False   \n",
       "1454258                  2399                    37                     False   \n",
       "1454259                  2399                    37                     False   \n",
       "\n",
       "         num_articles_sampled_check  sanity_check  \n",
       "59999                          True         False  \n",
       "60000                          True         False  \n",
       "60001                          True         False  \n",
       "60002                          True         False  \n",
       "60003                          True         False  \n",
       "...                             ...           ...  \n",
       "1454255                        True         False  \n",
       "1454256                        True         False  \n",
       "1454257                        True         False  \n",
       "1454258                        True         False  \n",
       "1454259                        True         False  \n",
       "\n",
       "[297092 rows x 15 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sanity_check_df[~sanity_check_df.sanity_check]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country</th>\n",
       "      <th>year</th>\n",
       "      <th>publisher</th>\n",
       "      <th>total_articles</th>\n",
       "      <th>num_articles_to_sample</th>\n",
       "      <th>valid_articles_found</th>\n",
       "      <th>empty_articles_found</th>\n",
       "      <th>to_sample_file_path</th>\n",
       "      <th>id</th>\n",
       "      <th>month</th>\n",
       "      <th>total_articles_found</th>\n",
       "      <th>num_articles_sampled</th>\n",
       "      <th>articles_in_folder_check</th>\n",
       "      <th>num_articles_sampled_check</th>\n",
       "      <th>sanity_check</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>59999</th>\n",
       "      <td>AU</td>\n",
       "      <td>2016</td>\n",
       "      <td>9Honey</td>\n",
       "      <td>136</td>\n",
       "      <td>6</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>AU\\9honey\\2016\\15895978_AU_13-12-16.txt</td>\n",
       "      <td>15895978.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>70</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60000</th>\n",
       "      <td>AU</td>\n",
       "      <td>2016</td>\n",
       "      <td>9Honey</td>\n",
       "      <td>136</td>\n",
       "      <td>6</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>AU\\9honey\\2016\\15855997_AU_11-12-16.txt</td>\n",
       "      <td>15855997.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>70</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60001</th>\n",
       "      <td>AU</td>\n",
       "      <td>2016</td>\n",
       "      <td>9Honey</td>\n",
       "      <td>136</td>\n",
       "      <td>6</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>AU\\9honey\\2016\\15849634_AU_11-12-16.txt</td>\n",
       "      <td>15849634.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>70</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60002</th>\n",
       "      <td>AU</td>\n",
       "      <td>2016</td>\n",
       "      <td>9Honey</td>\n",
       "      <td>136</td>\n",
       "      <td>6</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>AU\\9honey\\2016\\16008320_AU_20-12-16.txt</td>\n",
       "      <td>16008320.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>70</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60003</th>\n",
       "      <td>AU</td>\n",
       "      <td>2016</td>\n",
       "      <td>9Honey</td>\n",
       "      <td>136</td>\n",
       "      <td>6</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>AU\\9honey\\2016\\15732244_AU_04-12-16.txt</td>\n",
       "      <td>15732244.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>70</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1454255</th>\n",
       "      <td>ZA</td>\n",
       "      <td>2019</td>\n",
       "      <td>moneyweb.co.za</td>\n",
       "      <td>671</td>\n",
       "      <td>37</td>\n",
       "      <td>2095</td>\n",
       "      <td>304</td>\n",
       "      <td>ZA\\moneyweb-co-za\\2019\\30726246_ZA_01-04-19.txt</td>\n",
       "      <td>30726246.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2399</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1454256</th>\n",
       "      <td>ZA</td>\n",
       "      <td>2019</td>\n",
       "      <td>moneyweb.co.za</td>\n",
       "      <td>671</td>\n",
       "      <td>37</td>\n",
       "      <td>2095</td>\n",
       "      <td>304</td>\n",
       "      <td>ZA\\moneyweb-co-za\\2019\\30456195_ZA_15-01-19.txt</td>\n",
       "      <td>30456195.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2399</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1454257</th>\n",
       "      <td>ZA</td>\n",
       "      <td>2019</td>\n",
       "      <td>moneyweb.co.za</td>\n",
       "      <td>671</td>\n",
       "      <td>37</td>\n",
       "      <td>2095</td>\n",
       "      <td>304</td>\n",
       "      <td>ZA\\moneyweb-co-za\\2019\\30938411_ZA_03-06-19.txt</td>\n",
       "      <td>30938411.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2399</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1454258</th>\n",
       "      <td>ZA</td>\n",
       "      <td>2019</td>\n",
       "      <td>moneyweb.co.za</td>\n",
       "      <td>671</td>\n",
       "      <td>37</td>\n",
       "      <td>2095</td>\n",
       "      <td>304</td>\n",
       "      <td>ZA\\moneyweb-co-za\\2019\\30961580_ZA_10-06-19.txt</td>\n",
       "      <td>30961580.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2399</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1454259</th>\n",
       "      <td>ZA</td>\n",
       "      <td>2019</td>\n",
       "      <td>moneyweb.co.za</td>\n",
       "      <td>671</td>\n",
       "      <td>37</td>\n",
       "      <td>2095</td>\n",
       "      <td>304</td>\n",
       "      <td>ZA\\moneyweb-co-za\\2019\\30503858_ZA_28-01-19.txt</td>\n",
       "      <td>30503858.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2399</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>297092 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        country  year       publisher  total_articles  num_articles_to_sample  \\\n",
       "59999        AU  2016          9Honey             136                       6   \n",
       "60000        AU  2016          9Honey             136                       6   \n",
       "60001        AU  2016          9Honey             136                       6   \n",
       "60002        AU  2016          9Honey             136                       6   \n",
       "60003        AU  2016          9Honey             136                       6   \n",
       "...         ...   ...             ...             ...                     ...   \n",
       "1454255      ZA  2019  moneyweb.co.za             671                      37   \n",
       "1454256      ZA  2019  moneyweb.co.za             671                      37   \n",
       "1454257      ZA  2019  moneyweb.co.za             671                      37   \n",
       "1454258      ZA  2019  moneyweb.co.za             671                      37   \n",
       "1454259      ZA  2019  moneyweb.co.za             671                      37   \n",
       "\n",
       "         valid_articles_found  empty_articles_found  \\\n",
       "59999                      70                     0   \n",
       "60000                      70                     0   \n",
       "60001                      70                     0   \n",
       "60002                      70                     0   \n",
       "60003                      70                     0   \n",
       "...                       ...                   ...   \n",
       "1454255                  2095                   304   \n",
       "1454256                  2095                   304   \n",
       "1454257                  2095                   304   \n",
       "1454258                  2095                   304   \n",
       "1454259                  2095                   304   \n",
       "\n",
       "                                     to_sample_file_path          id  month  \\\n",
       "59999            AU\\9honey\\2016\\15895978_AU_13-12-16.txt  15895978.0   12.0   \n",
       "60000            AU\\9honey\\2016\\15855997_AU_11-12-16.txt  15855997.0   12.0   \n",
       "60001            AU\\9honey\\2016\\15849634_AU_11-12-16.txt  15849634.0   12.0   \n",
       "60002            AU\\9honey\\2016\\16008320_AU_20-12-16.txt  16008320.0   12.0   \n",
       "60003            AU\\9honey\\2016\\15732244_AU_04-12-16.txt  15732244.0   12.0   \n",
       "...                                                  ...         ...    ...   \n",
       "1454255  ZA\\moneyweb-co-za\\2019\\30726246_ZA_01-04-19.txt  30726246.0    4.0   \n",
       "1454256  ZA\\moneyweb-co-za\\2019\\30456195_ZA_15-01-19.txt  30456195.0    1.0   \n",
       "1454257  ZA\\moneyweb-co-za\\2019\\30938411_ZA_03-06-19.txt  30938411.0    6.0   \n",
       "1454258  ZA\\moneyweb-co-za\\2019\\30961580_ZA_10-06-19.txt  30961580.0    6.0   \n",
       "1454259  ZA\\moneyweb-co-za\\2019\\30503858_ZA_28-01-19.txt  30503858.0    1.0   \n",
       "\n",
       "         total_articles_found  num_articles_sampled  articles_in_folder_check  \\\n",
       "59999                      70                     6                     False   \n",
       "60000                      70                     6                     False   \n",
       "60001                      70                     6                     False   \n",
       "60002                      70                     6                     False   \n",
       "60003                      70                     6                     False   \n",
       "...                       ...                   ...                       ...   \n",
       "1454255                  2399                    37                     False   \n",
       "1454256                  2399                    37                     False   \n",
       "1454257                  2399                    37                     False   \n",
       "1454258                  2399                    37                     False   \n",
       "1454259                  2399                    37                     False   \n",
       "\n",
       "         num_articles_sampled_check  sanity_check  \n",
       "59999                          True         False  \n",
       "60000                          True         False  \n",
       "60001                          True         False  \n",
       "60002                          True         False  \n",
       "60003                          True         False  \n",
       "...                             ...           ...  \n",
       "1454255                        True         False  \n",
       "1454256                        True         False  \n",
       "1454257                        True         False  \n",
       "1454258                        True         False  \n",
       "1454259                        True         False  \n",
       "\n",
       "[297092 rows x 15 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sanity_check_df[~sanity_check_df.articles_in_folder_check]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country</th>\n",
       "      <th>year</th>\n",
       "      <th>publisher</th>\n",
       "      <th>total_articles</th>\n",
       "      <th>num_articles_to_sample</th>\n",
       "      <th>valid_articles_found</th>\n",
       "      <th>empty_articles_found</th>\n",
       "      <th>to_sample_file_path</th>\n",
       "      <th>id</th>\n",
       "      <th>month</th>\n",
       "      <th>total_articles_found</th>\n",
       "      <th>num_articles_sampled</th>\n",
       "      <th>articles_in_folder_check</th>\n",
       "      <th>num_articles_sampled_check</th>\n",
       "      <th>sanity_check</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>22843</th>\n",
       "      <td>AU</td>\n",
       "      <td>2012</td>\n",
       "      <td>Bendigo Advertiser</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52796</th>\n",
       "      <td>AU</td>\n",
       "      <td>2015</td>\n",
       "      <td>Australasian Lawyer</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52802</th>\n",
       "      <td>AU</td>\n",
       "      <td>2015</td>\n",
       "      <td>Ballarat Courier</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54164</th>\n",
       "      <td>AU</td>\n",
       "      <td>2015</td>\n",
       "      <td>E! Online</td>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54188</th>\n",
       "      <td>AU</td>\n",
       "      <td>2015</td>\n",
       "      <td>Eurosport.com AU</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>664615</th>\n",
       "      <td>JM</td>\n",
       "      <td>2020</td>\n",
       "      <td>loopjamaica.com</td>\n",
       "      <td>1839</td>\n",
       "      <td>1839</td>\n",
       "      <td>1535</td>\n",
       "      <td>304</td>\n",
       "      <td>JM\\loopjamaica-com\\2020\\72288381_JM_30-09-20.txt</td>\n",
       "      <td>72288381.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1839</td>\n",
       "      <td>1535</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>664616</th>\n",
       "      <td>JM</td>\n",
       "      <td>2020</td>\n",
       "      <td>loopjamaica.com</td>\n",
       "      <td>1839</td>\n",
       "      <td>1839</td>\n",
       "      <td>1535</td>\n",
       "      <td>304</td>\n",
       "      <td>JM\\loopjamaica-com\\2020\\72288384_JM_30-09-20.txt</td>\n",
       "      <td>72288384.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1839</td>\n",
       "      <td>1535</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>664617</th>\n",
       "      <td>JM</td>\n",
       "      <td>2020</td>\n",
       "      <td>loopjamaica.com</td>\n",
       "      <td>1839</td>\n",
       "      <td>1839</td>\n",
       "      <td>1535</td>\n",
       "      <td>304</td>\n",
       "      <td>JM\\loopjamaica-com\\2020\\72288461_JM_30-09-20.txt</td>\n",
       "      <td>72288461.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1839</td>\n",
       "      <td>1535</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>664618</th>\n",
       "      <td>JM</td>\n",
       "      <td>2020</td>\n",
       "      <td>loopjamaica.com</td>\n",
       "      <td>1839</td>\n",
       "      <td>1839</td>\n",
       "      <td>1535</td>\n",
       "      <td>304</td>\n",
       "      <td>JM\\loopjamaica-com\\2020\\72288881_JM_30-09-20.txt</td>\n",
       "      <td>72288881.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1839</td>\n",
       "      <td>1535</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>664619</th>\n",
       "      <td>JM</td>\n",
       "      <td>2020</td>\n",
       "      <td>loopjamaica.com</td>\n",
       "      <td>1839</td>\n",
       "      <td>1839</td>\n",
       "      <td>1535</td>\n",
       "      <td>304</td>\n",
       "      <td>JM\\loopjamaica-com\\2020\\72288882_JM_30-09-20.txt</td>\n",
       "      <td>72288882.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1839</td>\n",
       "      <td>1535</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8873 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       country  year            publisher  total_articles  \\\n",
       "22843       AU  2012   Bendigo Advertiser               1   \n",
       "52796       AU  2015  Australasian Lawyer               8   \n",
       "52802       AU  2015     Ballarat Courier               3   \n",
       "54164       AU  2015            E! Online              21   \n",
       "54188       AU  2015     Eurosport.com AU              10   \n",
       "...        ...   ...                  ...             ...   \n",
       "664615      JM  2020      loopjamaica.com            1839   \n",
       "664616      JM  2020      loopjamaica.com            1839   \n",
       "664617      JM  2020      loopjamaica.com            1839   \n",
       "664618      JM  2020      loopjamaica.com            1839   \n",
       "664619      JM  2020      loopjamaica.com            1839   \n",
       "\n",
       "        num_articles_to_sample  valid_articles_found  empty_articles_found  \\\n",
       "22843                        1                     0                     1   \n",
       "52796                        1                     0                     8   \n",
       "52802                        1                     0                     3   \n",
       "54164                        4                     0                    21   \n",
       "54188                        2                     0                    10   \n",
       "...                        ...                   ...                   ...   \n",
       "664615                    1839                  1535                   304   \n",
       "664616                    1839                  1535                   304   \n",
       "664617                    1839                  1535                   304   \n",
       "664618                    1839                  1535                   304   \n",
       "664619                    1839                  1535                   304   \n",
       "\n",
       "                                     to_sample_file_path          id  month  \\\n",
       "22843                                                NaN         NaN    NaN   \n",
       "52796                                                NaN         NaN    NaN   \n",
       "52802                                                NaN         NaN    NaN   \n",
       "54164                                                NaN         NaN    NaN   \n",
       "54188                                                NaN         NaN    NaN   \n",
       "...                                                  ...         ...    ...   \n",
       "664615  JM\\loopjamaica-com\\2020\\72288381_JM_30-09-20.txt  72288381.0    9.0   \n",
       "664616  JM\\loopjamaica-com\\2020\\72288384_JM_30-09-20.txt  72288384.0    9.0   \n",
       "664617  JM\\loopjamaica-com\\2020\\72288461_JM_30-09-20.txt  72288461.0    9.0   \n",
       "664618  JM\\loopjamaica-com\\2020\\72288881_JM_30-09-20.txt  72288881.0    9.0   \n",
       "664619  JM\\loopjamaica-com\\2020\\72288882_JM_30-09-20.txt  72288882.0    9.0   \n",
       "\n",
       "        total_articles_found  num_articles_sampled  articles_in_folder_check  \\\n",
       "22843                      1                     0                      True   \n",
       "52796                      8                     0                      True   \n",
       "52802                      3                     0                      True   \n",
       "54164                     21                     0                      True   \n",
       "54188                     10                     0                      True   \n",
       "...                      ...                   ...                       ...   \n",
       "664615                  1839                  1535                      True   \n",
       "664616                  1839                  1535                      True   \n",
       "664617                  1839                  1535                      True   \n",
       "664618                  1839                  1535                      True   \n",
       "664619                  1839                  1535                      True   \n",
       "\n",
       "        num_articles_sampled_check  sanity_check  \n",
       "22843                        False          True  \n",
       "52796                        False          True  \n",
       "52802                        False          True  \n",
       "54164                        False          True  \n",
       "54188                        False          True  \n",
       "...                            ...           ...  \n",
       "664615                       False          True  \n",
       "664616                       False          True  \n",
       "664617                       False          True  \n",
       "664618                       False          True  \n",
       "664619                       False          True  \n",
       "\n",
       "[8873 rows x 15 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sanity_check_df[~sanity_check_df.num_articles_sampled_check]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████| 1465233/1465233 [2:15:06<00:00, 180.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 2h 15min 6s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "_ = to_sample_df[~to_sample_df.to_sample_file_path.isna()].to_sample_file_path.progress_apply(copy_sampled_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Number of paths where the directory wasn't found."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_articles[sampled_articles.to_sample_file_path.isna()].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country</th>\n",
       "      <th>year</th>\n",
       "      <th>publisher</th>\n",
       "      <th>total_articles</th>\n",
       "      <th>num_articles_to_sample</th>\n",
       "      <th>valid_articles_found</th>\n",
       "      <th>empty_articles_found</th>\n",
       "      <th>to_sample_file_path</th>\n",
       "      <th>id</th>\n",
       "      <th>month</th>\n",
       "      <th>total_articles_found</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>22843</th>\n",
       "      <td>AU</td>\n",
       "      <td>2012</td>\n",
       "      <td>Bendigo Advertiser</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52796</th>\n",
       "      <td>AU</td>\n",
       "      <td>2015</td>\n",
       "      <td>Australasian Lawyer</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52802</th>\n",
       "      <td>AU</td>\n",
       "      <td>2015</td>\n",
       "      <td>Ballarat Courier</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54164</th>\n",
       "      <td>AU</td>\n",
       "      <td>2015</td>\n",
       "      <td>E! Online</td>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54188</th>\n",
       "      <td>AU</td>\n",
       "      <td>2015</td>\n",
       "      <td>Eurosport.com AU</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59954</th>\n",
       "      <td>AU</td>\n",
       "      <td>2015</td>\n",
       "      <td>Western Advocate</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100129</th>\n",
       "      <td>AU</td>\n",
       "      <td>2020</td>\n",
       "      <td>CyclingTips</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       country  year            publisher  total_articles  \\\n",
       "22843       AU  2012   Bendigo Advertiser               1   \n",
       "52796       AU  2015  Australasian Lawyer               8   \n",
       "52802       AU  2015     Ballarat Courier               3   \n",
       "54164       AU  2015            E! Online              21   \n",
       "54188       AU  2015     Eurosport.com AU              10   \n",
       "59954       AU  2015     Western Advocate               7   \n",
       "100129      AU  2020          CyclingTips               1   \n",
       "\n",
       "        num_articles_to_sample  valid_articles_found  empty_articles_found  \\\n",
       "22843                        1                     0                     1   \n",
       "52796                        1                     0                     8   \n",
       "52802                        1                     0                     3   \n",
       "54164                        4                     0                    21   \n",
       "54188                        2                     0                    10   \n",
       "59954                        1                     0                     7   \n",
       "100129                       1                     0                     1   \n",
       "\n",
       "       to_sample_file_path  id  month  total_articles_found  \n",
       "22843                  NaN NaN    NaN                     1  \n",
       "52796                  NaN NaN    NaN                     8  \n",
       "52802                  NaN NaN    NaN                     3  \n",
       "54164                  NaN NaN    NaN                    21  \n",
       "54188                  NaN NaN    NaN                    10  \n",
       "59954                  NaN NaN    NaN                     7  \n",
       "100129                 NaN NaN    NaN                     1  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_articles[sampled_articles.to_sample_file_path.isna()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Two reasons for this. \n",
    "#### 1. The folder wasn't created beacuse none of the files in the folder were selected when subsampled. <br> (i.e. our 'clean' folder is output of running Clean Text Files- SAMPLE.ipynb, with MAX_SIZE = 50). <br> For example, none of the articles from year 2010 of the publisher skysports-com were selected thus the folder wasn't created. This won't be a problem on your side since you are using all the data. \n",
    "#### 2. Most of the US publishers don't have a 2019/2020 year folder. I remember you saying that there was a problem in how some of the US raw files were formatted. I'm not sure if you solved this problem but I think this is the reason why there are so many US 2019/2020 directories not found. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country</th>\n",
       "      <th>year</th>\n",
       "      <th>publisher</th>\n",
       "      <th>articles_found</th>\n",
       "      <th>to_sample_file_paths</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AU</td>\n",
       "      <td>2010</td>\n",
       "      <td>ABC Local</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2263</th>\n",
       "      <td>AU</td>\n",
       "      <td>2010</td>\n",
       "      <td>AdNews</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2265</th>\n",
       "      <td>AU</td>\n",
       "      <td>2010</td>\n",
       "      <td>AdNews</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2367</th>\n",
       "      <td>AU</td>\n",
       "      <td>2010</td>\n",
       "      <td>Architecture and Design</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2568</th>\n",
       "      <td>AU</td>\n",
       "      <td>2010</td>\n",
       "      <td>Ballarat Courier</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1511934</th>\n",
       "      <td>ZA</td>\n",
       "      <td>2020</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1514118</th>\n",
       "      <td>ZA</td>\n",
       "      <td>2020</td>\n",
       "      <td>channel24.co.za</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1515955</th>\n",
       "      <td>ZA</td>\n",
       "      <td>2020</td>\n",
       "      <td>fin24.com</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1516052</th>\n",
       "      <td>ZA</td>\n",
       "      <td>2020</td>\n",
       "      <td>goal.com</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1518888</th>\n",
       "      <td>ZA</td>\n",
       "      <td>2020</td>\n",
       "      <td>news24.com</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>27674 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        country  year                publisher  articles_found  \\\n",
       "0            AU  2010                ABC Local             NaN   \n",
       "2263         AU  2010                   AdNews             NaN   \n",
       "2265         AU  2010                   AdNews             NaN   \n",
       "2367         AU  2010  Architecture and Design             NaN   \n",
       "2568         AU  2010         Ballarat Courier             NaN   \n",
       "...         ...   ...                      ...             ...   \n",
       "1511934      ZA  2020                  Reuters             NaN   \n",
       "1514118      ZA  2020          channel24.co.za             NaN   \n",
       "1515955      ZA  2020                fin24.com             NaN   \n",
       "1516052      ZA  2020                 goal.com             NaN   \n",
       "1518888      ZA  2020               news24.com             NaN   \n",
       "\n",
       "        to_sample_file_paths  id  \n",
       "0                        NaN NaN  \n",
       "2263                     NaN NaN  \n",
       "2265                     NaN NaN  \n",
       "2367                     NaN NaN  \n",
       "2568                     NaN NaN  \n",
       "...                      ...  ..  \n",
       "1511934                  NaN NaN  \n",
       "1514118                  NaN NaN  \n",
       "1515955                  NaN NaN  \n",
       "1516052                  NaN NaN  \n",
       "1518888                  NaN NaN  \n",
       "\n",
       "[27674 rows x 6 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_sample_df[to_sample_df.to_sample_file_paths.isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}