1-clean-texts-novel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import re\n",
    "from itertools import groupby \n",
    "from string import punctuation\n",
    "import random\n",
    "import os\n",
    "from glob import glob\n",
    "from tqdm import tqdm\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_file_path = 'output/0-novels.csv'\n",
    "output_dir = 'output'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "VALID_BANGLA_REGEX = re.compile(r'[^\\u0980-\\u09FF ।,?!\\(\\)-.”“/;\"‘:\\']+')\n",
    "SPLIT_REGEX = re.compile(r'\\n|।|\\?')\n",
    "SPLIT_BY_NEWLINE_REGEX = re.compile(r'\\n')\n",
    "EMOJI_REGEX = re.compile(\n",
    "    \"([\"\n",
    "    \"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
    "    \"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
    "    \"\\U0001F600-\\U0001F64F\"  # emoticons\n",
    "    \"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
    "    \"\\U0001F700-\\U0001F77F\"  # alchemical symbols\n",
    "    \"\\U0001F780-\\U0001F7FF\"  # Geometric Shapes Extended\n",
    "    \"\\U0001F800-\\U0001F8FF\"  # Supplemental Arrows-C\n",
    "    \"\\U0001F900-\\U0001F9FF\"  # Supplemental Symbols and Pictographs\n",
    "    \"\\U0001FA00-\\U0001FA6F\"  # Chess Symbols\n",
    "    \"\\U0001FA70-\\U0001FAFF\"  # Symbols and Pictographs Extended-A\n",
    "    \"\\U00002702-\\U000027B0\"  # Dingbats\n",
    "    \"])\"\n",
    "    )\n",
    "\n",
    "\n",
    "def is_all_bangla(text, allowed_threshold=96.9):\n",
    "    \"\"\"\n",
    "    Allows some non-bangla characters (excluding punctuations) and \n",
    "    finds out if contains greater than <percent> characters are from Bangla language\n",
    "    \"\"\"\n",
    "    valid_bangla_text = VALID_BANGLA_REGEX.sub('', text)\n",
    "    valid_bangla_text_len = len(valid_bangla_text)\n",
    "    all_text_len = len(text)\n",
    "    \n",
    "    percent_bangla = (valid_bangla_text_len*100.0) / all_text_len\n",
    "\n",
    "    if percent_bangla < allowed_threshold:\n",
    "        return False\n",
    "    return True\n",
    "    \n",
    "    \n",
    "    \n",
    "def is_valid(text):\n",
    "    l= len(text)\n",
    "    if l<MIN_CHAR_LEN or l>MAX_CHAR_LEN:\n",
    "        return False\n",
    "    if len(text.split())<MIN_WORD_LEN:\n",
    "        return False\n",
    "    if not is_all_bangla(text):\n",
    "        return False\n",
    "    \n",
    "    return True\n",
    "\n",
    "def split_sentences(text):\n",
    "    return SPLIT_REGEX.split(text)\n",
    "\n",
    "\n",
    "def split_sentences_from_paragraphs(paragraphs):\n",
    "    sentences = []\n",
    "    for p in paragraphs:\n",
    "        sentences.append(SPLIT_REGEX.split(p))\n",
    "    return sentences\n",
    "\n",
    "\n",
    "def split_paragraphs(text):\n",
    "    return SPLIT_BY_NEWLINE_REGEX.split(text)\n",
    "\n",
    "\n",
    "def remove_repeating_puncts(texts, punctuations=set(punctuation+ '।\\n')):\n",
    "    # ref: https://stackoverflow.com/a/32485876\n",
    "    newtext = []\n",
    "    for k, g in groupby(texts):\n",
    "        if k in punctuations:\n",
    "            newtext.append(k)\n",
    "        else:\n",
    "            newtext.extend(g)\n",
    "    return ''.join(newtext)\n",
    "\n",
    "\n",
    "def remove_emoticons(text):\n",
    "    # Ref: https://gist.github.com/Alex-Just/e86110836f3f93fe7932290526529cd1#gistcomment-3208085\n",
    "    # Ref: https://en.wikipedia.org/wiki/Unicode_block\n",
    "    text = re.sub(EMOJI_REGEX, r' ', text)\n",
    "    return text\n",
    "\n",
    "\n",
    "def remove_non_bangla_characters(text):\n",
    "    if isinstance(text, str):\n",
    "        cleaned = VALID_BANGLA_REGEX.sub(' ', text)\n",
    "        return cleaned\n",
    "    elif isinstance(text, list):\n",
    "        results = []\n",
    "        for item in text:\n",
    "            cleaned = VALID_BANGLA_REGEX.sub(' ', item)\n",
    "            results.append(cleaned)\n",
    "        return results\n",
    "    else:\n",
    "        raise Exception('invalid param')\n",
    "        \n",
    "    \n",
    "def is_poem(paragraphs):\n",
    "    \"\"\"Find out if blog article is poem or not, by analyzing avg. paragraph length\"\"\"\n",
    "    \n",
    "    total_len = 0\n",
    "    for p in paragraphs:\n",
    "        if not isinstance(p, str):\n",
    "            print('error ', p)\n",
    "        total_len += len(p)\n",
    "    avg_para_len = total_len / len(paragraphs)\n",
    "\n",
    "    if avg_para_len < 100:\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_df = pd.read_csv(input_file_path)\n",
    "input_df.head() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_chars = []\n",
    "char_map = {}\n",
    "count = 0\n",
    "count_not_bangla = 0\n",
    "\n",
    "filtered_texts = []\n",
    "for idx, row in tqdm(input_df.iterrows()):\n",
    "    text = row[text]\n",
    "    url = row[novel]\n",
    "    if not isinstance(text, str):\n",
    "        continue\n",
    "\n",
    "    text = remove_repeating_puncts(text)\n",
    "    text = remove_emoticons(text)\n",
    "    if not is_all_bangla(text):\n",
    "        count_not_bangla += 1\n",
    "        continue\n",
    "\n",
    "    paragraphs = split_paragraphs(text)\n",
    "\n",
    "    if is_poem(paragraphs):\n",
    "        print(\"Found poem: \", url)\n",
    "        count += 1\n",
    "        print(paragraphs[0])\n",
    "        continue\n",
    "    \n",
    "    filtered_texts.append((text, url))\n",
    "    \n",
    "print(count)\n",
    "print(count_not_bangla)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "paragraphs_for_next_step = []\n",
    "documents = []\n",
    "need_document = 18000\n",
    "\n",
    "rejected_count = 0\n",
    "for text, url in tqdm(filtered_texts):\n",
    "    # if already found n documents needed, break\n",
    "    if len(documents) >= need_document:\n",
    "        break\n",
    "        \n",
    "    if not isinstance(text, str):\n",
    "        continue\n",
    "        \n",
    "    paragraphs = split_paragraphs(text)\n",
    "    cleaned_paragraphs = remove_non_bangla_characters(paragraphs)\n",
    "    string = \" \".join(cleaned_paragraphs)\n",
    "    \n",
    "    if len(paragraphs) < 3:\n",
    "        rejected_count += 1\n",
    "        continue\n",
    "        \n",
    "    if len(string) < 800: # character count\n",
    "        rejected_count += 1\n",
    "        continue\n",
    "    \n",
    "    # trim documents with more than 10 paragraphs \n",
    "    if len(paragraphs) > 10:\n",
    "        no_of_paragraphs = random.randint(3, 10)\n",
    "        cleaned_paragraphs = cleaned_paragraphs[:no_of_paragraphs]\n",
    "        cleaned_text = '\\n'.join(cleaned_paragraphs)\n",
    "        \n",
    "        words = cleaned_text.split()\n",
    "        documents.append({'text': cleaned_text, 'url':url, 'para_count': no_of_paragraphs, 'word_count': len(words)})\n",
    "        \n",
    "        for p in paragraphs[:no_of_paragraphs]:\n",
    "            paragraphs_for_next_step.append((p, url))\n",
    "    else:\n",
    "        cleaned_text = '\\n'.join(cleaned_paragraphs)\n",
    "        words = cleaned_text.split()\n",
    "        documents.append({'text': cleaned_text, 'url':url, 'para_count': len(paragraphs), 'word_count': len(words)})\n",
    "        \n",
    "\n",
    "print('Rejected documents: ', rejected_count)\n",
    "print(\"Got clean documents: \", len(documents))\n",
    "print(\"Left paragraph for next: \", len(paragraphs_for_next_step))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check stats and export Document CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "document_df = pd.DataFrame(documents)\n",
    "document_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter lower word count documents, if some slipped through\n",
    "document_df = document_df[document_df['word_count'] > 40]\n",
    "\n",
    "# drop some columns to match with our standard CSV format\n",
    "document_df = document_df.drop('para_count', axis=1)\n",
    "document_df = document_df.drop('word_count', axis=1)\n",
    "document_df['source'] = 'novel'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "document_df.to_csv(os.path.join('output', '1-novel-documents.csv'), index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract Paragraphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences_for_next_step = []\n",
    "paragraphs = []\n",
    "\n",
    "rejected = 0\n",
    "for text, url in tqdm(paragraphs_for_next_step):\n",
    "    if not isinstance(text, str):\n",
    "        continue\n",
    "        \n",
    "    sentences = split_sentences(text)\n",
    "    sentences = remove_non_bangla_characters(sentences)\n",
    "    \n",
    "    if len(text) < 200:\n",
    "        for s in sentences:\n",
    "            sentences_for_next_step.append((s, url))\n",
    "        rejected += 1\n",
    "        continue\n",
    "        \n",
    "    if len(sentences) < 3 or len(sentences) > 15:\n",
    "        for s in sentences:\n",
    "            sentences_for_next_step.append((s, url))\n",
    "        rejected += 1\n",
    "        continue    \n",
    "    \n",
    "    cleaned_text = \"। \".join(sentences)\n",
    "    word_count = len(cleaned_text.split())\n",
    "    paragraphs.append({'text': cleaned_text, 'url': url, 'word_count': word_count, 'sent_count': len(sentences) })\n",
    "    \n",
    "print('Rejected paragraphs: ', rejected)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check stats and export Paragraph CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check stats\n",
    "para_df = pd.DataFrame(paragraphs)\n",
    "para_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter lower word count documents, if some slipped through\n",
    "# para_df = para_df[para_df['word_count'] > 20]\n",
    "\n",
    "# drop some columns to match with our standard CSV format\n",
    "para_df = para_df.drop('sent_count', axis=1)\n",
    "para_df = para_df.drop('word_count', axis=1)\n",
    "para_df['source'] = 'blog'\n",
    "\n",
    "para_df.to_csv(os.path.join(output_dir, '1-novel-paragraphs.csv'), index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rejected = 0\n",
    "sentences = []\n",
    "for s, url in tqdm(sentences_for_next_step):\n",
    "    if not isinstance(s, str):\n",
    "        continue\n",
    "    s = remove_non_bangla_characters(s)\n",
    "    if len(s) < 20:\n",
    "        rejected += 1\n",
    "        continue\n",
    "    \n",
    "    words = s.split()\n",
    "    if len(words) < 3 and len(words) > 15:\n",
    "        rejected += 1\n",
    "        continue\n",
    "    \n",
    "    sentences.append({'text': s, 'word_count': len(words), 'char_count': len(s)})\n",
    "    \n",
    "print(\"Rejected sentences: \", rejected)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check stats and export sentenc CSVs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_df = pd.DataFrame(sentences)\n",
    "sentence_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter lower word count sentences, if some slipped through\n",
    "sentence_df = sentence_df[sentence_df['word_count'] > 3]\n",
    "\n",
    "# drop some columns to match with our standard CSV format\n",
    "sentence_df = sentence_df.drop('char_count', axis=1)\n",
    "sentence_df = sentence_df.drop('word_count', axis=1)\n",
    "sentence_df['source'] = 'novel'\n",
    "\n",
    "sentence_df.to_csv(os.path.join(output_dir, '1-novel-sentences.csv'), index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}