Text_Cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import replace_contraction #to replace contractions with their full forms "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'do not'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "expandContractions(\"don't\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=\"Hi Steve  Thank you so much for reaching out and taking the time to send us feedback! It is very much appreciated! Please excuse the delayed response. Unfortunately, these scanning issues mostly occur due to older scanning technologies still used by some stores. Some devices (e.g. laser scanners or flatbed scanners) have difficulties reading smartphone displays. We hope to be able to solve this problem soon though as most stores will hopefully replace these scanning devices with newer image-based scanners in the course of their next cash system update. Until then, you can always ask the store staff to type in the card number manually in these cases so that you don't miss out on any rewards points or discounts. In the meantime, I sincerely apologize for the inconvenience this causes.  Once again, thank you for taking the time to contact us about this issue. If you have any further questions, suggestions for improvements or general feedback, please don't hesitate to contact us again.Best regards,Isabelle van Capelleveen Customer Support GmbH C-HUB / Hafenstraße 25-27 68159 Mannheim\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=expandContractions(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now to get frquency of iimportant words we need to do stemming as same word can have different representation in diff context\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "\n",
    "sbEng = SnowballStemmer('english')\n",
    "sbEsp = SnowballStemmer('spanish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hi\n",
      "steve\n",
      "\n",
      "thank\n",
      "you\n",
      "so\n",
      "much\n",
      "for\n",
      "reach\n",
      "out\n",
      "and\n",
      "take\n",
      "the\n",
      "time\n",
      "to\n",
      "send\n",
      "us\n",
      "feedback!\n",
      "it\n",
      "is\n",
      "veri\n",
      "much\n",
      "appreciated!\n",
      "pleas\n",
      "excus\n",
      "the\n",
      "delay\n",
      "response.\n",
      "unfortunately,\n",
      "these\n",
      "scan\n",
      "issu\n",
      "most\n",
      "occur\n",
      "due\n",
      "to\n",
      "older\n",
      "scan\n",
      "technolog\n",
      "still\n",
      "use\n",
      "by\n",
      "some\n",
      "stores.\n",
      "some\n",
      "devic\n",
      "(e.g.\n",
      "laser\n",
      "scanner\n",
      "or\n",
      "flatb\n",
      "scanners)\n",
      "have\n",
      "difficulti\n",
      "read\n",
      "smartphon\n",
      "displays.\n",
      "we\n",
      "hope\n",
      "to\n",
      "be\n",
      "abl\n",
      "to\n",
      "solv\n",
      "this\n",
      "problem\n",
      "soon\n",
      "though\n",
      "as\n",
      "most\n",
      "store\n",
      "will\n",
      "hope\n",
      "replac\n",
      "these\n",
      "scan\n",
      "devic\n",
      "with\n",
      "newer\n",
      "image-bas\n",
      "scanner\n",
      "in\n",
      "the\n",
      "cours\n",
      "of\n",
      "their\n",
      "next\n",
      "cash\n",
      "system\n",
      "update.\n",
      "until\n",
      "then,\n",
      "you\n",
      "can\n",
      "alway\n",
      "ask\n",
      "the\n",
      "store\n",
      "staff\n",
      "to\n",
      "type\n",
      "in\n",
      "the\n",
      "card\n",
      "number\n",
      "manual\n",
      "in\n",
      "these\n",
      "case\n",
      "so\n",
      "that\n",
      "you\n",
      "don't\n",
      "miss\n",
      "out\n",
      "on\n",
      "ani\n",
      "reward\n",
      "point\n",
      "or\n",
      "discounts.\n",
      "in\n",
      "the\n",
      "meantime,\n",
      "i\n",
      "sincer\n",
      "apolog\n",
      "for\n",
      "the\n",
      "inconveni\n",
      "this\n",
      "causes.\n",
      "\n",
      "onc\n",
      "again,\n",
      "thank\n",
      "you\n",
      "for\n",
      "take\n",
      "the\n",
      "time\n",
      "to\n",
      "contact\n",
      "us\n",
      "about\n",
      "this\n",
      "issue.\n",
      "if\n",
      "you\n",
      "have\n",
      "ani\n",
      "further\n",
      "questions,\n",
      "suggest\n",
      "for\n",
      "improv\n",
      "or\n",
      "general\n",
      "feedback,\n",
      "pleas\n",
      "don't\n",
      "hesit\n",
      "to\n",
      "contact\n",
      "us\n",
      "again.best\n",
      "regards,isabell\n",
      "van\n",
      "capelleveen\n",
      "custom\n",
      "support\n",
      "gmbh\n",
      "c-hub\n",
      "/\n",
      "hafenstraß\n",
      "25-27\n",
      "68159\n",
      "mannheim\n"
     ]
    }
   ],
   "source": [
    "for item in (data).split(' '):\n",
    "    print (sbEng.stem(item) )\n",
    "data1=' '.join([sbEng.stem(item) for item in (data).split(' ')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"hi steve  thank you so much for reach out and take the time to send us feedback! it is veri much appreciated! pleas excus the delay response. unfortunately, these scan issu most occur due to older scan technolog still use by some stores. some devic (e.g. laser scanner or flatb scanners) have difficulti read smartphon displays. we hope to be abl to solv this problem soon though as most store will hope replac these scan devic with newer image-bas scanner in the cours of their next cash system update. until then, you can alway ask the store staff to type in the card number manual in these case so that you don't miss out on ani reward point or discounts. in the meantime, i sincer apolog for the inconveni this causes.  onc again, thank you for take the time to contact us about this issue. if you have ani further questions, suggest for improv or general feedback, pleas don't hesit to contact us again.best regards,isabell van capelleveen custom support gmbh c-hub / hafenstraß 25-27 68159 mannheim\""
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data1   #after stemming and removing contraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem.lancaster import LancasterStemmer#Stemming only performed cutting prefix or suffix sometimes thats not useful so lemmatization uses morphological analysis too over words\n",
    "lancaster_stemmer = LancasterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "data2=' '.join(lancaster_stemmer.stem(item) for item in data.split(\" \"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'hi stev  thank you so much for reach out and tak the tim to send us feedback! it is very much appreciated! pleas excus the delay response. unfortunately, thes scan issu most occ due to old scan technolog stil us by som stores. som dev (e.g. las scan or flatb scanners) hav difficul read smartphon displays. we hop to be abl to solv thi problem soon though as most stor wil hop replac thes scan dev with new image-based scan in the cours of their next cash system update. until then, you can alway ask the stor staff to typ in the card numb man in thes cas so that you do not miss out on any reward point or discounts. in the meantime, i sint apolog for the inconveny thi causes.  ont again, thank you for tak the tim to contact us about thi issue. if you hav any furth questions, suggest for improv or gen feedback, pleas do not hesit to contact us again.best regards,isabelle van capelleveen custom support gmbh c-hub / hafenstraß 25-27 68159 mannheim'"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'tim'"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lancaster_stemmer.stem(\"time\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pattern.en as lemEng\n",
    " #Stemming only performed cutting prefix or suffix sometimes thats not useful so lemmatization uses morphological analysis too over words\n",
    "import pattern.es as lemEsp #for spanish words    \n",
    "  # this is not supported in python3 thats why not running this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We need to remove the punctuation marks and all of that extra whitespace. Another thing we want to get rid of are non-signal, or stop words, that are likely to be common across texts, such as ‘a’, ‘the’, and ‘in’. These tasks fall into a process called normalisation.\n",
    "# There is a multi-language package called cucco that can do all of the most common normalisation tasks in English, Spanish and about 10 other languages.\n",
    "\n",
    "from cucco import Cucco\n",
    "\n",
    "normEng = Cucco(language='en')\n",
    "normEsp = Cucco(language='es') # for spanish"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "norms = ['remove_stop_words', 'replace_punctuation', 'remove_extra_whitespaces']\n",
    "data2=normEng.normalize(data2, norms)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Mojibake---when your text gets changed from one form of encoding to another and your special characters and punctuation turn into that crazy character.We can reclaim our special characters using ftfy(fix txt for you)\n",
    "import ftfy\n",
    "\n",
    "data2= (ftfy.fix_encoding(data2))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['hi'], ['stev'], ['thank'], ['much'], ['reach'], ['tak'], ['tim'], ['send'], ['us'], ['feedback'], ['much'], ['appreciated'], ['pleas'], ['excus'], ['delay'], ['response'], ['unfortunately'], ['thes'], ['scan'], ['issu'], ['occ'], ['due'], ['old'], ['scan'], ['technolog'], ['stil'], ['us'], ['som'], ['stores'], ['som'], ['dev'], ['eg'], ['las'], ['scan'], ['flatb'], ['scanners'], ['hav'], ['difficul'], ['read'], ['smartphon'], ['displays'], ['hop'], ['abl'], ['solv'], ['thi'], ['problem'], ['soon'], ['though'], ['stor'], ['wil'], ['hop'], ['replac'], ['thes'], ['scan'], ['dev'], ['new'], ['imagebased'], ['scan'], ['cours'], ['next'], ['cash'], ['system'], ['update'], ['then'], ['can'], ['alway'], ['ask'], ['stor'], ['staff'], ['typ'], ['card'], ['numb'], ['man'], ['thes'], ['cas'], ['miss'], ['reward'], ['point'], ['discounts'], ['meantime'], ['sint'], ['apolog'], ['inconveny'], ['thi'], ['causes'], ['ont'], ['again'], ['thank'], ['tak'], ['tim'], ['contact'], ['us'], ['thi'], ['issue'], ['hav'], ['furth'], ['questions'], ['suggest'], ['improv'], ['gen'], ['feedback'], ['pleas'], ['hesit'], ['contact'], ['us'], ['againbest'], ['regardsisabelle'], ['van'], ['capelleveen'], ['custom'], ['support'], ['gmbh'], ['chub'], ['hafenstraß'], ['2527'], ['68159'], ['mannheim']]\n"
     ]
    }
   ],
   "source": [
    "engTokens = [word_tokenize(text) for text in data2.split(\" \")] #tokenization\n",
    "print(engTokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import FreqDist #frequency count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "flatList = [word for sentList in engTokens for word in sentList] #flattening "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['hi',\n",
       " 'stev',\n",
       " 'thank',\n",
       " 'much',\n",
       " 'reach',\n",
       " 'tak',\n",
       " 'tim',\n",
       " 'send',\n",
       " 'us',\n",
       " 'feedback',\n",
       " 'much',\n",
       " 'appreciated',\n",
       " 'pleas',\n",
       " 'excus',\n",
       " 'delay',\n",
       " 'response',\n",
       " 'unfortunately',\n",
       " 'thes',\n",
       " 'scan',\n",
       " 'issu',\n",
       " 'occ',\n",
       " 'due',\n",
       " 'old',\n",
       " 'scan',\n",
       " 'technolog',\n",
       " 'stil',\n",
       " 'us',\n",
       " 'som',\n",
       " 'stores',\n",
       " 'som',\n",
       " 'dev',\n",
       " 'eg',\n",
       " 'las',\n",
       " 'scan',\n",
       " 'flatb',\n",
       " 'scanners',\n",
       " 'hav',\n",
       " 'difficul',\n",
       " 'read',\n",
       " 'smartphon',\n",
       " 'displays',\n",
       " 'hop',\n",
       " 'abl',\n",
       " 'solv',\n",
       " 'thi',\n",
       " 'problem',\n",
       " 'soon',\n",
       " 'though',\n",
       " 'stor',\n",
       " 'wil',\n",
       " 'hop',\n",
       " 'replac',\n",
       " 'thes',\n",
       " 'scan',\n",
       " 'dev',\n",
       " 'new',\n",
       " 'imagebased',\n",
       " 'scan',\n",
       " 'cours',\n",
       " 'next',\n",
       " 'cash',\n",
       " 'system',\n",
       " 'update',\n",
       " 'then',\n",
       " 'can',\n",
       " 'alway',\n",
       " 'ask',\n",
       " 'stor',\n",
       " 'staff',\n",
       " 'typ',\n",
       " 'card',\n",
       " 'numb',\n",
       " 'man',\n",
       " 'thes',\n",
       " 'cas',\n",
       " 'miss',\n",
       " 'reward',\n",
       " 'point',\n",
       " 'discounts',\n",
       " 'meantime',\n",
       " 'sint',\n",
       " 'apolog',\n",
       " 'inconveny',\n",
       " 'thi',\n",
       " 'causes',\n",
       " 'ont',\n",
       " 'again',\n",
       " 'thank',\n",
       " 'tak',\n",
       " 'tim',\n",
       " 'contact',\n",
       " 'us',\n",
       " 'thi',\n",
       " 'issue',\n",
       " 'hav',\n",
       " 'furth',\n",
       " 'questions',\n",
       " 'suggest',\n",
       " 'improv',\n",
       " 'gen',\n",
       " 'feedback',\n",
       " 'pleas',\n",
       " 'hesit',\n",
       " 'contact',\n",
       " 'us',\n",
       " 'againbest',\n",
       " 'regardsisabelle',\n",
       " 'van',\n",
       " 'capelleveen',\n",
       " 'custom',\n",
       " 'support',\n",
       " 'gmbh',\n",
       " 'chub',\n",
       " 'hafenstraß',\n",
       " '2527',\n",
       " '68159',\n",
       " 'mannheim']"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "flatList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scan: 5\n",
      "us: 4\n",
      "thes: 3\n",
      "thi: 3\n",
      "much: 2\n",
      "pleas: 2\n",
      "thank: 2\n",
      "hop: 2\n",
      "tak: 2\n",
      "contact: 2\n",
      "feedback: 2\n",
      "dev: 2\n",
      "som: 2\n",
      "tim: 2\n",
      "hav: 2\n"
     ]
    }
   ],
   "source": [
    "Freq = FreqDist(word for word in flatList)\n",
    "\n",
    "for word, frequency in Freq.most_common(15):\n",
    "\n",
    "    print(u'{}: {}'.format(word, frequency))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "                                 Text cleaning task done for multiple languages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}