hackathon_tcs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\cheekushivam\\Anconda_file\\lib\\site-packages\\requests\\__init__.py:80: RequestsDependencyWarning: urllib3 (1.23) or chardet (3.0.4) doesn't match a supported version!\n",
      "  RequestsDependencyWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing of Files Done\n"
     ]
    }
   ],
   "source": [
    "# Importing of Packages\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import itertools\n",
    "import pandas as pd\n",
    "import re\n",
    "import chardet\n",
    "import numpy as np\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import requests\n",
    "site = \"https://fbookshelf.herokuapp.com\"\n",
    "page = requests.get(site)\n",
    "print(\"Importing of Files Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data sent To CSV\n"
     ]
    }
   ],
   "source": [
    "# Scraping Data from this Site \"https://fbookshelf.herokuapp.com/#\" and converting into pandas DataFrame\n",
    "\n",
    "columnNames = ['Title','Author','Genre','NumberofPages','StarRating','NumberofReviews','ShortDescription','ISBNNumber','NumbeofVotes','Comments']\n",
    "hackathonDF = pd.DataFrame(columns = columnNames)\n",
    "\n",
    "titleArray=[]\n",
    "authorArray=[]\n",
    "genreArray=[]\n",
    "pagesArray=[]\n",
    "starRatingArray=[]\n",
    "descriptionArray=[]\n",
    "reviewsArray=[]\n",
    "isbnArray=[]\n",
    "votesArray=[]\n",
    "commentsArray=[]\n",
    "commentsInternalData=\"\"\n",
    "soup = BeautifulSoup(page.content, 'html.parser')\n",
    "contentDiv = soup.find_all(class_=\"col-md-8\")\n",
    "for content in contentDiv:\n",
    "    title = content.find('h3').get_text()\n",
    "    titleArray.append(title)\n",
    "    authorby = content.find(class_=\"author\").get_text()\n",
    "    author = authorby.split(\"by\")[1]\n",
    "    authorArray.append(author)\n",
    "    genre = content.find(class_=\"genre\").get_text()\n",
    "    genreorg = genre.split(\"genre :\")[1]\n",
    "    genreArray.append(genreorg)\n",
    "    pages = content.find(class_=\"pages\").get_text()\n",
    "    pagesorg = pages.split(\"pages\")[0]\n",
    "    pagesArray.append(pagesorg)\n",
    "    starRating = content.find(class_=\"rating\").get_text()\n",
    "    starRatingArray.append(starRating)\n",
    "    numOfReviews = content.find(class_=\"#reviews\").get_text()\n",
    "    numOfReviewsorg = numOfReviews.split(\"No_of_reviews\")[1]\n",
    "    reviewsArray.append(numOfReviewsorg)\n",
    "    description = content.find(class_=\"description\").get_text()\n",
    "    descriptionArray.append(description)\n",
    "    isbn = content.find(class_=\"isbn\").get_text()\n",
    "    isbnorg = isbn.split(\"ISBN\")[1]\n",
    "    isbnArray.append(isbnorg)\n",
    "    votes = content.find(class_=\"votes\").get_text()\n",
    "    votesorg = votes.split(\"votes\")[0]\n",
    "    votesArray.append(votesorg)\n",
    "    comments = content.find_all('p', attrs={'class': re.compile(\"^c[0-9]*\")})\n",
    "    if len(comments) == 0 :\n",
    "        commentsArray.append(\"None\")\n",
    "        continue\n",
    "    else:\n",
    "        for comment in comments:\n",
    "                commentsInternalData += comment.get_text()\n",
    "                commentsInternalData += \"|\"        \n",
    "        commentsArray.append(commentsInternalData)\n",
    "        commentsInternalData = \"\"\n",
    "    \n",
    "hackathonDF['Title'] = titleArray\n",
    "hackathonDF['Author'] = authorArray\n",
    "hackathonDF['Genre'] = genreArray\n",
    "hackathonDF['NumberofPages'] = pagesArray\n",
    "hackathonDF['StarRating'] = starRatingArray\n",
    "hackathonDF['NumberofReviews'] = reviewsArray\n",
    "hackathonDF['ShortDescription'] = descriptionArray\n",
    "hackathonDF['ISBNNumber'] = isbnArray\n",
    "hackathonDF['NumbeofVotes'] = votesArray\n",
    "hackathonDF['Comments'] = commentsArray\n",
    "export_csv = hackathonDF.to_csv (r'scraping_hackathon_dataFinal.csv', index = None, header=True)\n",
    "print(\"Data sent To CSV\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Finding out What's the Encoding of Data Whichwe got from Site.\n",
    "\n",
    "with open(\"scraping_hackathon_dataFinal.csv\", 'rb') as rawdata:\n",
    "    result = chardet.detect(rawdata.read(10000))\n",
    "#print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Imported\n"
     ]
    }
   ],
   "source": [
    "# Getting from CSV for Data Exploration using Encoding \"Windows-1252\"\n",
    "\n",
    "scraping_data = pd.read_csv('scraping_hackathon_dataFinal.csv',encoding=\"Windows-1252\")\n",
    "print(\"Data Imported\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Count Done\n"
     ]
    }
   ],
   "source": [
    "# Performing Data Exploration\n",
    "resultClassics = (scraping_data.Genre.values == ' Classics').sum()\n",
    "#print(resultClassics) #252\n",
    "resultSF = (scraping_data.Genre.values == ' Science Fiction').sum()\n",
    "#print(resultSF) #59\n",
    "\n",
    "# Calculating Top 10 Genres\n",
    "resultUniqueGenres = scraping_data.Genre.unique()\n",
    "#print(len(resultUniqueGenres)) #101\n",
    "genreCount = []\n",
    "columnNamesGenre  = ['Genre','Count']\n",
    "genreWithCount = pd.DataFrame(columns = columnNamesGenre)\n",
    "genCount = 0\n",
    "for genre in resultUniqueGenres:\n",
    "    genCount = (scraping_data.Genre.values == genre).sum()\n",
    "    genreCount.append(genCount)\n",
    "genreWithCount.Genre = resultUniqueGenres\n",
    "genreWithCount.Count = genreCount\n",
    "genreWithCount = genreWithCount.sort_values(by=['Count'],ascending=False)\n",
    "genreWithCount = genreWithCount.head(10)\n",
    "#print(genreWithCount)\n",
    "print(\"Count Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Count Done\n"
     ]
    }
   ],
   "source": [
    "# Authors appearing maximum number of genres\n",
    "columnNamesGenreWithAuthor  = ['Author','Genre']\n",
    "genreSet = \"\"\n",
    "authorSet = \"\"\n",
    "genreSetArray = []\n",
    "authorSetArray = []\n",
    "authorWithGenreCount = pd.DataFrame(columns = columnNamesGenreWithAuthor)\n",
    "for index,row in scraping_data.iterrows():\n",
    "    for genreUniqueOne in genreWithCount.Genre:\n",
    "        if row.Genre == genreUniqueOne:\n",
    "            genreSet = genreUniqueOne\n",
    "            authorSet = row.Author\n",
    "            genreSetArray.append(genreSet)\n",
    "            authorSetArray.append(authorSet)\n",
    "authorWithGenreCount.Genre = genreSetArray\n",
    "authorWithGenreCount.Author = authorSetArray\n",
    "#print(authorWithGenreCount)\n",
    "authorWithGenreCount = authorWithGenreCount.sort_values(by=['Genre'])\n",
    "authorWithGenreCount = authorWithGenreCount.drop_duplicates(subset=['Author','Genre'],keep = \"first\")\n",
    "export_csv = authorWithGenreCount.to_csv (r'scraping_hackathon_dataWithAuthorandGenre.csv', index = None, header=True)\n",
    "#print(authorWithGenreCount)\n",
    "print(\"Count Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "# Range of Ratings Under Category Classics Genre\n",
    "resultClassics = scraping_data.loc[scraping_data.Genre == ' Classics']\n",
    "#print(resultClassics.StarRating.max())\n",
    "#print(resultClassics.StarRating.min())\n",
    "# ratings would be between 2.91 and 4.79\n",
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "# Median of Ratings Under Category Classics Fantasy\n",
    "resultFantasy = scraping_data.loc[scraping_data.Genre == ' Fantasy']\n",
    "#print(resultFantasy.StarRating.median())\n",
    "# Median : 4.295\n",
    "\n",
    "# Number of votes required for the high rated book in fantasy\n",
    "resultFantasy = resultFantasy.loc[resultFantasy.StarRating.idxmax]\n",
    "#print(resultFantasy.NumbeofVotes)\n",
    "# votes : 38476\n",
    "\n",
    "#print(resultFantasy.StarRating.max()) #4.78\n",
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "# Standard Deviation of the Rating of books under Business\n",
    "resultBusiness = scraping_data.loc[scraping_data.Genre == ' Business']\n",
    "resultStdDeviation = resultBusiness.StarRating.std()\n",
    "#print(resultStdDeviation)\n",
    "# std : 0.35252665700363184\n",
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2097     Fantasy\n",
      "Name: Genre, dtype: object\n"
     ]
    }
   ],
   "source": [
    "# What is the genre of the highest voted book\n",
    "columnNamesGenreWithAuthor  = ['Genre','MaxVotes']\n",
    "maxVotesWithGenre = pd.DataFrame(columns = columnNamesGenreWithAuthor)\n",
    "maxVotedSet = scraping_data\n",
    "maxVotedSet.NumbeofVotes = maxVotedSet.NumbeofVotes.replace({',': ''}, regex=True)\n",
    "maxVotedSet.NumbeofVotes = pd.to_numeric(maxVotedSet.NumbeofVotes)\n",
    "maxDataset = maxVotedSet.sort_values(['NumbeofVotes'],ascending=False).groupby('Genre').head(10)\n",
    "maxVotesWithGenre.Genre = maxDataset.Genre\n",
    "maxVotesWithGenre.MaxVotes = maxDataset.NumbeofVotes\n",
    "print(maxDataset.Genre.head(1))\n",
    "#Answer : Fantasy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49     None\n",
      "Name: Genre, dtype: object\n"
     ]
    }
   ],
   "source": [
    "# Which genre has more no of authors with above average ratings\n",
    "avg_rating = scraping_data.StarRating.mean()\n",
    "dataWithAboveAvgRating = scraping_data.loc[scraping_data.StarRating > avg_rating].groupby('Genre',as_index=False).count().sort_values(['StarRating'],ascending=False).head(1)\n",
    "print(dataWithAboveAvgRating.Genre)\n",
    "#Answer None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.series.Series'>\n",
      " Fantasy\n"
     ]
    }
   ],
   "source": [
    "# Which Genre has the author with the highest Std. Deviation in number of pages\n",
    "pagesSet = scraping_data\n",
    "pagesSet.NumberofPages = pagesSet.NumberofPages.replace({None : 0.0}, regex=True)\n",
    "pagesSet.NumberofPages = pd.to_numeric(pagesSet.NumberofPages)\n",
    "stdDeviationSet = pagesSet.groupby('Genre').NumberofPages.std(ddof=0).sort_values(ascending=False).head(1)\n",
    "print(stdDeviationSet.index[0])\n",
    "#Answer Fantasy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Among All the available description of books which word in the descriptions has the frequency greater than 6000 and less than 7000\n",
    "wordsArrayCombine = []\n",
    "wordsFramePre = pd.DataFrame(columns=['Word'])\n",
    "wordsFramePost = pd.DataFrame(columns=['Word','Frequency'])\n",
    "for word in scraping_data.ShortDescription:\n",
    "        wordsArrayCombine.append(re.findall('\\w+',word))\n",
    "wordsArrayIndividual  = list(itertools.chain(*wordsArrayCombine))\n",
    "wordsFramePre.Word = wordsArrayIndividual\n",
    "uniqueWords = wordsFramePre.Word.unique()\n",
    "wordsFramePost.Word = uniqueWords\n",
    "count = 0\n",
    "wordUnique = \"\"\n",
    "wordSame = \"\"\n",
    "\n",
    "wordArray=[]\n",
    "frequencyArray=[]\n",
    "#print(wordsFramePost.Word)\n",
    "\n",
    "for wordUnique in wordsFramePost.Word:\n",
    "    for wordSame in wordsFramePre.Word:\n",
    "        if wordUnique == wordSame:\n",
    "            count = count + 1\n",
    "    frequencyArray.append(count)\n",
    "    wordArray.append(wordUnique)\n",
    "    count = 0\n",
    "wordsFramePost.Frequency = frequencyArray\n",
    "wordsFramePost.Word = wordArray\n",
    "#print(wordsFramePost.sort_values(ascending = False,by=['Frequency']).head(1))  \n",
    "#Answer is \"The\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}