Topic Modelling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8d07872b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import docx2txt\n",
    "import PyPDF2\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "np.random.seed(0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "945c2e65",
   "metadata": {},
   "outputs": [],
   "source": [
    "directory = 'C:/Users/arund/OneDrive/Desktop/NLP/Project/Data/Resumes'\n",
    "extension = '.docx'\n",
    "text = ''\n",
    "resume_dict = {}\n",
    "for a_file in os.listdir(directory):\n",
    "    content = []\n",
    "    if a_file.endswith('.docx'):\n",
    "        text = docx2txt.process(directory + '/' + str(a_file))\n",
    "        #print(text, '\\n\\n------------------------------------------------------------------------------------------\\n\\n')\n",
    "        resume_dict[str(a_file[:-5])] = text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2e8ef9ef",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Resume</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Names</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ABHAY RAJ DWIVEDI</th>\n",
       "      <td>ABHAY RAJ DWIVEDIFeatured Resume  \\n\\n \\n\\nSof...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Abhishek Sharma</th>\n",
       "      <td>Abhishek SharmaFeatured Resume  \\n\\n \\n\\nDevel...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ADRYJA GHOSH</th>\n",
       "      <td>ADRYJA GHOSHFeatured Resume  \\n\\n \\n\\nEnglish ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AJAY CHINNI</th>\n",
       "      <td>AJAY CHINNIFeatured Resume  \\n\\n \\n\\nSeeking r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ALANKRIT NIRJHAR</th>\n",
       "      <td>ALANKRIT NIRJHARPremium CV Featured Resume  \\n...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                              Resume\n",
       "Names                                                               \n",
       "ABHAY RAJ DWIVEDI  ABHAY RAJ DWIVEDIFeatured Resume  \\n\\n \\n\\nSof...\n",
       "Abhishek Sharma    Abhishek SharmaFeatured Resume  \\n\\n \\n\\nDevel...\n",
       "ADRYJA GHOSH       ADRYJA GHOSHFeatured Resume  \\n\\n \\n\\nEnglish ...\n",
       "AJAY CHINNI        AJAY CHINNIFeatured Resume  \\n\\n \\n\\nSeeking r...\n",
       "ALANKRIT NIRJHAR   ALANKRIT NIRJHARPremium CV Featured Resume  \\n..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resume_data = pd.DataFrame.from_dict(resume_dict, orient  = 'index', columns = ['Resume'])\n",
    "resume_data.index.name = 'Names'\n",
    "resume_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f8f9e147",
   "metadata": {},
   "outputs": [],
   "source": [
    "directory = 'C:/Users/arund/OneDrive/Desktop/NLP/Project/Data/JobDesc'\n",
    "extension = '.docx'\n",
    "text = ''\n",
    "job_desc_dict = {}\n",
    "for a_file in os.listdir(directory):\n",
    "    content = []\n",
    "    if a_file.endswith('.docx'):\n",
    "        text = docx2txt.process(directory + '/' + str(a_file))\n",
    "        #print(text, '\\n\\n------------------------------------------------------------------------------------------\\n\\n')\n",
    "        job_desc_dict[str(a_file[:-5])] = text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "564123f9",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Job_Description</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Profile</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Backend Developer</th>\n",
       "      <td>Backend Developer - Python/Django\\n\\n\\n\\nWe ar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Billing cum Logistics Manager</th>\n",
       "      <td>Billing cum Logistics Manager\\n\\nJob Summary\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Data Scientist</th>\n",
       "      <td>Data Scientist/ ML Engineer\\n\\nGame Change Sol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Director of Engineering</th>\n",
       "      <td>Director of Engineering, Google CloudDirector ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Global Industry Content Manager</th>\n",
       "      <td>Global Industry Content Manager\\n\\nMinimum qua...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                   Job_Description\n",
       "Profile                                                                           \n",
       "Backend Developer                Backend Developer - Python/Django\\n\\n\\n\\nWe ar...\n",
       "Billing cum Logistics Manager    Billing cum Logistics Manager\\n\\nJob Summary\\n...\n",
       "Data Scientist                   Data Scientist/ ML Engineer\\n\\nGame Change Sol...\n",
       "Director of Engineering          Director of Engineering, Google CloudDirector ...\n",
       "Global Industry Content Manager  Global Industry Content Manager\\n\\nMinimum qua..."
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "job_desc_data = pd.DataFrame.from_dict(job_desc_dict, orient  = 'index', columns = ['Job_Description'])\n",
    "job_desc_data.index.name = 'Profile'\n",
    "job_desc_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df1b46b5",
   "metadata": {},
   "source": [
    "## NLTK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c320690f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "showing info https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/index.xml\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6337c91",
   "metadata": {},
   "source": [
    "## Cleaning Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c51d9c98",
   "metadata": {},
   "outputs": [],
   "source": [
    "from string import punctuation\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.tag import pos_tag\n",
    "import nltk.stem\n",
    "from nltk.stem import LancasterStemmer, SnowballStemmer\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "import re\n",
    "from string import punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "17a18247",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Checks for punctuations and stopwords, converts to lower case, performs tokenization, POS tagging and lemmatization and \n",
    "also removes irrelevant data''' \n",
    "\n",
    "def clean_data(text): \n",
    "    stop_words = stopwords.words('english')\n",
    "    tokens = word_tokenize(text)\n",
    "    cleaned_tokens = []\n",
    "\n",
    "    for tok, tag in pos_tag(tokens):\n",
    "        tok = re.sub('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+#]|[!*\\(\\)]|'\\\n",
    "                       '(?:%[0-9a-fA-F][0-9a-fA-F]))+','', tok)\n",
    "        tok = re.sub(\"(@[A-Za-z0-9_]+)\",\"\", tok)\n",
    "\n",
    "        if tag.startswith(\"NN\"):\n",
    "            pos = 'n'\n",
    "        elif tag.startswith('VB'):\n",
    "            pos = 'v'\n",
    "        else:\n",
    "            pos = 'a'\n",
    "\n",
    "        lemmatizer = WordNetLemmatizer()\n",
    "        tok = lemmatizer.lemmatize(tok, pos)\n",
    "\n",
    "        if len(tok) > 0 and tok not in punctuation and tok.lower() not in stop_words:\n",
    "            cleaned_tokens.append(tok.lower())\n",
    "    return cleaned_tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85e46a90",
   "metadata": {},
   "source": [
    "## Topic Modelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c5c6d69d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Resume</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Names</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MADHU H</th>\n",
       "      <td>MADHU H.KPremium CV Featured Resume  \\n\\n \\n\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Manvi Singh</th>\n",
       "      <td>Manvi SinghFeatured Resume  \\n\\n \\n\\nHR Manage...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MONICA CHOPRA</th>\n",
       "      <td>MONICA CHOPRAFeatured Resume  \\n\\n \\n\\nForeign...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RISHABH SHARMA</th>\n",
       "      <td>RISHABH SHARMAFeatured Resume  \\n\\n \\n\\nJunior...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Seshadri Sastry Kunapuli</th>\n",
       "      <td>Seshadri Sastry Kunapuli     \\n\\n \\n\\nManager ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                     Resume\n",
       "Names                                                                      \n",
       "MADHU H                   MADHU H.KPremium CV Featured Resume  \\n\\n \\n\\n...\n",
       "Manvi Singh               Manvi SinghFeatured Resume  \\n\\n \\n\\nHR Manage...\n",
       "MONICA CHOPRA             MONICA CHOPRAFeatured Resume  \\n\\n \\n\\nForeign...\n",
       "RISHABH SHARMA            RISHABH SHARMAFeatured Resume  \\n\\n \\n\\nJunior...\n",
       "Seshadri Sastry Kunapuli  Seshadri Sastry Kunapuli     \\n\\n \\n\\nManager ..."
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc = resume_data[['Resume']]\n",
    "doc.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "539b2989",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "import re\n",
    "from gensim.utils import simple_preprocess\n",
    "from gensim.parsing.preprocessing import STOPWORDS\n",
    "from nltk.stem import WordNetLemmatizer, SnowballStemmer, LancasterStemmer\n",
    "from nltk.stem.porter import *\n",
    "from string import punctuation\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.tag import pos_tag\n",
    "from string import punctuation\n",
    "np.random.seed(2018)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ce5bd84a",
   "metadata": {},
   "outputs": [],
   "source": [
    "stemmer = SnowballStemmer('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ed309ef7",
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_docs = doc['Resume'].map(clean_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e6db1864",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Names\n",
       "ABHAY RAJ DWIVEDI       [abhay, raj, dwivedifeatured, resume, software...\n",
       "Abhishek Sharma         [abhishek, sharmafeatured, resume, developer, ...\n",
       "ADRYJA GHOSH            [adryja, ghoshfeatured, resume, english, teach...\n",
       "AJAY CHINNI             [ajay, chinnifeatured, resume, seeking, role, ...\n",
       "ALANKRIT NIRJHAR        [alankrit, nirjharpremium, cv, featured, resum...\n",
       "Amarpreet Singh         [amarpreet, singhpremium, cv, featured, resume...\n",
       "AMITABHA                [amitabha, ghoshpremium, cv, featured, resume,...\n",
       "Arun Kumar              [arun, kumar, ufeatured, resume, hr, manager, ...\n",
       "Ashish Singh            [ashish, singh, thakurfeatured, resume, hr, ma...\n",
       "Ashwani Kumar Rajput    [ashwani, kumar, rajputpremium, cv, featured, ...\n",
       "Name: Resume, dtype: object"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processed_docs[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ae0f01bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionary = gensim.corpora.Dictionary(processed_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3ea51e1b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 'india\n",
      "1 's\n",
      "2 'world\n",
      "3 .net\n",
      "4 1\n",
      "5 1+\n",
      "6 15\n",
      "7 17-sep-20\n",
      "8 17-sep-20last\n",
      "9 2018\n",
      "10 2019\n"
     ]
    }
   ],
   "source": [
    "count = 0\n",
    "for k, v in dictionary.iteritems():\n",
    "    print(k, v)\n",
    "    count += 1\n",
    "    if count > 10:\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13aff54d",
   "metadata": {},
   "source": [
    "## Bag of Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e34518bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(9, 5),\n",
       " (10, 2),\n",
       " (12, 2),\n",
       " (17, 1),\n",
       " (20, 1),\n",
       " (23, 3),\n",
       " (25, 22),\n",
       " (27, 1),\n",
       " (33, 1),\n",
       " (34, 2),\n",
       " (35, 6),\n",
       " (36, 1),\n",
       " (38, 1),\n",
       " (40, 3),\n",
       " (43, 1),\n",
       " (44, 4),\n",
       " (46, 2),\n",
       " (48, 1),\n",
       " (52, 2),\n",
       " (54, 4),\n",
       " (55, 3),\n",
       " (56, 2),\n",
       " (57, 1),\n",
       " (61, 1),\n",
       " (64, 2),\n",
       " (65, 1),\n",
       " (66, 1),\n",
       " (67, 3),\n",
       " (70, 1),\n",
       " (71, 7),\n",
       " (72, 7),\n",
       " (76, 3),\n",
       " (78, 1),\n",
       " (79, 2),\n",
       " (80, 3),\n",
       " (83, 3),\n",
       " (84, 1),\n",
       " (85, 1),\n",
       " (87, 1),\n",
       " (88, 6),\n",
       " (92, 3),\n",
       " (94, 1),\n",
       " (95, 2),\n",
       " (98, 1),\n",
       " (99, 1),\n",
       " (100, 1),\n",
       " (103, 2),\n",
       " (107, 1),\n",
       " (114, 1),\n",
       " (116, 6),\n",
       " (120, 2),\n",
       " (121, 1),\n",
       " (122, 1),\n",
       " (123, 1),\n",
       " (124, 1),\n",
       " (125, 1),\n",
       " (126, 1),\n",
       " (129, 6),\n",
       " (131, 2),\n",
       " (135, 4),\n",
       " (136, 13),\n",
       " (141, 1),\n",
       " (143, 1),\n",
       " (145, 1),\n",
       " (146, 2),\n",
       " (147, 3),\n",
       " (150, 1),\n",
       " (151, 3),\n",
       " (154, 1),\n",
       " (156, 1),\n",
       " (157, 3),\n",
       " (158, 6),\n",
       " (162, 1),\n",
       " (163, 1),\n",
       " (165, 5),\n",
       " (167, 1),\n",
       " (169, 1),\n",
       " (172, 1),\n",
       " (173, 12),\n",
       " (174, 2),\n",
       " (175, 2),\n",
       " (181, 3),\n",
       " (183, 1),\n",
       " (184, 4),\n",
       " (185, 1),\n",
       " (186, 2),\n",
       " (191, 1),\n",
       " (193, 1),\n",
       " (195, 2),\n",
       " (196, 4),\n",
       " (197, 1),\n",
       " (199, 1),\n",
       " (202, 1),\n",
       " (203, 3),\n",
       " (204, 10),\n",
       " (207, 1),\n",
       " (208, 2),\n",
       " (211, 1),\n",
       " (215, 1),\n",
       " (218, 1),\n",
       " (221, 1),\n",
       " (223, 1),\n",
       " (224, 3),\n",
       " (225, 3),\n",
       " (226, 4),\n",
       " (227, 1),\n",
       " (229, 1),\n",
       " (230, 1),\n",
       " (231, 14),\n",
       " (232, 1),\n",
       " (235, 1),\n",
       " (236, 1),\n",
       " (238, 10),\n",
       " (240, 1),\n",
       " (241, 2),\n",
       " (242, 1),\n",
       " (243, 3),\n",
       " (249, 1),\n",
       " (266, 2),\n",
       " (291, 3),\n",
       " (298, 2),\n",
       " (312, 1),\n",
       " (317, 1),\n",
       " (425, 2),\n",
       " (451, 3),\n",
       " (454, 8),\n",
       " (455, 1),\n",
       " (471, 1),\n",
       " (480, 1),\n",
       " (506, 3),\n",
       " (515, 8),\n",
       " (519, 2),\n",
       " (589, 2),\n",
       " (602, 2),\n",
       " (649, 1),\n",
       " (659, 2),\n",
       " (689, 2),\n",
       " (692, 3),\n",
       " (700, 7),\n",
       " (720, 1),\n",
       " (742, 3),\n",
       " (754, 1),\n",
       " (759, 3),\n",
       " (798, 1),\n",
       " (823, 5),\n",
       " (829, 1),\n",
       " (835, 1),\n",
       " (836, 1),\n",
       " (862, 1),\n",
       " (871, 1),\n",
       " (894, 1),\n",
       " (919, 4),\n",
       " (937, 1),\n",
       " (965, 1),\n",
       " (994, 1),\n",
       " (1006, 4),\n",
       " (1025, 1),\n",
       " (1042, 8),\n",
       " (1052, 2),\n",
       " (1118, 1),\n",
       " (1167, 3),\n",
       " (1177, 2),\n",
       " (1214, 1),\n",
       " (1225, 1),\n",
       " (1227, 2),\n",
       " (1228, 1),\n",
       " (1241, 2),\n",
       " (1299, 3),\n",
       " (1300, 5),\n",
       " (1337, 5),\n",
       " (1349, 2),\n",
       " (1351, 1),\n",
       " (1393, 2),\n",
       " (1397, 1),\n",
       " (1398, 3),\n",
       " (1410, 2),\n",
       " (1414, 1),\n",
       " (1431, 1),\n",
       " (1443, 2),\n",
       " (1446, 1),\n",
       " (1460, 1),\n",
       " (1492, 1),\n",
       " (1493, 1),\n",
       " (1494, 1),\n",
       " (1495, 2),\n",
       " (1496, 1),\n",
       " (1497, 1),\n",
       " (1498, 4),\n",
       " (1499, 1),\n",
       " (1500, 1),\n",
       " (1501, 1),\n",
       " (1502, 1),\n",
       " (1503, 1),\n",
       " (1504, 3),\n",
       " (1505, 2),\n",
       " (1506, 2),\n",
       " (1507, 1),\n",
       " (1508, 1),\n",
       " (1509, 1),\n",
       " (1510, 3),\n",
       " (1511, 5),\n",
       " (1512, 1),\n",
       " (1513, 1),\n",
       " (1514, 3),\n",
       " (1515, 1),\n",
       " (1516, 1),\n",
       " (1517, 1),\n",
       " (1518, 1),\n",
       " (1519, 1),\n",
       " (1520, 1),\n",
       " (1521, 1),\n",
       " (1522, 5),\n",
       " (1523, 1),\n",
       " (1524, 3),\n",
       " (1525, 1),\n",
       " (1526, 2),\n",
       " (1527, 3),\n",
       " (1528, 1),\n",
       " (1529, 2),\n",
       " (1530, 2),\n",
       " (1531, 11),\n",
       " (1532, 2),\n",
       " (1533, 1),\n",
       " (1534, 1),\n",
       " (1535, 4),\n",
       " (1536, 1),\n",
       " (1537, 1),\n",
       " (1538, 1),\n",
       " (1539, 5),\n",
       " (1540, 1),\n",
       " (1541, 1),\n",
       " (1542, 1),\n",
       " (1543, 1),\n",
       " (1544, 2),\n",
       " (1545, 2),\n",
       " (1546, 1),\n",
       " (1547, 3),\n",
       " (1548, 1),\n",
       " (1549, 3),\n",
       " (1550, 1),\n",
       " (1551, 6)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bow_corpus = [dictionary.doc2bow(doc) for doc in processed_docs]\n",
    "bow_corpus[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b2427bc8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word 9 (\"2018\") appears 5 time.\n",
      "Word 10 (\"2019\") appears 2 time.\n",
      "Word 12 (\"3\") appears 2 time.\n",
      "Word 17 (\"active\") appears 1 time.\n",
      "Word 20 (\"agile\") appears 1 time.\n",
      "Word 23 (\"analysis\") appears 3 time.\n",
      "Word 25 (\"application\") appears 22 time.\n",
      "Word 27 (\"area\") appears 1 time.\n",
      "Word 33 (\"aug\") appears 1 time.\n",
      "Word 34 (\"b.tech/b.e\") appears 2 time.\n",
      "Word 35 (\"back\") appears 6 time.\n",
      "Word 36 (\"bangalore\") appears 1 time.\n",
      "Word 38 (\"bengaluru\") appears 1 time.\n",
      "Word 40 (\"business\") appears 3 time.\n",
      "Word 43 (\"case\") appears 1 time.\n",
      "Word 44 (\"client\") appears 4 time.\n",
      "Word 46 (\"code\") appears 2 time.\n",
      "Word 48 (\"company\") appears 1 time.\n",
      "Word 52 (\"create\") appears 2 time.\n",
      "Word 54 (\"css\") appears 4 time.\n",
      "Word 55 (\"current\") appears 3 time.\n",
      "Word 56 (\"data\") appears 2 time.\n",
      "Word 57 (\"date\") appears 1 time.\n",
      "Word 61 (\"degree\") appears 1 time.\n",
      "Word 64 (\"design\") appears 2 time.\n",
      "Word 65 (\"designation\") appears 1 time.\n",
      "Word 66 (\"detail\") appears 1 time.\n",
      "Word 67 (\"details\") appears 3 time.\n",
      "Word 70 (\"developed\") appears 1 time.\n",
      "Word 71 (\"developer\") appears 7 time.\n",
      "Word 72 (\"development\") appears 7 time.\n",
      "Word 76 (\"duration\") appears 3 time.\n",
      "Word 78 (\"e-mail\") appears 1 time.\n",
      "Word 79 (\"education\") appears 2 time.\n",
      "Word 80 (\"email\") appears 3 time.\n",
      "Word 83 (\"employment\") appears 3 time.\n",
      "Word 84 (\"engineering\") appears 1 time.\n",
      "Word 85 (\"english\") appears 1 time.\n",
      "Word 87 (\"exist\") appears 1 time.\n",
      "Word 88 (\"experience\") appears 6 time.\n",
      "Word 92 (\"full\") appears 3 time.\n",
      "Word 94 (\"functional\") appears 1 time.\n",
      "Word 95 (\"gathering\") appears 2 time.\n",
      "Word 98 (\"highest\") appears 1 time.\n",
      "Word 99 (\"hindi\") appears 1 time.\n",
      "Word 100 (\"html\") appears 1 time.\n",
      "Word 103 (\"id\") appears 2 time.\n",
      "Word 107 (\"industry\") appears 1 time.\n",
      "Word 114 (\"it-software/software\") appears 1 time.\n",
      "Word 116 (\"javascript\") appears 6 time.\n",
      "Word 120 (\"jul\") appears 2 time.\n",
      "Word 121 (\"jump\") appears 1 time.\n",
      "Word 122 (\"key\") appears 1 time.\n",
      "Word 123 (\"known\") appears 1 time.\n",
      "Word 124 (\"language\") appears 1 time.\n",
      "Word 125 (\"languages\") appears 1 time.\n",
      "Word 126 (\"last\") appears 1 time.\n",
      "Word 129 (\"like\") appears 6 time.\n",
      "Word 131 (\"location\") appears 2 time.\n",
      "Word 135 (\"maintenance\") appears 4 time.\n",
      "Word 136 (\"management\") appears 13 time.\n",
      "Word 141 (\"marital\") appears 1 time.\n",
      "Word 143 (\"methodology\") appears 1 time.\n",
      "Word 145 (\"modified\") appears 1 time.\n",
      "Word 146 (\"module\") appears 2 time.\n",
      "Word 147 (\"month\") appears 3 time.\n",
      "Word 150 (\"name\") appears 1 time.\n",
      "Word 151 (\"nature\") appears 3 time.\n",
      "Word 154 (\"notice\") appears 1 time.\n",
      "Word 156 (\"number\") appears 1 time.\n",
      "Word 157 (\"offsite\") appears 3 time.\n",
      "Word 158 (\"onsite\") appears 6 time.\n",
      "Word 162 (\"period\") appears 1 time.\n",
      "Word 163 (\"phone\") appears 1 time.\n",
      "Word 165 (\"portal\") appears 5 time.\n",
      "Word 167 (\"pref\") appears 1 time.\n",
      "Word 169 (\"proficiency\") appears 1 time.\n",
      "Word 172 (\"programming\") appears 1 time.\n",
      "Word 173 (\"project\") appears 12 time.\n",
      "Word 174 (\"projects\") appears 2 time.\n",
      "Word 175 (\"provide\") appears 2 time.\n",
      "Word 181 (\"read\") appears 3 time.\n",
      "Word 183 (\"request\") appears 1 time.\n",
      "Word 184 (\"requirement\") appears 4 time.\n",
      "Word 185 (\"resume\") appears 1 time.\n",
      "Word 186 (\"role\") appears 2 time.\n",
      "Word 191 (\"sectionwork\") appears 1 time.\n",
      "Word 193 (\"seek\") appears 1 time.\n",
      "Word 195 (\"server\") appears 2 time.\n",
      "Word 196 (\"service\") appears 4 time.\n",
      "Word 197 (\"services\") appears 1 time.\n",
      "Word 199 (\"single/unmarried\") appears 1 time.\n",
      "Word 202 (\"skill\") appears 1 time.\n",
      "Word 203 (\"skills\") appears 3 time.\n",
      "Word 204 (\"software\") appears 10 time.\n",
      "Word 207 (\"speak\") appears 1 time.\n",
      "Word 208 (\"specification\") appears 2 time.\n",
      "Word 211 (\"status\") appears 1 time.\n",
      "Word 215 (\"summary\") appears 1 time.\n",
      "Word 218 (\"take\") appears 1 time.\n",
      "Word 221 (\"test\") appears 1 time.\n",
      "Word 223 (\"till\") appears 1 time.\n",
      "Word 224 (\"time\") appears 3 time.\n",
      "Word 225 (\"title\") appears 3 time.\n",
      "Word 226 (\"top\") appears 4 time.\n",
      "Word 227 (\"total\") appears 1 time.\n",
      "Word 229 (\"ug\") appears 1 time.\n",
      "Word 230 (\"unit\") appears 1 time.\n",
      "Word 231 (\"use\") appears 14 time.\n",
      "Word 232 (\"used\") appears 1 time.\n",
      "Word 235 (\"verified\") appears 1 time.\n",
      "Word 236 (\"version\") appears 1 time.\n",
      "Word 238 (\"web\") appears 10 time.\n",
      "Word 240 (\"work\") appears 1 time.\n",
      "Word 241 (\"worked\") appears 2 time.\n",
      "Word 242 (\"write\") appears 1 time.\n",
      "Word 243 (\"year\") appears 3 time.\n",
      "Word 249 (\"2017\") appears 1 time.\n",
      "Word 266 (\"database\") appears 2 time.\n",
      "Word 291 (\"nov\") appears 3 time.\n",
      "Word 298 (\"pune\") appears 2 time.\n",
      "Word 312 (\"0\") appears 1 time.\n",
      "Word 317 (\"4\") appears 1 time.\n",
      "Word 425 (\"designed\") appears 2 time.\n",
      "Word 451 (\"interface\") appears 3 time.\n",
      "Word 454 (\"java\") appears 8 time.\n",
      "Word 455 (\"jun\") appears 1 time.\n",
      "Word 471 (\"months\") appears 1 time.\n",
      "Word 480 (\"patch\") appears 1 time.\n",
      "Word 506 (\"technologies\") appears 3 time.\n",
      "Word 515 (\"user\") appears 8 time.\n",
      "Word 519 (\"via\") appears 2 time.\n",
      "Word 589 (\"framework\") appears 2 time.\n",
      "Word 602 (\"layer\") appears 2 time.\n",
      "Word 649 (\"show\") appears 1 time.\n",
      "Word 659 (\"tool\") appears 2 time.\n",
      "Word 689 (\"also\") appears 2 time.\n",
      "Word 692 (\"apache\") appears 3 time.\n",
      "Word 700 (\"base\") appears 7 time.\n",
      "Word 720 (\"designing\") appears 1 time.\n",
      "Word 742 (\"git\") appears 3 time.\n",
      "Word 754 (\"include\") appears 1 time.\n",
      "Word 759 (\"jenkins\") appears 3 time.\n",
      "Word 798 (\"prepare\") appears 1 time.\n",
      "Word 823 (\"spring\") appears 5 time.\n",
      "Word 829 (\"testing\") appears 1 time.\n",
      "Word 835 (\"understanding\") appears 1 time.\n",
      "Word 836 (\"various\") appears 1 time.\n",
      "Word 862 (\"add\") appears 1 time.\n",
      "Word 871 (\"bengali\") appears 1 time.\n",
      "Word 894 (\"group\") appears 1 time.\n",
      "Word 919 (\"proficient\") appears 4 time.\n",
      "Word 937 (\"structure\") appears 1 time.\n",
      "Word 965 (\"approval\") appears 1 time.\n",
      "Word 994 (\"chennai\") appears 1 time.\n",
      "Word 1006 (\"compliance\") appears 4 time.\n",
      "Word 1025 (\"critical\") appears 1 time.\n",
      "Word 1042 (\"document\") appears 8 time.\n",
      "Word 1052 (\"end\") appears 2 time.\n",
      "Word 1118 (\"involve\") appears 1 time.\n",
      "Word 1167 (\"oracle\") appears 3 time.\n",
      "Word 1177 (\"part\") appears 2 time.\n",
      "Word 1214 (\"providing\") appears 1 time.\n",
      "Word 1225 (\"register\") appears 1 time.\n",
      "Word 1227 (\"relate\") appears 2 time.\n",
      "Word 1228 (\"related\") appears 1 time.\n",
      "Word 1241 (\"responsibility\") appears 2 time.\n",
      "Word 1299 (\"2015\") appears 3 time.\n",
      "Word 1300 (\"6\") appears 5 time.\n",
      "Word 1337 (\"delivery\") appears 5 time.\n",
      "Word 1349 (\"erp\") appears 2 time.\n",
      "Word 1351 (\"feb\") appears 1 time.\n",
      "Word 1393 (\"quality\") appears 2 time.\n",
      "Word 1397 (\"responsibilities\") appears 1 time.\n",
      "Word 1398 (\"roles\") appears 3 time.\n",
      "Word 1410 (\"supply\") appears 2 time.\n",
      "Word 1414 (\"update\") appears 1 time.\n",
      "Word 1431 (\"apr\") appears 1 time.\n",
      "Word 1443 (\"continuous\") appears 2 time.\n",
      "Word 1446 (\"device\") appears 1 time.\n",
      "Word 1460 (\"jquery\") appears 1 time.\n",
      "Word 1492 (\"04f71ddb0a1ab33c85last\") appears 1 time.\n",
      "Word 1493 (\"15-sep-20\") appears 1 time.\n",
      "Word 1494 (\"15-sep-20last\") appears 1 time.\n",
      "Word 1495 (\"7\") appears 2 time.\n",
      "Word 1496 (\"8\") appears 1 time.\n",
      "Word 1497 (\"allow\") appears 1 time.\n",
      "Word 1498 (\"angular\") appears 4 time.\n",
      "Word 1499 (\"angularjs\") appears 1 time.\n",
      "Word 1500 (\"avik\") appears 1 time.\n",
      "Word 1501 (\"backend\") appears 1 time.\n",
      "Word 1502 (\"bhattacharyafeatured\") appears 1 time.\n",
      "Word 1503 (\"bill\") appears 1 time.\n",
      "Word 1504 (\"cassandra\") appears 3 time.\n",
      "Word 1505 (\"chain\") appears 2 time.\n",
      "Word 1506 (\"coding\") appears 2 time.\n",
      "Word 1507 (\"coimbatore\") appears 1 time.\n",
      "Word 1508 (\"consist\") appears 1 time.\n",
      "Word 1509 (\"contact\") appears 1 time.\n",
      "Word 1510 (\"conversion\") appears 3 time.\n",
      "Word 1511 (\"creation\") appears 5 time.\n",
      "Word 1512 (\"davies\") appears 1 time.\n",
      "Word 1513 (\"depth\") appears 1 time.\n",
      "Word 1514 (\"desktop\") appears 3 time.\n",
      "Word 1515 (\"developer-it\") appears 1 time.\n",
      "Word 1516 (\"display\") appears 1 time.\n",
      "Word 1517 (\"draft\") appears 1 time.\n",
      "Word 1518 (\"enhancement\") appears 1 time.\n",
      "Word 1519 (\"execution\") appears 1 time.\n",
      "Word 1520 (\"finalize\") appears 1 time.\n",
      "Word 1521 (\"flow\") appears 1 time.\n",
      "Word 1522 (\"functionality\") appears 5 time.\n",
      "Word 1523 (\"info\") appears 1 time.\n",
      "Word 1524 (\"infoview\") appears 3 time.\n",
      "Word 1525 (\"institute\") appears 1 time.\n",
      "Word 1526 (\"integration\") appears 2 time.\n",
      "Word 1527 (\"ivtl\") appears 3 time.\n",
      "Word 1528 (\"manual\") appears 1 time.\n",
      "Word 1529 (\"mechanical\") appears 2 time.\n",
      "Word 1530 (\"option\") appears 2 time.\n",
      "Word 1531 (\"order\") appears 11 time.\n",
      "Word 1532 (\"previous\") appears 2 time.\n",
      "Word 1533 (\"recreation\") appears 1 time.\n",
      "Word 1534 (\"registerd\") appears 1 time.\n",
      "Word 1535 (\"repository\") appears 4 time.\n",
      "Word 1536 (\"scm\") appears 1 time.\n",
      "Word 1537 (\"search\") appears 1 time.\n",
      "Word 1538 (\"select\") appears 1 time.\n",
      "Word 1539 (\"send\") appears 5 time.\n",
      "Word 1540 (\"subsystem\") appears 1 time.\n",
      "Word 1541 (\"suggestion\") appears 1 time.\n",
      "Word 1542 (\"supplier\") appears 1 time.\n",
      "Word 1543 (\"support\") appears 1 time.\n",
      "Word 1544 (\"syntel\") appears 2 time.\n",
      "Word 1545 (\"table\") appears 2 time.\n",
      "Word 1546 (\"tamil\") appears 1 time.\n",
      "Word 1547 (\"tomcat\") appears 3 time.\n",
      "Word 1548 (\"unread\") appears 1 time.\n",
      "Word 1549 (\"upload\") appears 3 time.\n",
      "Word 1550 (\"waterfall\") appears 1 time.\n",
      "Word 1551 (\"xml\") appears 6 time.\n"
     ]
    }
   ],
   "source": [
    "bow_doc_10 = bow_corpus[10]\n",
    "\n",
    "for i in range(len(bow_doc_10)):\n",
    "    print(\"Word {} (\\\"{}\\\") appears {} time.\".format(bow_doc_10[i][0], \n",
    "                                                     dictionary[bow_doc_10[i][0]], \n",
    "                                                     bow_doc_10[i][1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b745fd0",
   "metadata": {},
   "source": [
    "## TF-IDF Vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6b152502",
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim import corpora, models\n",
    "\n",
    "tfidf = models.TfidfModel(bow_corpus)\n",
    "corpus_tfidf = tfidf[bow_corpus]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "bfa3bb0e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 0.05854746539431067),\n",
      " (1, 0.03210010617032598),\n",
      " (2, 0.05854746539431067),\n",
      " (3, 0.02737767107618461),\n",
      " (4, 0.010298131250104106),\n",
      " (5, 0.05854746539431067),\n",
      " (6, 0.013463642263235204),\n",
      " (7, 0.021109838395992522),\n",
      " (8, 0.01921556409829409),\n",
      " (9, 0.032337174434220455),\n",
      " (10, 0.017045137629016126),\n",
      " (11, 0.011792773917121576),\n",
      " (12, 0.02939596865833253),\n",
      " (13, 0.02904853942229823),\n",
      " (14, 0.05854746539431067),\n",
      " (15, 0.05854746539431067),\n",
      " (16, 0.035716003733171284),\n",
      " (18, 0.3007379776467335),\n",
      " (19, 0.05854746539431067),\n",
      " (20, 0.02737767107618461),\n",
      " (21, 0.07776302949260476),\n",
      " (22, 0.07776302949260476),\n",
      " (23, 0.032337174434220455),\n",
      " (24, 0.04906191930634081),\n",
      " (25, 0.0585843119882469),\n",
      " (26, 0.03888151474630238),\n",
      " (28, 0.16307632311067544),\n",
      " (29, 0.048714490070306524),\n",
      " (30, 0.024530959653170405),\n",
      " (31, 0.04296256823524764),\n",
      " (32, 0.09742898014061305),\n",
      " (33, 0.015357916560933637),\n",
      " (34, 0.017892124988214752),\n",
      " (35, 0.0009226860036190692),\n",
      " (36, 0.08025026542581494),\n",
      " (37, 0.025883028409167137),\n",
      " (38, 0.03210010617032598),\n",
      " (39, 0.05854746539431067),\n",
      " (40, 0.01077905814474015),\n",
      " (41, 0.045107452287953514),\n",
      " (42, 0.07776302949260476),\n",
      " (43, 0.02329661758723935),\n",
      " (44, 0.0215581162894803),\n",
      " (45, 0.03888151474630238),\n",
      " (46, 0.035089391504360926),\n",
      " (47, 0.08592513647049528),\n",
      " (48, 0.000692014502714302),\n",
      " (49, 0.175642396182932),\n",
      " (50, 0.048714490070306524),\n",
      " (51, 0.05854746539431067),\n",
      " (52, 0.016777698976461607),\n",
      " (53, 0.035716003733171284),\n",
      " (54, 0.09607782049147044),\n",
      " (56, 0.009382588774289944),\n",
      " (57, 0.0004651559260999608),\n",
      " (58, 0.015357916560933637),\n",
      " (59, 0.017544695752180463),\n",
      " (60, 0.05854746539431067),\n",
      " (62, 0.01232815763386031),\n",
      " (63, 0.04906191930634081),\n",
      " (64, 0.01288454207203189),\n",
      " (65, 0.0004651559260999608),\n",
      " (66, 0.02737767107618461),\n",
      " (67, 0.0465932351744787),\n",
      " (68, 0.017892124988214752),\n",
      " (69, 0.05854746539431067),\n",
      " (70, 0.022161132957864454),\n",
      " (71, 0.059657981701556444),\n",
      " (72, 0.0057755509919369345),\n",
      " (73, 0.175642396182932),\n",
      " (74, 0.04296256823524764),\n",
      " (75, 0.04296256823524764),\n",
      " (76, 0.04906191930634081),\n",
      " (77, 0.048714490070306524),\n",
      " (78, 0.048714490070306524),\n",
      " (81, 0.08864453183145782),\n",
      " (82, 0.03888151474630238),\n",
      " (83, 0.03843112819658818),\n",
      " (84, 0.011792773917121576),\n",
      " (85, 0.005191137959811534),\n",
      " (86, 0.016777698976461607),\n",
      " (87, 0.020131106574108253),\n",
      " (89, 0.030942813719996666),\n",
      " (90, 0.035716003733171284),\n",
      " (91, 0.05854746539431067),\n",
      " (92, 0.02939596865833253),\n",
      " (93, 0.05854746539431067),\n",
      " (95, 0.04409395298749879),\n",
      " (96, 0.10714801119951384),\n",
      " (97, 0.02737767107618461),\n",
      " (99, 0.008111352058857277),\n",
      " (100, 0.04039092678970562),\n",
      " (101, 0.035716003733171284),\n",
      " (102, 0.03888151474630238),\n",
      " (104, 0.048714490070306524),\n",
      " (105, 0.016777698976461607),\n",
      " (106, 0.03355539795292321),\n",
      " (108, 0.030715833121867275),\n",
      " (109, 0.3007379776467335),\n",
      " (110, 0.05854746539431067),\n",
      " (111, 0.0465932351744787),\n",
      " (112, 0.024530959653170405),\n",
      " (113, 0.02737767107618461),\n",
      " (114, 0.006576235798801425),\n",
      " (115, 0.46837972315448534),\n",
      " (116, 0.09214749936560183),\n",
      " (117, 0.06648339887359336),\n",
      " (118, 0.03888151474630238),\n",
      " (119, 0.09742898014061305),\n",
      " (120, 0.014697984329166264),\n",
      " (123, 0.005524941236929493),\n",
      " (124, 0.005191137959811534),\n",
      " (125, 0.005866788735832111),\n",
      " (126, 0.002770646979449178),\n",
      " (127, 0.030715833121867275),\n",
      " (128, 0.015357916560933637),\n",
      " (129, 0.01605005308516299),\n",
      " (130, 0.021109838395992522),\n",
      " (132, 0.028134784607596275),\n",
      " (133, 0.05854746539431067),\n",
      " (134, 0.03888151474630238),\n",
      " (135, 0.006944723652457461),\n",
      " (136, 0.01334978944039275),\n",
      " (137, 0.028134784607596275),\n",
      " (138, 0.05854746539431067),\n",
      " (139, 0.04296256823524764),\n",
      " (140, 0.025883028409167137),\n",
      " (141, 0.0009460828207360056),\n",
      " (142, 0.01605005308516299),\n",
      " (143, 0.02904853942229823),\n",
      " (144, 0.09282844115999),\n",
      " (146, 0.02904853942229823),\n",
      " (148, 0.08864453183145782),\n",
      " (149, 0.020131106574108253),\n",
      " (150, 0.002495182309856166),\n",
      " (151, 0.05176605681833427),\n",
      " (152, 0.015357916560933637),\n",
      " (153, 0.047171095668486304),\n",
      " (154, 0.0014438877479842336),\n",
      " (155, 0.048714490070306524),\n",
      " (156, 0.0007035814500111875),\n",
      " (157, 0.10353211363666855),\n",
      " (158, 0.05176605681833427),\n",
      " (159, 0.09742898014061305),\n",
      " (160, 0.03843112819658818),\n",
      " (161, 0.015357916560933637),\n",
      " (162, 0.0011928018173985768),\n",
      " (163, 0.0004651559260999608),\n",
      " (164, 0.024530959653170405),\n",
      " (165, 0.07143200746634257),\n",
      " (166, 0.06188562743999333),\n",
      " (168, 0.035716003733171284),\n",
      " (169, 0.006217077761158846),\n",
      " (170, 0.05263408725654139),\n",
      " (171, 0.07143200746634257),\n",
      " (172, 0.037530355097159775),\n",
      " (173, 0.06250251287211715),\n",
      " (174, 0.04432226591572891),\n",
      " (175, 0.013463642263235204),\n",
      " (176, 0.05854746539431067),\n",
      " (177, 0.025883028409167137),\n",
      " (178, 0.09742898014061305),\n",
      " (179, 0.048714490070306524),\n",
      " (180, 0.05854746539431067),\n",
      " (181, 0.006576235798801425),\n",
      " (182, 0.05854746539431067),\n",
      " (183, 0.0331295929112435),\n",
      " (184, 0.04261284407254031),\n",
      " (185, 0.002495182309856166),\n",
      " (187, 0.17185027294099056),\n",
      " (188, 0.035716003733171284),\n",
      " (189, 0.02939596865833253),\n",
      " (190, 0.05854746539431067),\n",
      " (191, 0.002224964906732125),\n",
      " (192, 0.04296256823524764),\n",
      " (193, 0.002495182309856166),\n",
      " (194, 0.01921556409829409),\n",
      " (195, 0.10750541592653547),\n",
      " (196, 0.014697984329166264),\n",
      " (197, 0.010382275919623069),\n",
      " (198, 0.05854746539431067),\n",
      " (199, 0.006217077761158846),\n",
      " (200, 0.04026221314821651),\n",
      " (201, 0.05854746539431067),\n",
      " (202, 0.006797991968124069),\n",
      " (204, 0.0662992948431539),\n",
      " (205, 0.015357916560933637),\n",
      " (206, 0.14286401493268513),\n",
      " (207, 0.006217077761158846),\n",
      " (208, 0.066259185822487),\n",
      " (209, 0.12972051308833735),\n",
      " (210, 0.022161132957864454),\n",
      " (211, 0.0009460828207360056),\n",
      " (212, 0.08864453183145782),\n",
      " (213, 0.05854746539431067),\n",
      " (214, 0.048714490070306524),\n",
      " (216, 0.045107452287953514),\n",
      " (217, 0.05854746539431067),\n",
      " (218, 0.02737767107618461),\n",
      " (219, 0.015423440856352636),\n",
      " (220, 0.032337174434220455),\n",
      " (221, 0.03671108827702442),\n",
      " (222, 0.03888151474630238),\n",
      " (223, 0.0004651559260999608),\n",
      " (224, 0.028134784607596275),\n",
      " (225, 0.0465932351744787),\n",
      " (226, 0.0009226860036190692),\n",
      " (228, 0.04296256823524764),\n",
      " (230, 0.025883028409167137),\n",
      " (231, 0.01077905814474015),\n",
      " (232, 0.008311940938347534),\n",
      " (233, 0.05854746539431067),\n",
      " (234, 0.04296256823524764),\n",
      " (236, 0.002770646979449178),\n",
      " (237, 0.17728906366291564),\n",
      " (238, 0.028134784607596275),\n",
      " (239, 0.048714490070306524),\n",
      " (240, 0.000692014502714302),\n",
      " (241, 0.060393319722324756),\n",
      " (242, 0.005866788735832111)]\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "\n",
    "for doc in corpus_tfidf:\n",
    "    pprint(doc)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9173a1a",
   "metadata": {},
   "source": [
    "## LDA on Bag of Words Corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "4f6767cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "lda_model = gensim.models.LdaMulticore(bow_corpus, num_topics=10, id2word=dictionary, passes=2, workers=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "fcb3e32f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic: 0 \n",
      "Words: 0.012*\"software\" + 0.011*\"management\" + 0.011*\"experience\" + 0.011*\"project\" + 0.009*\"development\" + 0.007*\"top\" + 0.007*\"back\" + 0.007*\"year\" + 0.007*\"current\" + 0.006*\"skills\" \n",
      "\n",
      "Topic: 1 \n",
      "Words: 0.013*\"management\" + 0.010*\"team\" + 0.007*\"experience\" + 0.007*\"marketing\" + 0.007*\"year\" + 0.007*\"business\" + 0.006*\"month\" + 0.006*\"•\" + 0.005*\"web\" + 0.005*\"development\" \n",
      "\n",
      "Topic: 2 \n",
      "Words: 0.021*\"management\" + 0.017*\"employee\" + 0.017*\"hr\" + 0.010*\"experience\" + 0.009*\"manager\" + 0.007*\"business\" + 0.006*\"development\" + 0.006*\"year\" + 0.005*\"work\" + 0.005*\"sales\" \n",
      "\n",
      "Topic: 3 \n",
      "Words: 0.015*\"sales\" + 0.014*\"management\" + 0.013*\"business\" + 0.013*\"marketing\" + 0.008*\"product\" + 0.007*\"experience\" + 0.007*\"channel\" + 0.007*\"market\" + 0.006*\"retail\" + 0.006*\"development\" \n",
      "\n",
      "Topic: 4 \n",
      "Words: 0.011*\"student\" + 0.010*\"development\" + 0.008*\"management\" + 0.008*\"teacher\" + 0.006*\"english\" + 0.006*\"experience\" + 0.005*\"role\" + 0.005*\"education\" + 0.005*\"work\" + 0.005*\"learning\" \n",
      "\n",
      "Topic: 5 \n",
      "Words: 0.026*\"--\" + 0.013*\"software\" + 0.013*\"sharepoint\" + 0.011*\"english\" + 0.010*\"project\" + 0.009*\"management\" + 0.008*\"experience\" + 0.008*\"development\" + 0.008*\"developer\" + 0.007*\"year\" \n",
      "\n",
      "Topic: 6 \n",
      "Words: 0.015*\"management\" + 0.009*\"experience\" + 0.008*\"sales\" + 0.008*\"development\" + 0.007*\"hr\" + 0.006*\"education\" + 0.006*\"employee\" + 0.006*\"business\" + 0.006*\"current\" + 0.006*\"marketing\" \n",
      "\n",
      "Topic: 7 \n",
      "Words: 0.026*\"software\" + 0.017*\"developer\" + 0.016*\"application\" + 0.014*\"experience\" + 0.013*\"development\" + 0.012*\"•\" + 0.012*\"management\" + 0.011*\"project\" + 0.009*\"year\" + 0.009*\"back\" \n",
      "\n",
      "Topic: 8 \n",
      "Words: 0.012*\"learning\" + 0.012*\"software\" + 0.011*\"experience\" + 0.010*\"machine\" + 0.010*\"application\" + 0.009*\"project\" + 0.009*\"data\" + 0.008*\"year\" + 0.007*\"back\" + 0.007*\"developer\" \n",
      "\n",
      "Topic: 9 \n",
      "Words: 0.016*\"software\" + 0.015*\"management\" + 0.013*\"experience\" + 0.013*\"application\" + 0.011*\"development\" + 0.009*\"year\" + 0.009*\"sql\" + 0.009*\"project\" + 0.009*\"server\" + 0.008*\"developer\" \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for idx, topic in lda_model.print_topics(-1):\n",
    "    print('Topic: {} \\nWords: {} \\n'.format(idx, topic))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3fd8c8f",
   "metadata": {},
   "source": [
    "## LDA on TF-IDF Corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f8afcbff",
   "metadata": {},
   "outputs": [],
   "source": [
    "lda_model_tfidf = gensim.models.LdaMulticore(corpus_tfidf, num_topics=10, id2word=dictionary, passes=2, workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "73e802a1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic: 0 Word: 0.001*\"student\" + 0.001*\"trading\" + 0.001*\"price\" + 0.001*\"pgt\" + 0.001*\"educational\" + 0.001*\"exchange\" + 0.001*\"lucknow\" + 0.001*\"teacher\" + 0.001*\"school\" + 0.001*\"symphony\" \n",
      "\n",
      "Topic: 1 Word: 0.002*\"e\" + 0.001*\"student\" + 0.001*\"xamarin\" + 0.001*\"•\" + 0.001*\"linq\" + 0.001*\"asp.net\" + 0.001*\"teacher\" + 0.001*\"ef\" + 0.001*\"sr.\" + 0.001*\"core\" \n",
      "\n",
      "Topic: 2 Word: 0.002*\"•\" + 0.001*\"hr\" + 0.001*\"employee\" + 0.001*\"student\" + 0.001*\"luxora\" + 0.001*\"pgt\" + 0.001*\"m.b.girls\" + 0.001*\"offsite\" + 0.001*\"learning\" + 0.001*\"teacher\" \n",
      "\n",
      "Topic: 3 Word: 0.002*\"sharepoint\" + 0.001*\"machine\" + 0.001*\"learning\" + 0.001*\"cloud\" + 0.001*\"mvc\" + 0.001*\"jquery\" + 0.001*\"developer\" + 0.001*\"spring\" + 0.001*\"4\" + 0.001*\"software\" \n",
      "\n",
      "Topic: 4 Word: 0.003*\"hr\" + 0.003*\"employee\" + 0.001*\"salary\" + 0.001*\"manager\" + 0.001*\"recruitment\" + 0.001*\"payroll\" + 0.001*\"attendance\" + 0.001*\"administration\" + 0.001*\"exit\" + 0.001*\"statutory\" \n",
      "\n",
      "Topic: 5 Word: 0.002*\"sales\" + 0.002*\"http\" + 0.001*\"c++\" + 0.001*\"oracle\" + 0.001*\"marketing\" + 0.001*\"structures\" + 0.001*\"software\" + 0.001*\"php\" + 0.001*\"cordova\" + 0.001*\"salesforce\" \n",
      "\n",
      "Topic: 6 Word: 0.002*\"sales\" + 0.001*\"plant\" + 0.001*\"hr\" + 0.001*\"•\" + 0.001*\"marketing\" + 0.001*\"territory\" + 0.001*\"order\" + 0.001*\"abbott\" + 0.001*\"tractors\" + 0.001*\"lucknow\" \n",
      "\n",
      "Topic: 7 Word: 0.003*\"•\" + 0.002*\"software\" + 0.002*\"asp.net\" + 0.002*\"mvc\" + 0.002*\"server\" + 0.002*\"machine\" + 0.002*\"application\" + 0.001*\"developer\" + 0.001*\"senior\" + 0.001*\"jquery\" \n",
      "\n",
      "Topic: 8 Word: 0.002*\"--\" + 0.002*\"•\" + 0.002*\"teradata\" + 0.001*\"retail\" + 0.001*\"learning\" + 0.001*\"software\" + 0.001*\"itc\" + 0.001*\"machine\" + 0.001*\"application\" + 0.001*\"register\" \n",
      "\n",
      "Topic: 9 Word: 0.001*\"sales\" + 0.001*\"marketing\" + 0.001*\"channel\" + 0.001*\"l1\" + 0.001*\"iterate\" + 0.001*\".ltd\" + 0.001*\"queries\" + 0.001*\"market\" + 0.001*\"brand\" + 0.001*\"p\" \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for idx, topic in lda_model_tfidf.print_topics(-1):\n",
    "    print('Topic: {} Word: {} \\n'.format(idx, topic))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0aca4653",
   "metadata": {},
   "source": [
    "## LSA on Bag of Words Corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "742f5f64",
   "metadata": {},
   "outputs": [],
   "source": [
    "lsa_model = gensim.models.LsiModel(bow_corpus, num_topics=10, id2word=dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "3dfc8ef1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic: 0 Word: 0.331*\"management\" + 0.226*\"experience\" + 0.223*\"software\" + 0.209*\"project\" + 0.184*\"development\" + 0.169*\"employee\" + 0.168*\"application\" + 0.168*\"hr\" + 0.167*\"year\" + 0.138*\"developer\" \n",
      "\n",
      "Topic: 1 Word: -0.366*\"employee\" + -0.342*\"hr\" + -0.297*\"management\" + 0.270*\"software\" + 0.202*\"application\" + 0.182*\"project\" + 0.178*\"developer\" + 0.168*\"sharepoint\" + -0.153*\"manager\" + -0.104*\"recruitment\" \n",
      "\n",
      "Topic: 2 Word: 0.930*\"--\" + 0.192*\"english\" + 0.086*\"student\" + 0.084*\"teacher\" + -0.083*\"sharepoint\" + -0.070*\"hr\" + -0.069*\"employee\" + 0.060*\"teaching\" + 0.055*\"literature\" + 0.050*\"classroom\" \n",
      "\n",
      "Topic: 3 Word: -0.299*\"marketing\" + -0.290*\"sales\" + 0.285*\"hr\" + 0.283*\"employee\" + -0.237*\"business\" + -0.198*\"sale\" + -0.179*\"market\" + -0.175*\"channel\" + -0.158*\"team\" + -0.148*\"customer\" \n",
      "\n",
      "Topic: 4 Word: 0.498*\"sharepoint\" + -0.199*\"software\" + 0.171*\"2013\" + 0.161*\"2010\" + -0.158*\"•\" + 0.151*\"--\" + 0.147*\"site\" + 0.136*\"client\" + -0.128*\"experience\" + 0.124*\"project\" \n",
      "\n",
      "Topic: 5 Word: -0.422*\"learning\" + -0.289*\"machine\" + 0.255*\"software\" + -0.187*\"data\" + 0.158*\"developer\" + -0.154*\"deep\" + 0.150*\"management\" + -0.140*\"use\" + 0.135*\"application\" + -0.128*\"model\" \n",
      "\n",
      "Topic: 6 Word: -0.606*\"•\" + 0.252*\"software\" + 0.181*\"student\" + -0.167*\"php\" + -0.157*\"mysql\" + 0.130*\"english\" + -0.125*\"team\" + 0.115*\"teacher\" + -0.111*\"project\" + -0.106*\"database\" \n",
      "\n",
      "Topic: 7 Word: 0.355*\"student\" + 0.302*\"•\" + 0.241*\"english\" + 0.226*\"teacher\" + -0.183*\"--\" + -0.171*\"software\" + 0.139*\"educational\" + 0.132*\"pgt\" + 0.130*\"development\" + 0.126*\"school\" \n",
      "\n",
      "Topic: 8 Word: -0.281*\"year\" + 0.237*\"project\" + -0.229*\"0\" + -0.220*\"2020\" + -0.207*\"month\" + -0.177*\"4\" + 0.154*\"•\" + -0.148*\"sales\" + -0.118*\"1\" + 0.110*\"design\" \n",
      "\n",
      "Topic: 9 Word: 0.191*\"team\" + -0.189*\"sales\" + 0.167*\"letter\" + -0.165*\"•\" + -0.160*\"payroll\" + -0.135*\"system\" + 0.134*\"connect\" + 0.121*\"4\" + 0.120*\"year\" + 0.114*\"talent\" \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for idx, topic in lsa_model.print_topics(-1):\n",
    "    print('Topic: {} Word: {} \\n'.format(idx, topic))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0400de54",
   "metadata": {},
   "source": [
    "## LSA on TF-IDF Corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "9452ab30",
   "metadata": {},
   "outputs": [],
   "source": [
    "lsa_model_tfidf = gensim.models.LsiModel(corpus_tfidf, num_topics=10, id2word=dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "8a145ba2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic: 0 Word: 0.317*\"•\" + 0.210*\"employee\" + 0.206*\"hr\" + 0.156*\"software\" + 0.132*\"developer\" + 0.126*\"sales\" + 0.120*\"application\" + 0.115*\"asp.net\" + 0.109*\"web\" + 0.101*\"server\" \n",
      "\n",
      "Topic: 1 Word: 0.376*\"employee\" + 0.368*\"hr\" + -0.236*\"•\" + -0.134*\"software\" + 0.130*\"recruitment\" + 0.123*\"manager\" + -0.116*\"developer\" + 0.112*\"payroll\" + -0.105*\"asp.net\" + 0.097*\"sales\" \n",
      "\n",
      "Topic: 2 Word: -0.424*\"sales\" + -0.309*\"marketing\" + 0.279*\"•\" + 0.195*\"hr\" + 0.186*\"employee\" + -0.177*\"channel\" + -0.154*\"sale\" + -0.133*\"business\" + -0.130*\"market\" + -0.129*\"brand\" \n",
      "\n",
      "Topic: 3 Word: 0.376*\"machine\" + -0.357*\"•\" + 0.325*\"learning\" + -0.159*\"sales\" + 0.155*\"deep\" + 0.149*\"python\" + 0.125*\"model\" + 0.121*\"image\" + -0.121*\"marketing\" + 0.121*\"engineer\" \n",
      "\n",
      "Topic: 4 Word: -0.615*\"•\" + 0.200*\"asp.net\" + -0.183*\"machine\" + -0.164*\"learning\" + 0.154*\"software\" + 0.141*\"mvc\" + 0.127*\"developer\" + 0.124*\"jquery\" + 0.102*\"ajax\" + 0.094*\"javascript\" \n",
      "\n",
      "Topic: 5 Word: -0.519*\"student\" + -0.354*\"teacher\" + -0.245*\"pgt\" + -0.155*\"teaching\" + -0.146*\"curriculum\" + -0.144*\"school\" + -0.133*\"classroom\" + -0.126*\"foreign\" + -0.125*\"m.b.girls\" + -0.124*\"educational\" \n",
      "\n",
      "Topic: 6 Word: -0.263*\"asp.net\" + 0.186*\"java\" + 0.179*\"c++\" + 0.162*\"spring\" + 0.159*\"oracle\" + 0.155*\"teradata\" + 0.154*\"hibernate\" + -0.141*\"server\" + -0.130*\"ajax\" + -0.113*\"mvc\" \n",
      "\n",
      "Topic: 7 Word: -0.299*\"sharepoint\" + 0.194*\"c++\" + 0.154*\"software\" + 0.145*\"c\" + -0.144*\"4\" + -0.137*\"web\" + 0.133*\"structures\" + 0.109*\"python\" + -0.107*\"unix\" + 0.106*\"r\" \n",
      "\n",
      "Topic: 8 Word: -0.225*\"teradata\" + 0.176*\"http\" + 0.175*\"jquery\" + 0.158*\"web\" + 0.151*\"bootstrap\" + -0.141*\"sql\" + 0.139*\"php\" + -0.131*\"server\" + 0.125*\"senior\" + 0.122*\"ajax\" \n",
      "\n",
      "Topic: 9 Word: -0.286*\"teradata\" + 0.201*\"spring\" + -0.166*\"cloud\" + -0.161*\"sharepoint\" + 0.156*\"offsite\" + 0.147*\"hibernate\" + -0.135*\"http\" + -0.131*\"dba\" + 0.128*\"onsite\" + 0.107*\"oracle\" \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for idx, topic in lsa_model_tfidf.print_topics(-1):\n",
    "    print('Topic: {} Word: {} \\n'.format(idx, topic))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63441125",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}