Nutshell_Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0b124443",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Libraries Required\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import os\n",
    "\n",
    "# Import functions for data preprocessing & data preparation\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.utils import resample\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
    "\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.stem import PorterStemmer, LancasterStemmer\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.corpus import wordnet\n",
    "import string\n",
    "from string import punctuation\n",
    "import nltk\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c4cebafb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'Comment'], dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reading data\n",
    "# For the second dataset: Nutshell: Black Money\n",
    "\n",
    "data = pd.read_csv('E:/SEM 7/H SET PROJECT/comments CSVs/scrappedfile_nutshell.csv')\n",
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4f1a0ff4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What other finance-related videos should we ma...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Great way of making new generation understand ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>This is something amazing I've saw on YouTube ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Randomly landed on this channel while watching...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I am still counfused why this 2 k people will ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>Hi 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>Ll</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>Can you make a video about how many people pay...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>Can you make a video about how many people pay...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>91 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Comment\n",
       "0   What other finance-related videos should we ma...\n",
       "1   Great way of making new generation understand ...\n",
       "2   This is something amazing I've saw on YouTube ...\n",
       "3   Randomly landed on this channel while watching...\n",
       "4   I am still counfused why this 2 k people will ...\n",
       "..                                                ...\n",
       "86                                               Hi 0\n",
       "87                                                 Ll\n",
       "88  Can you make a video about how many people pay...\n",
       "89                                                  P\n",
       "90  Can you make a video about how many people pay...\n",
       "\n",
       "[91 rows x 1 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data1=data.drop(['Unnamed: 0'],axis=1)\n",
    "data1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a24b21d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Comment</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Compound</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What other finance-related videos should we ma...</td>\n",
       "      <td>0.170</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.830</td>\n",
       "      <td>0.4329</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Great way of making new generation understand ...</td>\n",
       "      <td>0.397</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.603</td>\n",
       "      <td>0.8360</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>This is something amazing I've saw on YouTube ...</td>\n",
       "      <td>0.412</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.588</td>\n",
       "      <td>0.8271</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Randomly landed on this channel while watching...</td>\n",
       "      <td>0.250</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.750</td>\n",
       "      <td>0.7178</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I am still counfused why this 2 k people will ...</td>\n",
       "      <td>0.315</td>\n",
       "      <td>0.168</td>\n",
       "      <td>0.517</td>\n",
       "      <td>0.5065</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             Comment  Positive  Negative  \\\n",
       "0  What other finance-related videos should we ma...     0.170     0.000   \n",
       "1  Great way of making new generation understand ...     0.397     0.000   \n",
       "2  This is something amazing I've saw on YouTube ...     0.412     0.000   \n",
       "3  Randomly landed on this channel while watching...     0.250     0.000   \n",
       "4  I am still counfused why this 2 k people will ...     0.315     0.168   \n",
       "\n",
       "   Neutral  Compound Sentiment  \n",
       "0    0.830    0.4329  Positive  \n",
       "1    0.603    0.8360  Positive  \n",
       "2    0.588    0.8271  Positive  \n",
       "3    0.750    0.7178  Positive  \n",
       "4    0.517    0.5065  Positive  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Data Labelling\n",
    "\n",
    "sentiments = SentimentIntensityAnalyzer()\n",
    "data1[\"Positive\"] = [sentiments.polarity_scores(i)[\"pos\"] for i in data1[\"Comment\"]]\n",
    "data1[\"Negative\"] = [sentiments.polarity_scores(i)[\"neg\"] for i in data1[\"Comment\"]]\n",
    "data1[\"Neutral\"] = [sentiments.polarity_scores(i)[\"neu\"] for i in data1[\"Comment\"]]\n",
    "data1['Compound'] = [sentiments.polarity_scores(i)[\"compound\"] for i in data1[\"Comment\"]]\n",
    "score = data1[\"Compound\"].values\n",
    "sentiment = []\n",
    "for i in score:\n",
    "    if i >= 0.05 :\n",
    "        sentiment.append('Positive')\n",
    "    elif i <= -0.05 :\n",
    "        sentiment.append('Negative')\n",
    "    else:\n",
    "        sentiment.append('Neutral')\n",
    "data1[\"Sentiment\"] = sentiment\n",
    "data1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "701a427c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Comment</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What other finance-related videos should we ma...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Great way of making new generation understand ...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>This is something amazing I've saw on YouTube ...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Randomly landed on this channel while watching...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I am still counfused why this 2 k people will ...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             Comment Sentiment\n",
       "0  What other finance-related videos should we ma...  Positive\n",
       "1  Great way of making new generation understand ...  Positive\n",
       "2  This is something amazing I've saw on YouTube ...  Positive\n",
       "3  Randomly landed on this channel while watching...  Positive\n",
       "4  I am still counfused why this 2 k people will ...  Positive"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Final Data\n",
    "\n",
    "data2=data1.drop(['Positive','Negative','Neutral','Compound'],axis=1)\n",
    "data2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "777dabd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data Transformation\n",
    "\n",
    "stop_words = stopwords.words('english')\n",
    "porter_stemmer = PorterStemmer()\n",
    "lancaster_stemmer = LancasterStemmer() \n",
    "snowball_stemer = SnowballStemmer(language=\"english\")\n",
    "lzr = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "80519c88",
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_processing(text):   \n",
    "    # convert text into lowercase\n",
    "    text = text.lower()\n",
    "\n",
    "    # remove new line characters in text\n",
    "    text = re.sub(r'\\n',' ', text)\n",
    "    \n",
    "    # remove punctuations from text\n",
    "    text = re.sub('[%s]' % re.escape(punctuation), \"\", text)\n",
    "    \n",
    "    # remove references and hashtags from text\n",
    "    text = re.sub(\"^a-zA-Z0-9$,.\", \"\", text)\n",
    "    \n",
    "    # remove multiple spaces from text\n",
    "    text = re.sub(r'\\s+', ' ', text, flags=re.I)\n",
    "    \n",
    "    # remove special characters from text\n",
    "    text = re.sub(r'\\W', ' ', text)\n",
    "\n",
    "    text = ' '.join([word for word in word_tokenize(text) if word not in stop_words])\n",
    "    \n",
    "    # stemming using porter stemmer from nltk package - msh a7sn 7aga - momken: lancaster, snowball\n",
    "    # text=' '.join([porter_stemmer.stem(word) for word in word_tokenize(text)])\n",
    "    # text=' '.join([lancaster_stemmer.stem(word) for word in word_tokenize(text)])\n",
    "    # text=' '.join([snowball_stemer.stem(word) for word in word_tokenize(text)])\n",
    "    \n",
    "    # lemmatizer using WordNetLemmatizer from nltk package\n",
    "    text=' '.join([lzr.lemmatize(word) for word in word_tokenize(text)])\n",
    "\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3af7c93f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_copy = data2.copy()\n",
    "data_copy.Comment = data_copy.Comment.apply(lambda text: text_processing(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c97a0eb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "data_copy['Sentiment'] = le.fit_transform(data_copy['Sentiment'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f99241aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sentence</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>financerelated video make whatd like u break next</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>great way making new generation understand his...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>something amazing ive saw youtube day loved</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>randomly landed channel watching another video...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>still counfused 2 k people dislike masterpiece</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Sentence  Sentiment\n",
       "0  financerelated video make whatd like u break next          2\n",
       "1  great way making new generation understand his...          2\n",
       "2        something amazing ive saw youtube day loved          2\n",
       "3  randomly landed channel watching another video...          2\n",
       "4     still counfused 2 k people dislike masterpiece          2"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processed_data = {\n",
    "    'Sentence':data_copy.Comment,\n",
    "    'Sentiment':data_copy['Sentiment']\n",
    "}\n",
    "\n",
    "processed_data = pd.DataFrame(processed_data)\n",
    "processed_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b034215f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    45\n",
       "1    35\n",
       "0    11\n",
       "Name: Sentiment, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processed_data['Sentiment'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "85bffb19",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_neutral = processed_data[(processed_data['Sentiment']==1)] \n",
    "df_negative = processed_data[(processed_data['Sentiment']==0)]\n",
    "df_positive = processed_data[(processed_data['Sentiment']==2)]\n",
    "\n",
    "# upsample minority classes\n",
    "df_negative_upsampled = resample(df_negative, \n",
    "                                 replace=True,    \n",
    "                                 n_samples= 205, \n",
    "                                 random_state=42)  \n",
    "\n",
    "df_neutral_upsampled = resample(df_neutral, \n",
    "                                 replace=True,    \n",
    "                                 n_samples= 205, \n",
    "                                 random_state=42)  \n",
    "\n",
    "\n",
    "# Concatenate the upsampled dataframes with the neutral dataframe\n",
    "final_data = pd.concat([df_negative_upsampled,df_neutral_upsampled,df_positive])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9c265ef1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    205\n",
       "1    205\n",
       "2     45\n",
       "Name: Sentiment, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_data['Sentiment'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "941639c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['hard economy hit demonetization introduction gst covid happened within period 354 year way recovery',\n",
       " 'hell video disliked much',\n",
       " 'make video many people pay tax india much total transaction black popular case people got arrested',\n",
       " 'low tax control',\n",
       " 'indian govt trying hard']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus = []\n",
    "for sentence in final_data['Sentence']:\n",
    "    corpus.append(sentence)\n",
    "corpus[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "59dfc988",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "cv = CountVectorizer(max_features=1500)\n",
    "X = cv.fit_transform(corpus).toarray()\n",
    "y = final_data.iloc[:, -1].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "160bb5f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GaussianNB()"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Machine Learning Model\n",
    "\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)\n",
    "classifier = GaussianNB()\n",
    "classifier.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "6a13d83c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[65,  0,  0],\n",
       "       [ 0, 59,  0],\n",
       "       [ 2,  1, 10]], dtype=int64)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix, accuracy_score\n",
    "y_pred = classifier.predict(X_test)\n",
    "cm = confusion_matrix(y_test, y_pred)\n",
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "72e32ae2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy 0.9781021897810219\n"
     ]
    }
   ],
   "source": [
    "nb_score = accuracy_score(y_test, y_pred)\n",
    "print('accuracy',nb_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f20a8c1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}