News_Classification_Final_Project.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "News Classification_Final_Project.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kc8VrKhF7Dm0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ac40270c-135e-49ba-e6ff-ed0eaafc64d9"
      },
      "source": [
        "import re\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 693
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U2-fT3kq7FS0"
      },
      "source": [
        "import string\n",
        "import inflect\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.stem.porter import PorterStemmer\n",
        "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.metrics import accuracy_score, classification_report\n",
        "from sklearn.svm import SVC"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fkNJgahS7FWl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1603b213-8ed5-41b9-f571-cd71909d21b4"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sOaMMILe7FZR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aa2de95c-c897-4945-cc65-b0f617bfcaa1"
      },
      "source": [
        "%cd /content/drive/MyDrive/Colab Notebooks/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/Colab Notebooks\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WAWz1yAX7FcJ"
      },
      "source": [
        "def read_files(file_loc):\n",
        "  '''\n",
        "  This function reads txt data from a file in the drive\n",
        "\n",
        "  args - a string containing the files location\n",
        "  returns - a list containing the text data\n",
        "  '''\n",
        "\n",
        "  dataset = []\n",
        "\n",
        "  with open(file_loc, 'r') as train_file:\n",
        "    \n",
        "    for line in train_file:\n",
        "      \n",
        "        dataset.append(line)\n",
        "      \n",
        "  return dataset\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FeWfpFnV7QGF"
      },
      "source": [
        "def separate_labels(dataset):\n",
        "  '''This function will separate the labels/class and examples/documents from the dataset'''\n",
        "  labels = []\n",
        "  documents = []\n",
        "\n",
        "  for line in dataset:\n",
        "    splitted_line = line.strip().split('\\t', 1)\n",
        "    labels.append(splitted_line[0])\n",
        "    documents.append(splitted_line[1])\n",
        "\n",
        "  return labels, documents"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Usm9Iuq97QKk"
      },
      "source": [
        "def remove_url(documents):\n",
        "  '''This function removes URL's from Texts'''\n",
        "  url_removed = []\n",
        "\n",
        "  # Your code here\n",
        "  for line in documents:\n",
        "    url_removed.append(re.sub('http[s]?://\\S+', '', line))\n",
        "\n",
        "  return url_removed"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zaYO9juh7QOh"
      },
      "source": [
        "def remove_hashtag(documents):\n",
        "  '''This function will remove all occurences of # from the texts'''\n",
        "  hashtag_removed = []\n",
        "\n",
        "  # map hashtag to space\n",
        "  translator = str.maketrans('#', ' '*len('#'), '')\n",
        "\n",
        "  for line in documents:\n",
        "    hashtag_removed.append(line.translate(translator))\n",
        "\n",
        "  return hashtag_removed"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rG6-4gLk7Few"
      },
      "source": [
        "def remove_whitespaces(documents):\n",
        "  '''This function removes multiple whitespaces and replace them with a single whitespace'''\n",
        "  whitespace_removed = []\n",
        "\n",
        "  for line in documents:\n",
        "    whitespace_removed.append(' '.join(line.split()))\n",
        "\n",
        "  return whitespace_removed"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UKDfldIG7aS0"
      },
      "source": [
        "def text_lowercasing(documents):\n",
        "  lowercased_docs = []\n",
        "\n",
        "  for line in documents:\n",
        "    lowercased_docs.append(line.lower())\n",
        "\n",
        "  return lowercased_docs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ijjgAeg97aWt"
      },
      "source": [
        "def tokenize_sentence(documents):\n",
        "  '''This function takes a line and provides tokens/words by splitting them using NLTK'''\n",
        "  \n",
        "  tokenized_docs = []\n",
        "  \n",
        "  for line in documents:\n",
        "    tokenized_docs.append(word_tokenize(line))\n",
        "\n",
        "  return tokenized_docs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vaNe_6ma7aal"
      },
      "source": [
        "def char_n_gram_ready(documents):\n",
        "  \n",
        "  joined_docs = []\n",
        "\n",
        "  for line in documents:\n",
        "    joined_docs.append(' '.join(line))\n",
        "\n",
        "  return joined_docs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c5XIDmbq7FiC"
      },
      "source": [
        "def remove_punctuation(documents):\n",
        "\n",
        "  punct_removed = []\n",
        "\n",
        "  for doc in documents:\n",
        "    temp = []\n",
        "    for word in doc:\n",
        "      if word not in string.punctuation:\n",
        "        temp.append(word)\n",
        "    \n",
        "    punct_removed.append(temp)\n",
        "\n",
        "  return punct_removed"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YsYDw4Bx7-l4"
      },
      "source": [
        "def remove_stopwords(documents):\n",
        "  \n",
        "  stopword_removed = []\n",
        "\n",
        "  stop_words = set(stopwords.words('english'))\n",
        "\n",
        "  for doc in documents:\n",
        "    temp = []\n",
        "    for word in doc:\n",
        "      if word not in stop_words:\n",
        "        temp.append(word)\n",
        "    \n",
        "    stopword_removed.append(temp)\n",
        "\n",
        "  return stopword_removed"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CPoyDFPH8oTQ"
      },
      "source": [
        "def apply_stemmer(documents):\n",
        "  stemmed_docs = []\n",
        "  \n",
        "  stemmer = PorterStemmer()\n",
        "\n",
        "  for doc in documents:\n",
        "    stemmed_docs.append([stemmer.stem(plural) for plural in doc])\n",
        "\n",
        "  return stemmed_docs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DFPqofyT8pCP"
      },
      "source": [
        "def identity(X):\n",
        "  return X"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9eNOTf0A8pGB"
      },
      "source": [
        "def vec_tfidf(tfidf = True):\n",
        "\n",
        "  if tfidf:\n",
        "    vec = TfidfVectorizer(preprocessor = identity, analyzer='word',\n",
        "                          tokenizer = identity, ngram_range = (1,3))\n",
        "    # vec = TfidfVectorizer(preprocessor = identity, \n",
        "    #                       tokenizer = identity)\n",
        "  else:\n",
        "    # vec = CountVectorizer(preprocessor = identity, lowercase=True, analyzer='char',\n",
        "    #                       tokenizer = identity, ngram_range = (2,5))\n",
        "    \n",
        "    vec = CountVectorizer(preprocessor = identity,\n",
        "                          tokenizer = identity)\n",
        "    \n",
        "  return vec"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "45owyH8w8xH-"
      },
      "source": [
        "def SVM_Static(train_docs, train_lbls, test_docs, test_lbls):\n",
        "\n",
        "  vec = vec_tfidf(tfidf = True)\n",
        "    \n",
        "  # combines the vectorizer with the Naive Bayes classifier\n",
        "  classifier = Pipeline([('vec', vec),\n",
        "                         ('cls', SVC(kernel='linear'))])\n",
        "  \n",
        "  classifier.fit(train_docs, train_lbls)\n",
        "\n",
        "    # predict is for predicting label for document test data by using predict method\n",
        "  prediction = classifier.predict(test_docs)\n",
        "  \n",
        "  file = open(\"labels.txt\", mode=\"w\")\n",
        "  file.close()\n",
        "  for label in prediction:\n",
        "    file = open(\"labels.txt\", \"a\", encoding='utf-8')\n",
        "    file.write(str(label))\n",
        "    file.write(\"\\n\")\n",
        "  file.close()\n",
        "\n",
        "  print(\"SVM Accuracy = \", accuracy_score(test_lbls, prediction))\n",
        "  print()\n",
        "\n",
        "  print(classification_report(test_lbls, prediction, labels=classifier.classes_, digits=3))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_B-kLyaR8xLq"
      },
      "source": [
        "def Naive_Bayes(train_docs, train_lbls, test_docs, test_lbls):\n",
        "\n",
        "  vec = vec_tfidf(tfidf = False)\n",
        "    \n",
        "  # combines the vectorizer with the Naive Bayes classifier\n",
        "  classifier = Pipeline([('vec', vec),\n",
        "                         ('cls', MultinomialNB())])\n",
        "  \n",
        "  classifier.fit(train_docs, train_lbls)\n",
        "\n",
        "  prediction = classifier.predict(test_docs)\n",
        "\n",
        "  print(\"Naive Bayes Accuracy = \", accuracy_score(test_lbls, prediction))\n",
        "  print()\n",
        "\n",
        "  print(classification_report(test_lbls, prediction, labels=classifier.classes_, digits=3))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AYetygA_7-pu"
      },
      "source": [
        "def pre_processing(documents):\n",
        "\n",
        "  #documents = remove_url(documents)\n",
        "\n",
        "  documents = remove_hashtag(documents)\n",
        "\n",
        "  documents = remove_whitespaces(documents)\n",
        "\n",
        "  documents = text_lowercasing(documents)\n",
        "\n",
        "  documents = tokenize_sentence(documents)\n",
        "\n",
        "  #documents = remove_punctuation(documents)\n",
        "\n",
        "  documents = remove_stopwords(documents)\n",
        "\n",
        "  documents = apply_stemmer(documents)\n",
        "\n",
        "  # If we use character n_gram you have to enable it | else comment the below line\n",
        "  documents = char_n_gram_ready(documents)\n",
        "\n",
        "  return documents"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ul9rfjgE859f"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2SlyogWavY7e",
        "outputId": "18ad9bf9-a3b3-4732-daeb-27a0ffeae278"
      },
      "source": [
        "def main():\n",
        "  print('Reading The Dataset...')\n",
        "  \n",
        "  # Reading the training data\n",
        "  training_dataset = read_files('/content/drive/MyDrive/Colab Notebooks/News_Classification/News_Classification/dev.txt')\n",
        "  train_labels, train_docs = separate_labels(training_dataset[:5000])\n",
        "\n",
        "\n",
        "\n",
        "  # Reading the test data\n",
        "  test_dataset = read_files('/content/drive/MyDrive/Colab Notebooks/News_Classification/News_Classification/train.txt')\n",
        "  test_labels, test_docs = separate_labels(test_dataset[:5000])\n",
        "\n",
        "  # train_docs, train_labels = separate_labels(training_dataset[:2000])\n",
        "  # test_docs, test_labels = separate_labels(test_dataset[:2000])\n",
        "  \n",
        "  # calling the pre processing dunction\n",
        "  train_docs = pre_processing(train_docs)\n",
        "  test_docs = pre_processing(test_docs)\n",
        "  # print(train_docs)\n",
        "\n",
        "  print('\\nTraining the Classifier...')\n",
        "  Naive_Bayes(train_docs, train_labels, test_docs, test_labels)\n",
        "  SVM_Static(train_docs, train_labels, test_docs, test_labels)\n",
        "\n",
        "  for lbl, doc in zip(train_labels[:5], train_docs[:5]):\n",
        "    print(lbl)\n",
        "    print(doc)\n",
        "    print()\n",
        "\n",
        "if __name__ == '__main__':\n",
        "  main()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading The Dataset...\n",
            "\n",
            "Training the Classifier...\n",
            "Naive Bayes Accuracy =  0.5106\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "    Business      0.655     0.392     0.491      1236\n",
            "    Sci_Tech      0.603     0.468     0.527      1497\n",
            "      Sports      0.459     0.570     0.508      1032\n",
            "       World      0.430     0.632     0.511      1235\n",
            "\n",
            "    accuracy                          0.511      5000\n",
            "   macro avg      0.536     0.515     0.509      5000\n",
            "weighted avg      0.543     0.511     0.510      5000\n",
            "\n",
            "SVM Accuracy =  0.8364\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "    Business      0.810     0.811     0.810      1236\n",
            "    Sci_Tech      0.835     0.824     0.829      1497\n",
            "      Sports      0.851     0.933     0.890      1032\n",
            "       World      0.851     0.797     0.823      1235\n",
            "\n",
            "    accuracy                          0.836      5000\n",
            "   macro avg      0.837     0.841     0.838      5000\n",
            "weighted avg      0.836     0.836     0.836      5000\n",
            "\n",
            "Business\n",
            "fear n pension talk . union repres worker turner newal say 'disappoint ' talk stricken parent firm feder mogul . ''\n",
            "\n",
            "Sci_Tech\n",
            "race : second privat team set launch date human spaceflight ( space.com ) . space.com - toronto , canada -- second\\team rocket compet # 36 ; 10 million ansari x prize , contest for\\priv fund suborbit space flight , offici announc first\\launch date man rocket . ''\n",
            "\n",
            "Sci_Tech\n",
            "ky. compani win grant studi peptid ( ap ) . ap - compani found chemistri research univers louisvil grant develop method produc better peptid , short chain amino acid , build block protein . ''\n",
            "\n",
            "Sci_Tech\n",
            "predict unit help forecast wildfir ( ap ) . ap - 's bare dawn mike fitzpatrick start shift blur color map , figur endless chart , alreadi know day bring . lightn strike place expect . wind pick , moist place dri flame roar . ''\n",
            "\n",
            "Sci_Tech\n",
            "calif. aim limit farm-rel smog ( ap ) . ap - southern california 's smog-fight agenc went emiss bovin varieti friday , adopt nation 's first rule reduc air pollut dairi cow manur . ''\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}