assignment11_1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "assignment11-1.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNwjsM1p9qyadmaP3QEXSEn",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dlguswn3659/class-MachineLearning/blob/master/assignment11_1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gt9Wj3kH5q8w",
        "colab_type": "code",
        "outputId": "66e6888d-5084-48a0-c2a1-bae17a545684",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "source": [
        "import numpy as np\n",
        "import re\n",
        "import nltk\n",
        "from sklearn.datasets import load_files\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')\n",
        "import pickle\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.feature_extraction.text import TfidfTransformer\n",
        "from sklearn.model_selection import train_test_split\n",
        "# from google.colab import files\n",
        "\n",
        "# uploaded = files.upload()\n",
        "\n",
        "import zipfile\n",
        "import io\n",
        "zf = zipfile.ZipFile(io.BytesIO(uploaded['movie_review.zip']), \"r\")\n",
        "zf.extractall()\n",
        "\n",
        "review_data = load_files(r\"movie_review\")\n",
        "X, y = review_data.data, review_data.target\n",
        "\n",
        "documents = []\n",
        "\n",
        "stemmer = WordNetLemmatizer()\n",
        "\n",
        "for sen in range(0, len(X)):\n",
        "    # Remove all the special characters\n",
        "    document = re.sub(r'\\W', ' ', str(X[sen]))\n",
        "    \n",
        "    # remove all single characters\n",
        "    document = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', document)\n",
        "    \n",
        "    # Remove single characters from the start\n",
        "    document = re.sub(r'\\^[a-zA-Z]\\s+', ' ', document) \n",
        "    \n",
        "    # Substituting multiple spaces with single space\n",
        "    document = re.sub(r'\\s+', ' ', document, flags=re.I)\n",
        "    \n",
        "    # Removing prefixed 'b'\n",
        "    document = re.sub(r'^b\\s+', '', document)\n",
        "    \n",
        "    # Converting to Lowercase\n",
        "    document = document.lower()\n",
        "    \n",
        "    # Lemmatization\n",
        "    document = document.split()\n",
        "    document = [stemmer.lemmatize(word) for word in document]\n",
        "    document = ' '.join(document)\n",
        "    \n",
        "    documents.append(document)\n",
        "\n",
        "vectorizer = CountVectorizer(max_features=1500, min_df=5, max_df=0.7, stop_words=stopwords.words('english'))\n",
        "X = vectorizer.fit_transform(documents).toarray()\n",
        "\n",
        "tfidfconverter = TfidfTransformer()\n",
        "X = tfidfconverter.fit_transform(X).toarray()\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, shuffle=False)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WQpyXu3jGOvY",
        "colab_type": "code",
        "outputId": "e1401b75-858b-4971-e682-082abe247682",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 665
        }
      },
      "source": [
        "print(X_train)\n",
        "print(X_test)\n",
        "print(y_train)\n",
        "print(y_test)\n",
        "\n",
        "print(X_train.shape)\n",
        "print(X_test.shape)\n",
        "print(y_train.shape)\n",
        "print(y_test.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0.         0.         0.         ... 0.         0.03211483 0.        ]\n",
            " [0.         0.         0.         ... 0.         0.08401884 0.        ]\n",
            " [0.         0.         0.         ... 0.06909913 0.         0.        ]\n",
            " ...\n",
            " [0.         0.         0.         ... 0.         0.03917949 0.        ]\n",
            " [0.         0.         0.         ... 0.         0.         0.        ]\n",
            " [0.         0.05031797 0.         ... 0.         0.03543396 0.        ]]\n",
            "[[0.         0.         0.         ... 0.19464767 0.         0.07702493]\n",
            " [0.05225032 0.         0.05418345 ... 0.04473471 0.         0.        ]\n",
            " [0.         0.10246644 0.         ... 0.10573626 0.         0.        ]\n",
            " ...\n",
            " [0.         0.         0.         ... 0.         0.         0.        ]\n",
            " [0.         0.         0.         ... 0.         0.         0.        ]\n",
            " [0.         0.         0.         ... 0.         0.03878449 0.        ]]\n",
            "[0 0 1 ... 1 0 1]\n",
            "[1 1 0 1 1 1 1 0 0 0 0 1 1 0 0 1 1 1 0 0 0 0 1 1 1 0 0 1 1 1 0 0 0 1 1 0 1\n",
            " 0 1 1 0 1 1 1 1 1 0 0 1 0 1 0 0 1 1 1 1 1 1 0 0 0 0 0 1 1 1 0 1 1 0 0 0 0\n",
            " 0 0 1 1 1 0 1 1 1 0 1 1 0 1 1 0 1 1 0 0 0 1 0 0 0 1 1 1 0 0 1 1 1 1 1 0 0\n",
            " 0 1 1 0 0 0 0 0 1 0 1 1 0 1 0 1 0 0 0 1 0 1 0 1 1 1 0 1 0 1 1 0 0 1 0 1 0\n",
            " 1 1 1 1 0 0 0 0 1 1 1 1 1 0 0 1 1 1 1 0 1 1 0 0 1 0 1 1 1 0 1 0 0 1 1 1 0\n",
            " 1 1 1 1 0 0 0 0 0 1 1 1 1 1 0 1 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 1 0 1 0 0 1\n",
            " 1 1 1 0 1 0 0 0 1 1 0 1 1 0 0 0 1 1 1 0 0 0 1 0 0 1 0 0 1 1 1 1 1 1 1 0 0\n",
            " 0 1 1 0 0 0 0 0 1 0 1 1 1 0 0 1 1 0 0 0 1 1 1 1 1 0 0 0 1 1 1 0 0 0 1 0 0\n",
            " 0 0 0 1 1 1 0 0 1 1 0 0 1 1 1 0 0 0 0 0 1 0 0 1 1 0 1 1 1 0 0 1 1 1 1 0 1\n",
            " 1 1 1 1 0 1 1 1 1 0 1 0 1 1 1 0 1 0 0 0 1 0 0 0 1 1 0 0 1 0 1 1 0 0 0 1 1\n",
            " 0 0 0 1 0 1 1 1 0 0 1 1 1 0 0 1 0 1 0 0 1 1 0 0 1 0 1 0 1 0 0 1 1 0 0 1 0\n",
            " 1 0 1 1 0 1 1 0 1 1 0 1 0 1 0 0 1 1 1 1 0 0 1 1 0 1 0 0 0 0 1 0 0 1 1 1 1\n",
            " 0 1 0 0 1 0 0 0 0 0 1 0 0 0 1 0 1 1 0 0 1 0 0 0 0 0 1 1 0 0 1 1 0 1 1 1 1\n",
            " 1 0 1 1 0 1 0 0 0 0 0 1 1 1 1 1 0 0 1 1 1 0 0 0 0 0 0 0 1 0 0 0 1 0 0 1 1\n",
            " 0 0 1 0 0 1 1 1 1 0 0 0 0 1 1 1 1 1 0 0 0 1 0 1 0 0 0 0 0 0 0 0 0 1 1 0 0\n",
            " 1 0 0 1 1 0 0 0 1 1 1 0 0 0 1 0 0 0 0 1 0 1 0 0 0 1 0 0 1 0 1 1 0 1 1 0 1\n",
            " 1 1 1 0 0 1 1 0 0]\n",
            "(1401, 1500)\n",
            "(601, 1500)\n",
            "(1401,)\n",
            "(601,)\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}