current_ver_classifier_movie_review.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "assignment11-1.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPyAmjrLTXV61JgW/45fzht",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dlguswn3659/class-MachineLearning/blob/master/current_ver_classifier_movie_review.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uw-faMg3kxeX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "c4651efd-4cd5-40d6-9cf8-a1e2f99d745c"
      },
      "source": [
        "  import numpy as np\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "import nltk\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.preprocessing import LabelBinarizer\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem.porter import PorterStemmer\n",
        "from wordcloud import WordCloud,STOPWORDS\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from nltk.tokenize import word_tokenize,sent_tokenize\n",
        "from bs4 import BeautifulSoup\n",
        "import spacy\n",
        "import re,string,unicodedata\n",
        "from nltk.tokenize.toktok import ToktokTokenizer\n",
        "from nltk.stem import LancasterStemmer,WordNetLemmatizer\n",
        "from sklearn.linear_model import LogisticRegression,SGDClassifier\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.svm import SVC\n",
        "from textblob import TextBlob\n",
        "from textblob import Word\n",
        "from sklearn.metrics import classification_report,confusion_matrix,accuracy_score"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xEmx3Nerk1ud",
        "colab_type": "code",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCkgewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwogICAgICBwZXJjZW50LnRleHRDb250ZW50ID0KICAgICAgICAgIGAke01hdGgucm91bmQoKHBvc2l0aW9uIC8gZmlsZURhdGEuYnl0ZUxlbmd0aCkgKiAxMDApfSUgZG9uZWA7CiAgICB9CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 335
        },
        "outputId": "af81be9d-c9df-405b-fce1-cb765c92a820"
      },
      "source": [
        "import numpy as np\n",
        "import re\n",
        "import nltk\n",
        "from sklearn.datasets import load_files\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')\n",
        "import pickle\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.feature_extraction.text import TfidfTransformer\n",
        "from sklearn.model_selection import train_test_split\n",
        "from google.colab import files\n",
        "\n",
        "uploaded = files.upload()\n",
        "\n",
        "import zipfile\n",
        "import io\n",
        "zf = zipfile.ZipFile(io.BytesIO(uploaded['movie_review.zip']), \"r\")\n",
        "zf.extractall()\n",
        "\n",
        "review_data = load_files(r\"movie_review\")\n",
        "X, y = review_data.data, review_data.target\n",
        "\n",
        "documents = []\n",
        "\n",
        "stemmer = WordNetLemmatizer()\n",
        "\n",
        "for sen in range(0, len(X)):\n",
        "    # Remove all the special characters\n",
        "    document = re.sub(r'\\W', ' ', str(X[sen]))\n",
        "    \n",
        "    # remove all single characters\n",
        "    document = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', document)\n",
        "    \n",
        "    # Remove single characters from the start\n",
        "    document = re.sub(r'\\^[a-zA-Z]\\s+', ' ', document) \n",
        "    \n",
        "    # Substituting multiple spaces with single space\n",
        "    document = re.sub(r'\\s+', ' ', document, flags=re.I)\n",
        "    \n",
        "    # Removing prefixed 'b'\n",
        "    document = re.sub(r'^b\\s+', '', document)\n",
        "    \n",
        "    # Converting to Lowercase\n",
        "    document = document.lower()\n",
        "    \n",
        "    # Lemmatization\n",
        "    document = document.split()\n",
        "    document = [stemmer.lemmatize(word) for word in document]\n",
        "    document = ' '.join(document)\n",
        "    \n",
        "    documents.append(document)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-0a3ffb94-da76-4607-9114-124bda5a9b4b\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-0a3ffb94-da76-4607-9114-124bda5a9b4b\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "error",
          "ename": "KeyError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-2-6af326466b9a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mzipfile\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mio\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m \u001b[0mzf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mzipfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mZipFile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mBytesIO\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muploaded\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'movie_review.zip'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m \u001b[0mzf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextractall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyError\u001b[0m: 'movie_review.zip'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "matjDvyhlDJJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(documents)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gt9Wj3kH5q8w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "\n",
        "vectorizer = CountVectorizer(max_features=40000, min_df=5, max_df=0.7, stop_words=stopwords.words('english'))\n",
        "# vectorizer = TfidfVectorizer()\n",
        "X = vectorizer.fit_transform(documents).toarray()\n",
        "print(X)\n",
        "\n",
        "vector_size = 885\n",
        "\n",
        "# tfidfconverter = TfidfTransformer()\n",
        "# print(tfidfconverter)\n",
        "# X = tfidfconverter.fit_transform(X).toarray()\n",
        "# print(X)\n",
        "\n",
        "X_train_tmp, X_test_tmp, y_train, y_test = train_test_split(X, y, test_size=0.3, shuffle=False)\n",
        "\n",
        "# print(X_train.shape)\n",
        "\n",
        "X_pos_sum = np.zeros(12638)\n",
        "X_neg_sum = np.zeros(12638)\n",
        "\n",
        "X_train = np.zeros((1401, 884))\n",
        "X_test = np.zeros((601, 884))\n",
        "\n",
        "for i in range(0, 2002):\n",
        "  if y[i] == 0:\n",
        "    X_neg_sum = X_neg_sum + X[i]\n",
        "  else:\n",
        "    X_pos_sum = X_pos_sum + X[i]\n",
        "\n",
        "pos_neg_voca = 0\n",
        "\n",
        "for i in range(0, 12638):\n",
        "  # if abs(X_neg_sum[i] - X_pos_sum[i]) >= 30:\n",
        "  if (X_neg_sum[i]*3 <= X_pos_sum[i] and X_neg_sum[i] > 2) or (X_pos_sum[i]*3 <= X_neg_sum[i] and X_pos_sum[i] > 2):\n",
        "    for j in range(0, 1401):\n",
        "      X_train[j][pos_neg_voca] = X_train_tmp[j][i]\n",
        "    for j in range(0, 601):\n",
        "      X_test[j][pos_neg_voca] = X_test_tmp[j][i]\n",
        "    pos_neg_voca  = pos_neg_voca + 1\n",
        "\n",
        "print(X_neg_sum)\n",
        "print(X_pos_sum)\n",
        "print(pos_neg_voca)\n",
        "\n",
        "print(X_train)\n",
        "print(X_test)\n",
        "\n",
        "print(X_train[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WQpyXu3jGOvY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(X_train)\n",
        "print(X_test)\n",
        "print(y_train)\n",
        "print(y_test)\n",
        "\n",
        "print(X_train.shape)\n",
        "print(X_test.shape)\n",
        "print(y_train.shape)\n",
        "print(y_test.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iSA68oWdBX2j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#train_classifier에 대한 함수 정의하기\n",
        "# def train_classifier(X_Train, y_Train):\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CD42xdkL05UT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# #Codes for classification\n",
        "\n",
        "# classifier = train_classifier(X_train, y_train)\n",
        "# y_pred_test = classifier.predict(X_test)\n",
        "# y_pred_train = classifier.predict(X_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bV9K1MVaET0T",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "matrix_t = np.zeros((350, vector_size))  #350줄\n",
        "matrix_t[0][0] = 1               #y레이어의 bias를 위한 값 1\n",
        "\n",
        "for i in range(1, 350):\n",
        "  for j in range(0, vector_size):\n",
        "    matrix_t[i][j] = np.random.normal()\n",
        "\n",
        "\n",
        "# matrix_u[196][0] = 0            #줄, 칸\n",
        "##############################################\n",
        "matrix_u = np.zeros((50, 350))  #50줄\n",
        "matrix_u[0][0] = 1               #y레이어의 bias를 위한 값 1\n",
        "\n",
        "for i in range(1, 50):\n",
        "  for j in range(0, 350):\n",
        "    matrix_u[i][j] = np.random.normal()\n",
        "\n",
        "\n",
        "# matrix_u[196][0] = 0            #줄, 칸\n",
        "##############################################\n",
        "matrix_v = np.zeros((8, 50))  #8줄\n",
        "matrix_v[0][0] = 1               #y레이어의 bias를 위한 값 1\n",
        "\n",
        "for i in range(1, 8):\n",
        "  for j in range(0, 50):\n",
        "    matrix_v[i][j] = np.random.normal()\n",
        "\n",
        "\n",
        "# matrix_v[196][0] = 0            #줄, 칸\n",
        "##############################################\n",
        "matrix_w = np.zeros((1, 8))  #1줄\n",
        "\n",
        "for i in range(0, 1):\n",
        "  for j in range(0, 8):\n",
        "    matrix_w[i][j] = np.random.normal()\n",
        "\n",
        "# matrix_w[196][0] = 0            #줄, 칸\n",
        "\n",
        "# print(matrix)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AfHkfVcXRZIw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%matplotlib inline\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib as mpl\n",
        "import seaborn as sns\n",
        "import matplotlib.animation as animation\n",
        "import operator\n",
        "from scipy import stats \n",
        "from mpl_toolkits.mplot3d import Axes3D\n",
        "from sklearn.datasets.samples_generator import make_regression \n",
        "import csv\n",
        "import math\n",
        "import statistics"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dLiiakUcEfil",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import operator\n",
        "\n",
        "x_ = np.ones((350, 1))\n",
        "y_ = np.ones((50, 1))\n",
        "\n",
        "j_list = np.ones(shape=(1401, vector_size, 1))\n",
        "x_list = np.ones(shape=(1401, 350, 1))\n",
        "\n",
        "x__list = np.ones(shape=(1401, 350, 1))\n",
        "y__list = np.ones(shape=(1401, 50, 1))\n",
        "z__list = np.ones(shape=(1401, 8, 1))\n",
        "h__list = np.ones(shape=(1401, 1, 1))\n",
        "\n",
        "x_list = np.ones(shape=(1401, 350, 1))\n",
        "y_list = np.ones(shape=(1401, 50, 1))\n",
        "z_list = np.ones(shape=(1401, 8, 1))\n",
        "h_list = np.ones(shape=(1401, 1, 1))\n",
        "l_list = np.zeros(shape=(1401, 1, 1))\n",
        "\n",
        "x_list_tmp = np.ones(shape=(1401, 350, 1))\n",
        "y_list_tmp = np.ones(shape=(1401, 50, 1))\n",
        "z_list_tmp = np.ones(shape=(1401, 8, 1))\n",
        "h_list_tmp = np.ones(shape=(1401, 1, 1))\n",
        "\n",
        "x__list_tmp = np.ones(shape=(1401, 350, 1))\n",
        "y__list_tmp = np.ones(shape=(1401, 50, 1))\n",
        "z__list_tmp = np.ones(shape=(1401, 8, 1))\n",
        "h__list_tmp = np.ones(shape=(1401, 1, 1))\n",
        "\n",
        "\n",
        "#training set 1401개에 대해서\n",
        "for i in range(0, 1401):  \n",
        "  bias   = [1]       #bias 1\n",
        "  im_vector = np.concatenate((bias, X_train[i]), axis = None)\n",
        "  j_list[i] = im_vector.reshape((vector_size, 1))\n",
        "\n",
        "  x_ = np.dot(matrix_t, j_list[i])  #x와 가중치 u를 곱해서 y_를 만듦\n",
        "  x__list[i] = x_\n",
        "  \n",
        "  x_vector = np.ones((350,1)) #bias\n",
        "  for j in range (1, 350):\n",
        "    x_vector[j][0] = 1 / (1 + math.exp(-x_[j][0]))\n",
        "    # sigmoid(y_[j][0])  #sigmoid 함수를 적용해서 y_에서 y를 만듦\n",
        "\n",
        "  x   = x_vector.reshape((350, 1))\n",
        "  x_list[i] = x\n",
        "\n",
        "\n",
        "\n",
        "  y_ = np.dot(matrix_u, x)  #x와 가중치 u를 곱해서 y_를 만듦\n",
        "  y__list[i] = y_\n",
        "  \n",
        "  y_vector = np.ones((50,1)) #bias\n",
        "  for j in range (1, 50):\n",
        "    y_vector[j][0] = 1 / (1 + math.exp(-y_[j][0]))\n",
        "    # sigmoid(y_[j][0])  #sigmoid 함수를 적용해서 y_에서 y를 만듦\n",
        "\n",
        "  y   = y_vector.reshape((50, 1))\n",
        "  y_list[i] = y\n",
        "\n",
        "  z_  = np.dot(matrix_v, y)  #y와 가중치 v를 곱해서 z_를 만듦\n",
        "  z__list[i] = z_\n",
        "  z_vector = np.ones((8,1)) #bias\n",
        "  for j in range (1, 8):\n",
        "    z_vector[j][0] = 1 / (1 + math.exp(-z_[j][0]))  #sigmoid 함수를 적용해서 y_에서 y를 만듦\n",
        "\n",
        "  z   = z_vector.reshape((8, 1))\n",
        "  z_list[i] = z\n",
        "  \n",
        "  # print(z)\n",
        "\n",
        "  h_  = np.dot(matrix_w, z)  #z와 가중치 w를 곱해서 h_를 만듦\n",
        "  h__list[i] = h_\n",
        "  \n",
        "  h_vector = np.ones((1,1)) #bias\n",
        "  for j in range (0, 1):\n",
        "    h_vector[j][0] = 1 / (1 + math.exp(-h_[j][0]))  #sigmoid 함수를 적용해서 h_에서 h를 만듦\n",
        "\n",
        "  h   = h_vector.reshape((1, 1))\n",
        "  # tmp = h.ravel()\n",
        "  # max_value = max(tmp)\n",
        "  # print(max_value)\n",
        "  # max_index = tmp.index(max_value)\n",
        "  h_list[i] = h\n",
        "\n",
        "  tmp_label = 0\n",
        "  # print(h)\n",
        "\n",
        "  if h >= 0.5:\n",
        "    tmp_label = 1\n",
        "\n",
        "  l_list[i][0] = tmp_label\n",
        "  # print(l_list[i])\n",
        "  # print(h)\n",
        "\n",
        "  x_list_tmp[i] = x_list[i]\n",
        "  y_list_tmp[i] = y_list[i]\n",
        "  z_list_tmp[i] = z_list[i]\n",
        "  h_list_tmp[i] = h_list[i]\n",
        "  x__list_tmp[i] = x__list[i]\n",
        "  y__list_tmp[i] = y__list[i]\n",
        "  z__list_tmp[i] = z__list[i]\n",
        "  h__list_tmp[i] = h__list[i]\n",
        "\n",
        "# print(l_list)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b91lsWSoEg8H",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import operator\n",
        "\n",
        "x_2 = np.ones((350, 1))\n",
        "y_2 = np.ones((50, 1))\n",
        "\n",
        "j_list2 = np.ones(shape=(601, vector_size, 1))\n",
        "x_list2 = np.ones(shape=(601, 350, 1))\n",
        "\n",
        "x__list2 = np.ones(shape=(601, 350, 1))\n",
        "y__list2 = np.ones(shape=(601, 50, 1))\n",
        "z__list2 = np.ones(shape=(601, 8, 1))\n",
        "h__list2 = np.ones(shape=(601, 1, 1))\n",
        "\n",
        "x_list2 = np.ones(shape=(601, 350, 1))\n",
        "y_list2 = np.ones(shape=(601, 50, 1))\n",
        "z_list2 = np.ones(shape=(601, 8, 1))\n",
        "h_list2 = np.ones(shape=(601, 1, 1))\n",
        "l_list2 = np.zeros(shape=(601, 1, 1))\n",
        "\n",
        "x_list_tmp2 = np.ones(shape=(601, 350, 1))\n",
        "y_list_tmp2 = np.ones(shape=(601, 50, 1))\n",
        "z_list_tmp2 = np.ones(shape=(601, 8, 1))\n",
        "h_list_tmp2 = np.ones(shape=(601, 1, 1))\n",
        "\n",
        "x__list_tmp2 = np.ones(shape=(601, 350, 1))\n",
        "y__list_tmp2 = np.ones(shape=(601, 50, 1))\n",
        "z__list_tmp2 = np.ones(shape=(601, 8, 1))\n",
        "h__list_tmp2 = np.ones(shape=(601, 1, 1))\n",
        "\n",
        "\n",
        "#training set 1401개에 대해서\n",
        "for i in range(0, 601):  \n",
        "  bias2   = [1]       #bias 1\n",
        "  im_vector2 = np.concatenate((bias2, X_test[i]), axis = None)\n",
        "  j_list2[i] = im_vector2.reshape((vector_size, 1))\n",
        "\n",
        "  x_2 = np.dot(matrix_t, j_list2[i])  #x와 가중치 u를 곱해서 y_를 만듦\n",
        "  x__list2[i] = x_2\n",
        "  \n",
        "  x_vector2 = np.ones((350,1)) #bias\n",
        "  for j in range (1, 350):\n",
        "    x_vector2[j][0] = 1 / (1 + math.exp(-x_2[j][0]))\n",
        "    # sigmoid(y_[j][0])  #sigmoid 함수를 적용해서 y_에서 y를 만듦\n",
        "\n",
        "  x2   = x_vector2.reshape((350, 1))\n",
        "  x_list2[i] = x2\n",
        "\n",
        "\n",
        "\n",
        "  y_2 = np.dot(matrix_u, x2)  #x와 가중치 u를 곱해서 y_를 만듦\n",
        "  y__list2[i] = y_2\n",
        "  \n",
        "  y_vector2 = np.ones((50,1)) #bias\n",
        "  for j in range (1, 50):\n",
        "    y_vector2[j][0] = 1 / (1 + math.exp(-y_2[j][0]))\n",
        "    # sigmoid(y_[j][0])  #sigmoid 함수를 적용해서 y_에서 y를 만듦\n",
        "\n",
        "  y2   = y_vector2.reshape((50, 1))\n",
        "  y_list2[i] = y2\n",
        "\n",
        "  z_2  = np.dot(matrix_v, y2)  #y와 가중치 v를 곱해서 z_를 만듦\n",
        "  z__list2[i] = z_2\n",
        "  z_vector2 = np.ones((8,1)) #bias\n",
        "  for j in range (1, 8):\n",
        "    z_vector2[j][0] = 1 / (1 + math.exp(-z_2[j][0]))  #sigmoid 함수를 적용해서 y_에서 y를 만듦\n",
        "\n",
        "  z2   = z_vector2.reshape((8, 1))\n",
        "  z_list2[i] = z2\n",
        "  \n",
        "  # print(z)\n",
        "\n",
        "  h_2 = np.dot(matrix_w, z2)  #z와 가중치 w를 곱해서 h_를 만듦\n",
        "  h__list2[i] = h_2\n",
        "  \n",
        "  h_vector2 = np.ones((1,1)) #bias\n",
        "  for j in range (0, 1):\n",
        "    h_vector2[j][0] = 1 / (1 + math.exp(-h_2[j][0]))  #sigmoid 함수를 적용해서 h_에서 h를 만듦\n",
        "\n",
        "  h2   = h_vector2.reshape((1, 1))\n",
        "  # tmp = h.ravel()\n",
        "  # max_value = max(tmp)\n",
        "  # print(max_value)\n",
        "  # max_index = tmp.index(max_value)\n",
        "  h_list2[i] = h2\n",
        "\n",
        "  tmp_label = 0\n",
        "\n",
        "  if h2 >= 0.5:\n",
        "    tmp_label = 1\n",
        "\n",
        "  l_list2[i][0] = tmp_label\n",
        "  # print(l_list[i])\n",
        "  # print(h2)\n",
        "\n",
        "  x_list_tmp2[i] = x_list2[i]\n",
        "  y_list_tmp2[i] = y_list2[i]\n",
        "  z_list_tmp2[i] = z_list2[i]\n",
        "  h_list_tmp2[i] = h_list2[i]\n",
        "  x__list_tmp2[i] = x__list2[i]\n",
        "  y__list_tmp2[i] = y__list2[i]\n",
        "  z__list_tmp2[i] = z__list2[i]\n",
        "  h__list_tmp2[i] = h__list2[i]\n",
        "\n",
        "# print(l_list)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tVgg5PyfCGAn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# (1401, 1500)\n",
        "# (601, 1500)\n",
        "# (1401,)\n",
        "# (601,)\n",
        "\n",
        "train_x=X_train\n",
        "test_x=X_test\n",
        "train_l=y_train\n",
        "test_l=y_test\n",
        "\n",
        "train_x=np.transpose(train_x)\n",
        "test_x=np.transpose(test_x)\n",
        "\n",
        "iteration = 1000\n",
        "l_r = 0.4\n",
        "lambda_ = 100\n",
        "J_sum = 0\n",
        "theta_sum = 0\n",
        "\n",
        "cost_history = np.zeros(iteration)    #about train\n",
        "cost_history2 = np.zeros(iteration)   #about test\n",
        "\n",
        "label_group = np.zeros(shape = (1401, 1))\n",
        "label_group2 = np.zeros(shape = (601, 1))\n",
        "\n",
        "accuracy_history = np.zeros(iteration)\n",
        "accuracy_history2 = np.zeros(iteration)\n",
        "\n",
        "for i in range(0, 1401):\n",
        "  label_group[i][0] = l_list[i][0]\n",
        "\n",
        "for i in range(0, 601):\n",
        "  label_group2[i][0] = l_list2[i][0]\n",
        "\n",
        "for t in range(0, iteration):\n",
        "  h_s_t=h_list[:,:,:].reshape((1401, 1))\n",
        "  z_s_t=z_list[:,1:,:].reshape((1401, 7))\n",
        "  y_s_t=y_list[:,1:,:].reshape((1401, 49))\n",
        "  x_s_t=x_list[:,1:,:].reshape((1401, 349))\n",
        "  t_s_t=train_x[:,1:,:].reshape((1401, vector_size - 1))\n",
        "\n",
        "  w_t=w[1:,1:]\n",
        "  v_t=v[1:,1:]\n",
        "  u_t=u[1:,1:]\n",
        "  t_t=t[1:,1:]\n",
        "  \n",
        "  one_z=np.transpose(np.ones((7), dtype=float))\n",
        "  one_y=np.transpose(np.ones((49), dtype=float))\n",
        "  one_x=np.transpose(np.ones((349), dtype=float))\n",
        "\n",
        "  delta5=np.transpose((np.zeros((7), dtype=float)))\n",
        "  delta4=np.zeros((49,7), dtype=float)\n",
        "  delta3=np.zeros((349,49), dtype=float)\n",
        "  delta2=np.zeros((len(train_x)-1,349), dtype=float)\n",
        "\n",
        "  Jt_sum=0\n",
        "\n",
        "  number_of_correct = 0\n",
        "  number_of_correct2 = 0\n",
        "\n",
        "  print(t, end=' ')\n",
        "\n",
        "  parameter_sum = 0\n",
        "\n",
        "  for q in range(0, vector_size):\n",
        "    for w in range(1, 350):\n",
        "      parameter_sum = parameter_sum + matrix_t[w][q]**2\n",
        "\n",
        "  for q in range(0, 350):\n",
        "    for w in range(1, 50):\n",
        "      parameter_sum = parameter_sum + matrix_u[w][q]**2\n",
        "\n",
        "  for q in range(0, 50):\n",
        "    for w in range(1, 8):\n",
        "      parameter_sum = parameter_sum + matrix_v[w][q]**2\n",
        "\n",
        "  for q in range(0, 8):\n",
        "    for w in range(0, 1):\n",
        "      parameter_sum = parameter_sum + matrix_w[w][q]**2\n",
        "\n",
        "  # result = 0.0\n",
        "  # for i in range(0, 1401):\n",
        "  #   for k in range(0, 1):\n",
        "  #     result = result + ((-1)*label_group[i][k]*math.log(h_list[i][k][0]) - (1-label_group[i][k])*math.log(1-h_list[i][k][0])) + ((lambda_)/(2 * 543258)) * parameter_sum\n",
        "  # result = result / 1401\n",
        "  # cost_history[t] = result\n",
        "\n",
        "  for i in range(0,len(train_x[0])):\n",
        "    lable=train_l[i]\n",
        "    temp5=h_s_t[i]-train_l[i]\n",
        "    temp4=np.multiply( np.dot( np.transpose(w_t ) , temp5 ) ,np.transpose(np.multiply(k_s_t[i] , one_k - k_s_t[i])))\n",
        "    temp3=np.multiply( np.dot( np.transpose(v_t) , temp4) , np.transpose(np.multiply(z_s_t[i] , one_z - z_s_t[i])))\n",
        "    temp2=np.multiply( np.dot( np.transpose( u_t ) , temp3) , np.transpose(np.multiply(y_s_t[i] , one_y - y_s_t[i])))\n",
        "\n",
        "    delta5=delta5+ np.dot( np.transpose(k_s_t[i][np.newaxis]),temp5[np.newaxis])\n",
        "    delta4=delta4 + np.dot( np.transpose(z_s_t[i][np.newaxis]), temp4[np.newaxis])\n",
        "    delta3=delta3+ np.dot( np.transpose(y_s_t[i][np.newaxis]), temp3[np.newaxis] )\n",
        "    delta2=delta2+ np.dot(np.transpose(x_s_t[i][np.newaxis]), temp2[np.newaxis])\n",
        "\n",
        "    J_sum=J_sum+(-train_l[i]*math.log(h_s[0][i])-(1-train_l[i])*math.log(1-h_s[0][i]))\n",
        "\n",
        "    if i < len(test_x[0]):\n",
        "      Jt_sum=Jt_sum+(-test_l[i]*math.log(h_st[0][i])-(1-test_l[i])*math.log(1-h_st[0][i]))\n",
        "\n",
        "  u2=np.multiply(u_t,u_t)\n",
        "  v2=np.multiply(v_t,v_t)\n",
        "  w2=np.multiply(w_t,w_t)\n",
        "  t2=np.multiply(t_t,t_t)\n",
        "  theta_sum=np.sum(u2)+np.sum(v2)+np.sum(w2)+np.sum(t2)\n",
        "\n",
        "  delta5=delta5/len(train_x[0])+lambdaa*np.transpose(t_t)/(7+7*49+49*349+349*(len(train_x)-1))\n",
        "  delta4=delta4/len(train_x[0])+lambdaa*np.transpose(w_t)/(7+7*49+49*349+349*(len(train_x)-1))\n",
        "  delta3=delta3/len(train_x[0])+lambdaa*np.transpose(v_t)/(7+7*49+49*349+349*(len(train_x)-1))\n",
        "  delta2=delta2/len(train_x[0])+lambdaa*np.transpose(u_t)/(7+7*49+49*349+349*(len(train_x)-1))\n",
        "\n",
        "  J_sum=J_sum/len(train_x[0])+lambdaa*theta_sum/(2*(7+7*49+49*349+349*(len(train_x)-1)))\n",
        "  Jt_sum=Jt_sum/len(test_x[0])+lambdaa*theta_sum/(2*(7+7*49+49*349+349*(len(train_x)-1)))\n",
        "\n",
        "\n",
        "  cost_history[t] = J_sum\n",
        "  cost_history2[t] = Jt_sum\n",
        "\n",
        "\n",
        "##gradient descent\n",
        "  result_w = 0\n",
        "  result_v = 0\n",
        "  result_u = 0\n",
        "  result_t = 0\n",
        "\n",
        "  for i in range(0, 1401):\n",
        "    small_delta_2 = np.zeros(shape = (1, 1))\n",
        "    for j in range(0, 1):\n",
        "      small_delta_2[j][0] = h_list[i][j] - label_group[i][j]\n",
        "    # print(h_list[i])\n",
        "    # print(label_group[i])\n",
        "    # small_delta_2 = h_list[i] - label_group[i] #여기 label_group 다시 봐야함.\n",
        "    small_delta_1 = np.dot(np.transpose(matrix_w), small_delta_2)\n",
        "    small_delta_0 = np.dot(np.transpose(matrix_v), small_delta_1)*(y_list_tmp[i]*(1 - y_list_tmp[i]))\n",
        "    small_delta_ = np.dot(np.transpose(matrix_u), small_delta_0)*(x_list_tmp[i]*(1 - x_list_tmp[i])) #이거..괜찮은가..?\n",
        "\n",
        "    # print(small_delta_2)\n",
        "    result_w = result_w + np.dot(small_delta_2, np.transpose(z_list[i]))\n",
        "    result_v = result_v + np.dot(small_delta_1, np.transpose(y_list[i]))\n",
        "    result_u = result_u + np.dot(small_delta_0, np.transpose(x_list[i]))\n",
        "    result_t = result_t + np.dot(small_delta_, np.transpose(j_list[i]))\n",
        "\n",
        "  for a in range(0, 350):\n",
        "    for b in range(1,len(train_x)):\n",
        "      if a<1 and b<8:\n",
        "        matrix_w[a][b]=matrix_w[a][b]-l_r*(np.transpose(delta5)[a][b-1])\n",
        "      if 0<a<8 and b<50:\n",
        "        matrix_v[a][b]=matrix_v[a][b]-l_r*(np.transpose(delta4)[a][b-1])\n",
        "      if 0<a<50 and b<350:\n",
        "        matrix_u[a][b]=matrix_u[a][b]-l_r*np.transpose(delta3)[a][b-1]\n",
        "      matrix_t[a][b]=matrix_t[a][b]-l_r*np.transpose(delta2)[a][b-1]\n",
        "\n",
        "  # matrix_w = matrix_w - (result_w / 1401 *l_r)\n",
        "  # matrix_v = matrix_v - (result_v / 1401 *l_r)\n",
        "  # matrix_u = matrix_u - (result_u / 1401 *l_r)\n",
        "  # matrix_t = matrix_t - (result_t / 1401 *l_r)\n",
        "\n",
        "\n",
        "####################################################################\n",
        "\n",
        "  for k in range(0, 1401):\n",
        "    bias   = [1]       #bias 1\n",
        "    im_vector = np.concatenate((bias, X_train[k]), axis = None)\n",
        "    j_list[k] = im_vector.reshape((vector_size, 1))\n",
        "\n",
        "    x_ = np.dot(matrix_t, j_list[k])  #x와 가중치 u를 곱해서 y_를 만듦\n",
        "    x__list[k] = x_\n",
        "    \n",
        "    x_vector = np.ones((350,1)) #bias\n",
        "    for j in range (1, 350):\n",
        "      x_vector[j][0] = 1 / (1 + math.exp(-x_[j][0]))\n",
        "      # sigmoid(y_[j][0])  #sigmoid 함수를 적용해서 y_에서 y를 만듦\n",
        "\n",
        "    x   = x_vector.reshape((350, 1))\n",
        "    x_list[k] = x\n",
        "\n",
        "\n",
        "\n",
        "    y_ = np.dot(matrix_u, x)  #x와 가중치 u를 곱해서 y_를 만듦\n",
        "    y__list[k] = y_\n",
        "    \n",
        "    y_vector = np.ones((50,1)) #bias\n",
        "    for j in range (1, 50):\n",
        "      y_vector[j][0] = 1 / (1 + math.exp(-y_[j][0]))\n",
        "      # sigmoid(y_[j][0])  #sigmoid 함수를 적용해서 y_에서 y를 만듦\n",
        "\n",
        "    y   = y_vector.reshape((50, 1))\n",
        "    y_list[k] = y\n",
        "\n",
        "    z_  = np.dot(matrix_v, y)  #y와 가중치 v를 곱해서 z_를 만듦\n",
        "    z__list[k] = z_\n",
        "    z_vector = np.ones((8,1)) #bias\n",
        "    for j in range (1, 8):\n",
        "      z_vector[j][0] = 1 / (1 + math.exp(-z_[j][0]))  #sigmoid 함수를 적용해서 y_에서 y를 만듦\n",
        "\n",
        "    z   = z_vector.reshape((8, 1))\n",
        "    z_list[k] = z\n",
        "    \n",
        "    # print(z)\n",
        "\n",
        "    h_  = np.dot(matrix_w, z)  #z와 가중치 w를 곱해서 h_를 만듦\n",
        "    h__list[k] = h_\n",
        "    \n",
        "    h_vector = np.ones((1,1)) #bias\n",
        "    for j in range (0, 1):\n",
        "      h_vector[j][0] = 1 / (1 + math.exp(-h_[j][0]))  #sigmoid 함수를 적용해서 h_에서 h를 만듦\n",
        "\n",
        "    h   = h_vector.reshape((1, 1))\n",
        "    # tmp = h.ravel()\n",
        "    # max_value = max(tmp)\n",
        "    # print(max_value)\n",
        "    # max_index = tmp.index(max_value)\n",
        "\n",
        "    h_list11_tmp = h_list.reshape((1401))\n",
        "    median_val = statistics.median(h_list11_tmp)\n",
        "\n",
        "    h_list[k] = h\n",
        "\n",
        "    # tmp_label = 0\n",
        "\n",
        "    # if h >= median_val:\n",
        "    #     # print(h)\n",
        "    #     tmp_label = 1\n",
        "\n",
        "    if h >= 0.5:\n",
        "      # print(h)\n",
        "      tmp_label = 1\n",
        "\n",
        "    l_list[k][0] = tmp_label\n",
        "    # print(l_list[k])\n",
        "    # print(h)\n",
        "\n",
        "    x_list_tmp[k] = x_list[k]\n",
        "    y_list_tmp[k] = y_list[k]\n",
        "    z_list_tmp[k] = z_list[k]\n",
        "    h_list_tmp[k] = h_list[k]\n",
        "    x__list_tmp[k] = x__list[k]\n",
        "    y__list_tmp[k] = y__list[k]\n",
        "    z__list_tmp[k] = z__list[k]\n",
        "    h__list_tmp[k] = h__list[k]\n",
        "\n",
        "    if l_list[k][0] == y_train[k]:\n",
        "      number_of_correct = number_of_correct + 1\n",
        "\n",
        "  accuracy_history[t] = number_of_correct / 1401 * 100\n",
        "  print(accuracy_history[t])\n",
        "  # accuracy_history[t] = number_of_correct_predictions/total_number_of_predictions*100\n",
        "  # print(accuracy_history[t])\n",
        "\n",
        "############################\n",
        "  #test data!\n",
        "\n",
        "  number_of_correct_predictions2 = 0\n",
        "  \n",
        "  print(t, end=' ')\n",
        "\n",
        "  result2 = 0.0\n",
        "  # for i in range(0, 601):\n",
        "  #   for k in range(0, 1):\n",
        "  #     # if h_list_tmp[i][k][0] <= 0:\n",
        "  #     # print(h_list_tmp[i][k][0])\n",
        "  #     result2 = result2 + ((-1)*label_group2[i][k]*math.log(h_list2[i][k][0]) - (1-label_group2[i][k])*math.log(1-h_list2[i][k][0])) + ((lambda_)/(2 * 164995)) * parameter_sum\n",
        "  # result2 = result2 / 601\n",
        "  # cost_history2[t] = result2\n",
        "\n",
        "\n",
        "  for k in range(0, 601):\n",
        "  # print(i)\n",
        "    bias2   = [1]       #bias 1\n",
        "    im_vector2 = np.concatenate((bias2, X_test[k]), axis = None)\n",
        "    j_list2[k] = im_vector2.reshape((vector_size, 1))\n",
        "\n",
        "    x_2 = np.dot(matrix_t, j_list2[k])  #x와 가중치 u를 곱해서 y_를 만듦\n",
        "    x__list2[k] = x_2\n",
        "    \n",
        "    x_vector2 = np.ones((350,1)) #bias\n",
        "    for j in range (1, 350):\n",
        "      x_vector2[j][0] = 1 / (1 + math.exp(-x_2[j][0]))\n",
        "      # sigmoid(y_[j][0])  #sigmoid 함수를 적용해서 y_에서 y를 만듦\n",
        "\n",
        "    x2   = x_vector2.reshape((350, 1))\n",
        "    x_list2[k] = x2\n",
        "\n",
        "\n",
        "\n",
        "    y_2 = np.dot(matrix_u, x2)  #x와 가중치 u를 곱해서 y_를 만듦\n",
        "    y__list2[k] = y_2\n",
        "    \n",
        "    y_vector2 = np.ones((50,1)) #bias\n",
        "    for j in range (1, 50):\n",
        "      y_vector2[j][0] = 1 / (1 + math.exp(-y_2[j][0]))\n",
        "      # sigmoid(y_[j][0])  #sigmoid 함수를 적용해서 y_에서 y를 만듦\n",
        "\n",
        "    y2   = y_vector2.reshape((50, 1))\n",
        "    y_list2[k] = y2\n",
        "\n",
        "    z_2  = np.dot(matrix_v, y2)  #y와 가중치 v를 곱해서 z_를 만듦\n",
        "    z__list2[k] = z_2\n",
        "    z_vector2 = np.ones((8,1)) #bias\n",
        "    for j in range (1, 8):\n",
        "      z_vector2[j][0] = 1 / (1 + math.exp(-z_2[j][0]))  #sigmoid 함수를 적용해서 y_에서 y를 만듦\n",
        "\n",
        "    z2   = z_vector2.reshape((8, 1))\n",
        "    z_list2[k] = z2\n",
        "    \n",
        "    # print(z)\n",
        "\n",
        "    h_2 = np.dot(matrix_w, z2)  #z와 가중치 w를 곱해서 h_를 만듦\n",
        "    h__list2[k] = h_2\n",
        "    \n",
        "    h_vector2 = np.ones((1,1)) #bias\n",
        "    for j in range (0, 1):\n",
        "      h_vector2[j][0] = 1 / (1 + math.exp(-h_2[j][0]))  #sigmoid 함수를 적용해서 h_에서 h를 만듦\n",
        "\n",
        "    h2   = h_vector2.reshape((1, 1))\n",
        "    # tmp = h.ravel()\n",
        "    # max_value = max(tmp)\n",
        "    # print(max_value)\n",
        "    # max_index = tmp.index(max_value)\n",
        "\n",
        "    h_list22_tmp = h_list2.reshape((601))\n",
        "    median_val2 = statistics.median(h_list22_tmp)\n",
        "\n",
        "    h_list2[k] = h2\n",
        "\n",
        "    # tmp_label = 0\n",
        "\n",
        "    # if h2 >= median_val2:\n",
        "    #   # print(h2)\n",
        "    #   tmp_label = 1\n",
        "\n",
        "    if h2 >= 0.5:\n",
        "      # print(h2)\n",
        "      tmp_label = 1\n",
        "\n",
        "    l_list2[k][0] = tmp_label\n",
        "    # print(l_list[k])\n",
        "    # print(h2)\n",
        "\n",
        "    x_list_tmp2[k] = x_list2[k]\n",
        "    y_list_tmp2[k] = y_list2[k]\n",
        "    z_list_tmp2[k] = z_list2[k]\n",
        "    h_list_tmp2[k] = h_list2[k]\n",
        "    x__list_tmp2[k] = x__list2[k]\n",
        "    y__list_tmp2[k] = y__list2[k]\n",
        "    z__list_tmp2[k] = z__list2[k]\n",
        "    h__list_tmp2[k] = h__list2[k]\n",
        "\n",
        "    if l_list2[k][0] == y_test[k]:\n",
        "      number_of_correct2 = number_of_correct2 + 1\n",
        "\n",
        "  accuracy_history2[t] = number_of_correct2 / 601 * 100\n",
        "  print(accuracy_history2[t])\n",
        "\n",
        "  # accuracy_history2[t] = number_of_correct_predictions2/total_number_of_predictions2*100\n",
        "  t = t + 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dtFVXgejZDIS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#그 코드를 바탕으로 training loss plotting 하기.\n",
        "real_iter = iteration\n",
        "\n",
        "fig,ax = plt.subplots(figsize=(12,8))\n",
        "\n",
        "cost_tmp = np.zeros(real_iter)\n",
        "\n",
        "for i in range(0, real_iter):\n",
        "  cost_tmp[i] = cost_history[i]\n",
        "\n",
        "\n",
        "# print(cost_history[1])\n",
        "\n",
        "ax.set_ylabel('J(Theta)')\n",
        "ax.set_xlabel('Iterations')\n",
        "_=ax.plot(range(real_iter),cost_tmp,'b.', color = 'blue')\n",
        "_=ax.plot(range(real_iter),cost_history2,'b.', color = 'red')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fJEhxJanbFCg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#그 코드를 바탕으로 training loss plotting 하기.\n",
        "fig,ax = plt.subplots(figsize=(12,8))\n",
        "\n",
        "# print(cost_history[1])\n",
        "\n",
        "ax.set_ylabel('accuracy')\n",
        "ax.set_xlabel('Iterations')\n",
        "_=ax.plot(range(iteration),accuracy_history,'b.', color = 'blue')\n",
        "_=ax.plot(range(iteration),accuracy_history2,'b.', color = 'red')\n",
        "# print(ac_tmp)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PuTRLc611Ayn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Codes for Evaluation\n",
        "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
        "\n",
        "l_list_tmp = l_list.reshape((1401))\n",
        "l_list2_tmp = l_list2.reshape((601))\n",
        "\n",
        "print(confusion_matrix(y_train,l_list_tmp))\n",
        "print(classification_report(y_train,l_list_tmp))\n",
        "print(accuracy_score(y_train, l_list_tmp))\n",
        "\n",
        "print(confusion_matrix(y_test,l_list2_tmp))\n",
        "print(classification_report(y_test,l_list2_tmp))\n",
        "print(accuracy_score(y_test, l_list2_tmp))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HDpSEICHtNlB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class_1 = 0\n",
        "\n",
        "for i in range(0, 601):\n",
        "  if l_list2_tmp[i] == 1:\n",
        "    class_1 = class_1 + 1\n",
        "\n",
        "print(class_1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tbPP4IIn3lMu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(y_train)\n",
        "print(l_list_tmp)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}