Text Classification baseline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import string\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from xgboost import XGBClassifier\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5000\n",
      "(5000, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>response</th>\n",
       "      <th>context</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>SARCASM</td>\n",
       "      <td>@USER @USER @USER I don't get this .. obviousl...</td>\n",
       "      <td>[A minor child deserves privacy and should be ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>SARCASM</td>\n",
       "      <td>@USER @USER trying to protest about . Talking ...</td>\n",
       "      <td>[@USER @USER Why is he a loser ? He's just a P...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>SARCASM</td>\n",
       "      <td>@USER @USER @USER He makes an insane about of ...</td>\n",
       "      <td>[Donald J . Trump is guilty as charged . The e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>SARCASM</td>\n",
       "      <td>@USER @USER Meanwhile Trump won't even release...</td>\n",
       "      <td>[Jamie Raskin tanked Doug Collins . Collins lo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>SARCASM</td>\n",
       "      <td>@USER @USER Pretty Sure the Anti-Lincoln Crowd...</td>\n",
       "      <td>[Man ... y ’ all gone “ both sides ” the apoca...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                           response  \\\n",
       "0  SARCASM  @USER @USER @USER I don't get this .. obviousl...   \n",
       "1  SARCASM  @USER @USER trying to protest about . Talking ...   \n",
       "2  SARCASM  @USER @USER @USER He makes an insane about of ...   \n",
       "3  SARCASM  @USER @USER Meanwhile Trump won't even release...   \n",
       "4  SARCASM  @USER @USER Pretty Sure the Anti-Lincoln Crowd...   \n",
       "\n",
       "                                             context  \n",
       "0  [A minor child deserves privacy and should be ...  \n",
       "1  [@USER @USER Why is he a loser ? He's just a P...  \n",
       "2  [Donald J . Trump is guilty as charged . The e...  \n",
       "3  [Jamie Raskin tanked Doug Collins . Collins lo...  \n",
       "4  [Man ... y ’ all gone “ both sides ” the apoca...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews = []\n",
    "with open('data/train.jsonl') as fl:\n",
    "    for i, line in enumerate(fl):\n",
    "        reviews.append(json.loads(line))\n",
    "print(len(reviews))\n",
    "df = pd.DataFrame(reviews)\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re \n",
    "def function_clean(text):\n",
    "    text = re.sub(r\"http\\S+\", \"\", text) #removing the URL Http\n",
    "    # Removal of mentions\n",
    "    text = re.sub(\"@[^\\s]*\", \"\", text)\n",
    "    # Removal of hashtags\n",
    "    text = re.sub(\"#[^\\s]*\", \"\", text)\n",
    "    # Removal of numbers\n",
    "    text = re.sub('[0-9]*[+-:]*[0-9]+', '', text)\n",
    "    text = re.sub(\"'s\", \"\", text)   \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"context\"]= df[\"context\"].str.join(\" \") \n",
    "df['text'] = df['response'] + df['context']\n",
    "df['text'] = df['text'].apply(lambda text: function_clean(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>response</th>\n",
       "      <th>context</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>SARCASM</td>\n",
       "      <td>@USER @USER @USER I don't get this .. obviousl...</td>\n",
       "      <td>A minor child deserves privacy and should be k...</td>\n",
       "      <td>I don't get this .. obviously you do care o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>SARCASM</td>\n",
       "      <td>@USER @USER trying to protest about . Talking ...</td>\n",
       "      <td>@USER @USER Why is he a loser ? He's just a Pr...</td>\n",
       "      <td>trying to protest about . Talking about him ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>SARCASM</td>\n",
       "      <td>@USER @USER @USER He makes an insane about of ...</td>\n",
       "      <td>Donald J . Trump is guilty as charged . The ev...</td>\n",
       "      <td>He makes an insane about of money from the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>SARCASM</td>\n",
       "      <td>@USER @USER Meanwhile Trump won't even release...</td>\n",
       "      <td>Jamie Raskin tanked Doug Collins . Collins loo...</td>\n",
       "      <td>Meanwhile Trump won't even release his SAT s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>SARCASM</td>\n",
       "      <td>@USER @USER Pretty Sure the Anti-Lincoln Crowd...</td>\n",
       "      <td>Man ... y ’ all gone “ both sides ” the apocal...</td>\n",
       "      <td>Pretty Sure the Anti-Lincoln Crowd Claimed T...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     label                                           response  \\\n",
       "0  SARCASM  @USER @USER @USER I don't get this .. obviousl...   \n",
       "1  SARCASM  @USER @USER trying to protest about . Talking ...   \n",
       "2  SARCASM  @USER @USER @USER He makes an insane about of ...   \n",
       "3  SARCASM  @USER @USER Meanwhile Trump won't even release...   \n",
       "4  SARCASM  @USER @USER Pretty Sure the Anti-Lincoln Crowd...   \n",
       "\n",
       "                                             context  \\\n",
       "0  A minor child deserves privacy and should be k...   \n",
       "1  @USER @USER Why is he a loser ? He's just a Pr...   \n",
       "2  Donald J . Trump is guilty as charged . The ev...   \n",
       "3  Jamie Raskin tanked Doug Collins . Collins loo...   \n",
       "4  Man ... y ’ all gone “ both sides ” the apocal...   \n",
       "\n",
       "                                                text  \n",
       "0     I don't get this .. obviously you do care o...  \n",
       "1    trying to protest about . Talking about him ...  \n",
       "2     He makes an insane about of money from the ...  \n",
       "3    Meanwhile Trump won't even release his SAT s...  \n",
       "4    Pretty Sure the Anti-Lincoln Crowd Claimed T...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df['label'] = df['label'].apply(lambda x: np.where(df['label'].unique()== x )[0][0]) #0-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>response</th>\n",
       "      <th>context</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>4900</td>\n",
       "      <td>1</td>\n",
       "      <td>@USER @USER @USER I voted for Obama and Trump ...</td>\n",
       "      <td>I stand stronger than anyone in protecting you...</td>\n",
       "      <td>I voted for Obama and Trump , but , Dafuq f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4901</td>\n",
       "      <td>1</td>\n",
       "      <td>@USER @USER @USER Miserable people scrutinize ...</td>\n",
       "      <td>The goal of Barack and Michelle Obama is to be...</td>\n",
       "      <td>Miserable people scrutinize others . Try se...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4902</td>\n",
       "      <td>1</td>\n",
       "      <td>@USER @USER @USER ^ ^ ^ This one believes ever...</td>\n",
       "      <td>Hard to understand how Giuiliani , who was onc...</td>\n",
       "      <td>^ ^ ^ This one believes everything \" Dear L...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4903</td>\n",
       "      <td>1</td>\n",
       "      <td>@USER @USER @USER BS justify til the end of ti...</td>\n",
       "      <td>I'm very sorry to have to share this video wit...</td>\n",
       "      <td>BS justify til the end of time but no one w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4904</td>\n",
       "      <td>1</td>\n",
       "      <td>@USER @USER @USER You didn ’ t bother to read ...</td>\n",
       "      <td>@USER @USER @USER I don ’ t know who that guy ...</td>\n",
       "      <td>You didn ’ t bother to read the Mueller rep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4995</td>\n",
       "      <td>1</td>\n",
       "      <td>@USER You don't . I have purchased a lot on Am...</td>\n",
       "      <td>@USER Apologies for the inconvenience you face...</td>\n",
       "      <td>You don't . I have purchased a lot on Amazon ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4996</td>\n",
       "      <td>1</td>\n",
       "      <td>@USER #Emotions you say 🤔 never knew that I th...</td>\n",
       "      <td>@USER 🤔 idk tho , I think I ’ m #hungry . But ...</td>\n",
       "      <td>you say 🤔 never knew that I think I ’ m just...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4997</td>\n",
       "      <td>1</td>\n",
       "      <td>@USER @USER @USER You are so right ... \" Yes !...</td>\n",
       "      <td>@USER @USER @USER Peace to you , and two count...</td>\n",
       "      <td>You are so right ... \" Yes !  is not  is no...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4998</td>\n",
       "      <td>1</td>\n",
       "      <td>@USER @USER @USER Another lazy delusional vote...</td>\n",
       "      <td>Bernie Sanders told Elizabeth Warren in privat...</td>\n",
       "      <td>Another lazy delusional voter who takes the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4999</td>\n",
       "      <td>1</td>\n",
       "      <td>@USER @USER I hope you know no news outlet fro...</td>\n",
       "      <td>PDP PROTEST BRAINSTORMING SESSION Deji : We ne...</td>\n",
       "      <td>I hope you know no news outlet from Nigeria ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      label                                           response  \\\n",
       "4900      1  @USER @USER @USER I voted for Obama and Trump ...   \n",
       "4901      1  @USER @USER @USER Miserable people scrutinize ...   \n",
       "4902      1  @USER @USER @USER ^ ^ ^ This one believes ever...   \n",
       "4903      1  @USER @USER @USER BS justify til the end of ti...   \n",
       "4904      1  @USER @USER @USER You didn ’ t bother to read ...   \n",
       "...     ...                                                ...   \n",
       "4995      1  @USER You don't . I have purchased a lot on Am...   \n",
       "4996      1  @USER #Emotions you say 🤔 never knew that I th...   \n",
       "4997      1  @USER @USER @USER You are so right ... \" Yes !...   \n",
       "4998      1  @USER @USER @USER Another lazy delusional vote...   \n",
       "4999      1  @USER @USER I hope you know no news outlet fro...   \n",
       "\n",
       "                                                context  \\\n",
       "4900  I stand stronger than anyone in protecting you...   \n",
       "4901  The goal of Barack and Michelle Obama is to be...   \n",
       "4902  Hard to understand how Giuiliani , who was onc...   \n",
       "4903  I'm very sorry to have to share this video wit...   \n",
       "4904  @USER @USER @USER I don ’ t know who that guy ...   \n",
       "...                                                 ...   \n",
       "4995  @USER Apologies for the inconvenience you face...   \n",
       "4996  @USER 🤔 idk tho , I think I ’ m #hungry . But ...   \n",
       "4997  @USER @USER @USER Peace to you , and two count...   \n",
       "4998  Bernie Sanders told Elizabeth Warren in privat...   \n",
       "4999  PDP PROTEST BRAINSTORMING SESSION Deji : We ne...   \n",
       "\n",
       "                                                   text  \n",
       "4900     I voted for Obama and Trump , but , Dafuq f...  \n",
       "4901     Miserable people scrutinize others . Try se...  \n",
       "4902     ^ ^ ^ This one believes everything \" Dear L...  \n",
       "4903     BS justify til the end of time but no one w...  \n",
       "4904     You didn ’ t bother to read the Mueller rep...  \n",
       "...                                                 ...  \n",
       "4995   You don't . I have purchased a lot on Amazon ...  \n",
       "4996    you say 🤔 never knew that I think I ’ m just...  \n",
       "4997     You are so right ... \" Yes !  is not  is no...  \n",
       "4998     Another lazy delusional voter who takes the...  \n",
       "4999    I hope you know no news outlet from Nigeria ...  \n",
       "\n",
       "[100 rows x 4 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "X = df[\"text\"]\n",
    "y = df[\"label\"]\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((4000,), (1000,), (4000,), (1000,))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, X_test.shape, y_train.shape, y_test.shape #validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "vect_1 = TfidfVectorizer(stop_words = stopwords.words('english'), ngram_range=(1,3)).fit(X_train) #tuning 1, 2, 3, 1-3, 1-4\n",
    "X_train1 = vect_1.transform(X_train)\n",
    "X_test1 = vect_1.transform(X_test)\n",
    "#Vectorizer (deep learning will be different)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XG Boost train accuary: 0.96125\n",
      "XG Boost test accuary: 0.665\n"
     ]
    }
   ],
   "source": [
    "XGB_model = XGBClassifier(random_state=1)\n",
    "XGB_model.fit(X_train1, y_train)\n",
    "\n",
    "print(f\"XG Boost train accuary: {XGB_model.score(X_train1, y_train)}\")\n",
    "print(f\"XG Boost test accuary: {XGB_model.score(X_test1, y_test)}\")\n",
    "#1. overfit try different parameters to avoid overfitting\n",
    "#2. deep learning- LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1800\n",
      "(1800, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>response</th>\n",
       "      <th>context</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>twitter_1</td>\n",
       "      <td>@USER @USER @USER My 3 year old , that just fi...</td>\n",
       "      <td>[Well now that ’ s problematic AF &lt;URL&gt;, @USER...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>twitter_2</td>\n",
       "      <td>@USER @USER How many verifiable lies has he to...</td>\n",
       "      <td>[Last week the Fake News said that a section o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>twitter_3</td>\n",
       "      <td>@USER @USER @USER Maybe Docs just a scrub of a...</td>\n",
       "      <td>[@USER Let ’ s Aplaud Brett When he deserves i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>twitter_4</td>\n",
       "      <td>@USER @USER is just a cover up for the real ha...</td>\n",
       "      <td>[Women generally hate this president . What's ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>twitter_5</td>\n",
       "      <td>@USER @USER @USER The irony being that he even...</td>\n",
       "      <td>[Dear media Remoaners , you excitedly sharing ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                                           response  \\\n",
       "0  twitter_1  @USER @USER @USER My 3 year old , that just fi...   \n",
       "1  twitter_2  @USER @USER How many verifiable lies has he to...   \n",
       "2  twitter_3  @USER @USER @USER Maybe Docs just a scrub of a...   \n",
       "3  twitter_4  @USER @USER is just a cover up for the real ha...   \n",
       "4  twitter_5  @USER @USER @USER The irony being that he even...   \n",
       "\n",
       "                                             context  \n",
       "0  [Well now that ’ s problematic AF <URL>, @USER...  \n",
       "1  [Last week the Fake News said that a section o...  \n",
       "2  [@USER Let ’ s Aplaud Brett When he deserves i...  \n",
       "3  [Women generally hate this president . What's ...  \n",
       "4  [Dear media Remoaners , you excitedly sharing ...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#predict test: 1800\n",
    "test = []\n",
    "with open('data/test.jsonl') as fl2:\n",
    "    for i, line in enumerate(fl2):\n",
    "        test.append(json.loads(line))\n",
    "print(len(test))\n",
    "df2 = pd.DataFrame(test)\n",
    "print(df2.shape)\n",
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2[\"context\"]= df2[\"context\"].str.join(\" \") \n",
    "df2['text'] = df2['response'] + df['context']\n",
    "df2['text'] = df2['text'].apply(lambda text: function_clean(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "finaltest = df2[\"text\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0          My  year old , that just finished reading N...\n",
       "1         How many verifiable lies has he told now ? +...\n",
       "2          Maybe Docs just a scrub of a coach ... I me...\n",
       "3         is just a cover up for the real hate inside ...\n",
       "4          The irony being that he even has to ask why...\n",
       "                              ...                        \n",
       "1795       is definitely the best out there . No quest...\n",
       "1796      Ye let her out run wild and infect  more Chi...\n",
       "1797       Thanks for that , I would have never known ...\n",
       "1798       Yes also  this on  with loads of <URL> a  I...\n",
       "1799       you still need to send the link to the fan ...\n",
       "Name: text, Length: 1800, dtype: object"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finaltest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "finaltest = vect_1.transform(finaltest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 ... 1 0 1]\n"
     ]
    }
   ],
   "source": [
    "prediction = XGB_model.predict(finaltest)\n",
    "print(prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pred = pd.DataFrame(data = prediction, columns=[\"prediction\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1795</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1796</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1797</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1798</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1799</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1800 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      prediction\n",
       "0              0\n",
       "1              0\n",
       "2              0\n",
       "3              0\n",
       "4              0\n",
       "...          ...\n",
       "1795           0\n",
       "1796           0\n",
       "1797           1\n",
       "1798           0\n",
       "1799           1\n",
       "\n",
       "[1800 rows x 1 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_pred[\"twitter\"]= (df_pred.index + 1)\n",
    "df_pred[\"twitter_index\"]= \"twitter_\" + df_pred[\"twitter\"].astype(str)\n",
    "df_pred[\"result\"] = df_pred['prediction'].apply(lambda x: 'SARCASM' if x == 0 else 'NOT_SARCASM')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prediction</th>\n",
       "      <th>twitter</th>\n",
       "      <th>twitter_index</th>\n",
       "      <th>result</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>twitter_1</td>\n",
       "      <td>SARCASM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>twitter_2</td>\n",
       "      <td>SARCASM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>twitter_3</td>\n",
       "      <td>SARCASM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>twitter_4</td>\n",
       "      <td>SARCASM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>twitter_5</td>\n",
       "      <td>SARCASM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1795</td>\n",
       "      <td>0</td>\n",
       "      <td>1796</td>\n",
       "      <td>twitter_1796</td>\n",
       "      <td>SARCASM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1796</td>\n",
       "      <td>0</td>\n",
       "      <td>1797</td>\n",
       "      <td>twitter_1797</td>\n",
       "      <td>SARCASM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1797</td>\n",
       "      <td>1</td>\n",
       "      <td>1798</td>\n",
       "      <td>twitter_1798</td>\n",
       "      <td>NOT_SARCASM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1798</td>\n",
       "      <td>0</td>\n",
       "      <td>1799</td>\n",
       "      <td>twitter_1799</td>\n",
       "      <td>SARCASM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1799</td>\n",
       "      <td>1</td>\n",
       "      <td>1800</td>\n",
       "      <td>twitter_1800</td>\n",
       "      <td>NOT_SARCASM</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1800 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      prediction  twitter twitter_index       result\n",
       "0              0        1     twitter_1      SARCASM\n",
       "1              0        2     twitter_2      SARCASM\n",
       "2              0        3     twitter_3      SARCASM\n",
       "3              0        4     twitter_4      SARCASM\n",
       "4              0        5     twitter_5      SARCASM\n",
       "...          ...      ...           ...          ...\n",
       "1795           0     1796  twitter_1796      SARCASM\n",
       "1796           0     1797  twitter_1797      SARCASM\n",
       "1797           1     1798  twitter_1798  NOT_SARCASM\n",
       "1798           0     1799  twitter_1799      SARCASM\n",
       "1799           1     1800  twitter_1800  NOT_SARCASM\n",
       "\n",
       "[1800 rows x 4 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_pred[[\"twitter_index\", \"result\"]].to_csv(\"answer.txt\", header=None, index=None, sep=',', mode='w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}