Quora_3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**<font color=blue><h3>Featurizing Text Data with TFIDF-Weighted Word-Vectors</h3></font>**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "#avoid_decoding_problems\n",
    "\n",
    "data = pd.read_csv(\"D:/Applied_Ai/Case Studies/Quora/My Work/New/quora.csv\") #load_the_csv_file_\n",
    "\n",
    "#encode_questions_to_unicode_ref:_https://stackoverflow.com/a/6812069\n",
    "data['question1'] = data['question1'].apply(lambda x: str(x))\n",
    "data['question2'] = data['question2'].apply(lambda x: str(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>What is the story of Kohinoor (Koh-i-Noor) Dia...</td>\n",
       "      <td>What would happen if the Indian government sto...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  qid1  qid2                                          question1  \\\n",
       "0   0     1     2  What is the step by step guide to invest in sh...   \n",
       "1   1     3     4  What is the story of Kohinoor (Koh-i-Noor) Dia...   \n",
       "\n",
       "                                           question2  is_duplicate  \n",
       "0  What is the step by step guide to invest in sh...             0  \n",
       "1  What would happen if the Indian government sto...             0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "#merge_texts\n",
    "questions = list(data['question1']) + list(data['question2'])\n",
    "\n",
    "tfidf = TfidfVectorizer(lowercase=False)\n",
    "tfidf.fit_transform(questions)\n",
    "\n",
    "#dict_key:word_and_value:tfidf_score\n",
    "word2tfidf = dict(zip(tfidf.get_feature_names(), tfidf.idf_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- After we find TFIDF scores, we convert each question to a weighted average of word2vec vectors by these scores.\n",
    "- Here we use a pre-trained GLOVE model which comes free with \"Spacy\". Ref: https://spacy.io/usage/vectors-similarity\n",
    "- It is trained on Wikipedia and therefore, it is stronger in terms of word semantics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#exctract_word2vec_vectors\n",
    "#https://github.com/explosion/spaCy/issues/1721\n",
    "#http://landinghub.visualstudio.com/visual-cpp-build-tools\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 50000/50000 [05:20<00:00, 155.95it/s]\n"
     ]
    }
   ],
   "source": [
    "import en_core_web_sm #en_core_web_sm;_which_includes_over_1_million_unique_vectors_\n",
    "#https://github.com/explosion/spacy-models/releases/download/en_core_web_md-2.0.0/en_core_web_md-2.0.0.tar.gz\n",
    "\n",
    "nlp = en_core_web_sm.load()\n",
    "\n",
    "vecs1 = []\n",
    "\n",
    "#tqdm_is_used_to_print_the_progress_bar_\n",
    "for ques1 in tqdm(list(data['question1'])):\n",
    "    doc1 = nlp(ques1)  \n",
    "    mean_vec1 = np.zeros([len(doc1), len(doc1[0].vector)])\n",
    "    for word1 in doc1:\n",
    "        #word2vec\n",
    "        vec1 = word1.vector\n",
    "        #fetch_idf_score\n",
    "        try:\n",
    "            idf = word2tfidf[str(word1)]\n",
    "        except:\n",
    "            idf = 0\n",
    "        #compute_final_vec\n",
    "        mean_vec1 += vec1 * idf\n",
    "    mean_vec1 = mean_vec1.mean(axis=0)\n",
    "    vecs1.append(mean_vec1)\n",
    "data['q1_feats_m'] = list(vecs1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 50000/50000 [05:18<00:00, 157.13it/s]\n"
     ]
    }
   ],
   "source": [
    "vecs2 = []\n",
    "for ques2 in tqdm(list(data['question2'])):\n",
    "    doc2 = nlp(ques2) \n",
    "    mean_vec2 = np.zeros([len(doc1), len(doc2[0].vector)])\n",
    "    for word2 in doc2:\n",
    "        #word2vec\n",
    "        vec2 = word2.vector\n",
    "        #fetch_idf_score\n",
    "        try:\n",
    "            idf = word2tfidf[str(word2)]\n",
    "        except:\n",
    "            #print_word\n",
    "            idf = 0\n",
    "        #compute_final_vec\n",
    "        mean_vec2 += vec2 * idf\n",
    "    mean_vec2 = mean_vec2.mean(axis=0)\n",
    "    vecs2.append(mean_vec2)\n",
    "data['q2_feats_m'] = list(vecs2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Right now we have two csv files;\n",
    "1. data_without_prepro.csv (Simple Preprocessing Features)\n",
    "2. nlp_features.csv (NLP Features)\n",
    "\n",
    "_Load both the csv files._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load_simple_preprocessing_features.csv_file_(_Simple_Preprocessing_Features_)_\n",
    "p_pro_data = pd.read_csv(\"data_without_prepro.csv\", encoding='latin-1')\n",
    "\n",
    "#load_nlp_features.csv_file_(_NLP_Features_)_\n",
    "nlp_data = pd.read_csv(\"nlp_features.csv\", encoding='latin-1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#drop_qid1;qid2;question1;question2_columns_and_store_rest_of_advance_extracted_(nlp)_features_in_data1\n",
    "data1 = nlp_data.drop(['qid1','qid2','question1','question2'],axis=1)\n",
    "\n",
    "#drop_qid1;qid2;question1;question2;is_duplicate_columns_and_store_rest_of_basic_(simple_preprocessing)_features_in_data2\n",
    "data2 = p_pro_data.drop(['qid1','qid2','question1','question2','is_duplicate'],axis=1)\n",
    "\n",
    "#drop_qid1;qid2;question1;question2;is_duplicate_columns_and_store_rest_of_featurized_text_data_(tfidf_weighted_w2v)_in_data3\n",
    "data3 = data.drop(['qid1','qid2','question1','question2','is_duplicate'],axis=1)\n",
    "\n",
    "#store_featurized_text_data_(tfidf_weighted_w2v_of_question_1)_in_data3_q1\n",
    "data3_q1 = pd.DataFrame(data3.q1_feats_m.values.tolist(), index= data3.index)\n",
    "\n",
    "#store_featurized_text_data_(tfidf_weighted_w2v_of_question_2)_in_data3_q2\n",
    "data3_q2 = pd.DataFrame(data3.q2_feats_m.values.tolist(), index= data3.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>csc_min</th>\n",
       "      <th>csc_max</th>\n",
       "      <th>ctc_min</th>\n",
       "      <th>ctc_max</th>\n",
       "      <th>last_word_eq</th>\n",
       "      <th>first_word_eq</th>\n",
       "      <th>abs_len_diff</th>\n",
       "      <th>mean_len</th>\n",
       "      <th>token_set_ratio</th>\n",
       "      <th>token_sort_ratio</th>\n",
       "      <th>fuzz_ratio</th>\n",
       "      <th>fuzz_partial_ratio</th>\n",
       "      <th>longest_substr_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999980</td>\n",
       "      <td>0.833319</td>\n",
       "      <td>0.999983</td>\n",
       "      <td>0.999983</td>\n",
       "      <td>0.916659</td>\n",
       "      <td>0.785709</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>100</td>\n",
       "      <td>93</td>\n",
       "      <td>93</td>\n",
       "      <td>100</td>\n",
       "      <td>0.982759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>0.399996</td>\n",
       "      <td>0.749981</td>\n",
       "      <td>0.599988</td>\n",
       "      <td>0.699993</td>\n",
       "      <td>0.466664</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>12.5</td>\n",
       "      <td>86</td>\n",
       "      <td>63</td>\n",
       "      <td>66</td>\n",
       "      <td>75</td>\n",
       "      <td>0.596154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.249997</td>\n",
       "      <td>0.399996</td>\n",
       "      <td>0.285712</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>63</td>\n",
       "      <td>63</td>\n",
       "      <td>43</td>\n",
       "      <td>47</td>\n",
       "      <td>0.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>28</td>\n",
       "      <td>24</td>\n",
       "      <td>9</td>\n",
       "      <td>14</td>\n",
       "      <td>0.039216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.199998</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.571420</td>\n",
       "      <td>0.307690</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>67</td>\n",
       "      <td>47</td>\n",
       "      <td>35</td>\n",
       "      <td>56</td>\n",
       "      <td>0.175000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  is_duplicate   cwc_min   cwc_max   csc_min   csc_max   ctc_min  \\\n",
       "0   0             0  0.999980  0.833319  0.999983  0.999983  0.916659   \n",
       "1   1             0  0.799984  0.399996  0.749981  0.599988  0.699993   \n",
       "2   2             0  0.399992  0.333328  0.399992  0.249997  0.399996   \n",
       "3   3             0  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "4   4             0  0.399992  0.199998  0.999950  0.666644  0.571420   \n",
       "\n",
       "    ctc_max  last_word_eq  first_word_eq  abs_len_diff  mean_len  \\\n",
       "0  0.785709           0.0            1.0           2.0      13.0   \n",
       "1  0.466664           0.0            1.0           5.0      12.5   \n",
       "2  0.285712           0.0            1.0           4.0      12.0   \n",
       "3  0.000000           0.0            0.0           2.0      12.0   \n",
       "4  0.307690           0.0            1.0           6.0      10.0   \n",
       "\n",
       "   token_set_ratio  token_sort_ratio  fuzz_ratio  fuzz_partial_ratio  \\\n",
       "0              100                93          93                 100   \n",
       "1               86                63          66                  75   \n",
       "2               63                63          43                  47   \n",
       "3               28                24           9                  14   \n",
       "4               67                47          35                  56   \n",
       "\n",
       "   longest_substr_ratio  \n",
       "0              0.982759  \n",
       "1              0.596154  \n",
       "2              0.166667  \n",
       "3              0.039216  \n",
       "4              0.175000  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#dataframe_of_nlp_features\n",
    "data1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>freq_qid1</th>\n",
       "      <th>freq_qid2</th>\n",
       "      <th>q1len</th>\n",
       "      <th>q2len</th>\n",
       "      <th>q1_n_words</th>\n",
       "      <th>q2_n_words</th>\n",
       "      <th>word_Common</th>\n",
       "      <th>word_Total</th>\n",
       "      <th>word_share</th>\n",
       "      <th>freq_q1+q2</th>\n",
       "      <th>freq_q1-q2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>66</td>\n",
       "      <td>57</td>\n",
       "      <td>14</td>\n",
       "      <td>12</td>\n",
       "      <td>10.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.434783</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>51</td>\n",
       "      <td>88</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>4.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>73</td>\n",
       "      <td>59</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>4.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>65</td>\n",
       "      <td>11</td>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>76</td>\n",
       "      <td>39</td>\n",
       "      <td>13</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  freq_qid1  freq_qid2  q1len  q2len  q1_n_words  q2_n_words  \\\n",
       "0   0          1          1     66     57          14          12   \n",
       "1   1          1          1     51     88           8          13   \n",
       "2   2          1          1     73     59          14          10   \n",
       "3   3          1          1     50     65          11           9   \n",
       "4   4          1          1     76     39          13           7   \n",
       "\n",
       "   word_Common  word_Total  word_share  freq_q1+q2  freq_q1-q2  \n",
       "0         10.0        23.0    0.434783           2           0  \n",
       "1          4.0        20.0    0.200000           2           0  \n",
       "2          4.0        24.0    0.166667           2           0  \n",
       "3          0.0        19.0    0.000000           2           0  \n",
       "4          2.0        20.0    0.100000           2           0  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#data_before_preprocessing \n",
    "data2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>86</th>\n",
       "      <th>87</th>\n",
       "      <th>88</th>\n",
       "      <th>89</th>\n",
       "      <th>90</th>\n",
       "      <th>91</th>\n",
       "      <th>92</th>\n",
       "      <th>93</th>\n",
       "      <th>94</th>\n",
       "      <th>95</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>18.892710</td>\n",
       "      <td>41.648096</td>\n",
       "      <td>33.155665</td>\n",
       "      <td>4.310987</td>\n",
       "      <td>42.732992</td>\n",
       "      <td>-9.864838</td>\n",
       "      <td>-10.634421</td>\n",
       "      <td>-11.840693</td>\n",
       "      <td>5.617005</td>\n",
       "      <td>1.020409</td>\n",
       "      <td>...</td>\n",
       "      <td>20.647821</td>\n",
       "      <td>-21.506388</td>\n",
       "      <td>-24.173498</td>\n",
       "      <td>14.336091</td>\n",
       "      <td>-16.272346</td>\n",
       "      <td>25.371137</td>\n",
       "      <td>25.488638</td>\n",
       "      <td>39.202739</td>\n",
       "      <td>3.772958</td>\n",
       "      <td>9.927760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-23.343609</td>\n",
       "      <td>12.173132</td>\n",
       "      <td>-0.257286</td>\n",
       "      <td>-17.855058</td>\n",
       "      <td>7.816336</td>\n",
       "      <td>-6.990792</td>\n",
       "      <td>-16.245950</td>\n",
       "      <td>-12.547119</td>\n",
       "      <td>3.286317</td>\n",
       "      <td>-2.855047</td>\n",
       "      <td>...</td>\n",
       "      <td>49.590098</td>\n",
       "      <td>11.299516</td>\n",
       "      <td>-22.147376</td>\n",
       "      <td>-50.868811</td>\n",
       "      <td>-0.627945</td>\n",
       "      <td>-2.702794</td>\n",
       "      <td>-11.935560</td>\n",
       "      <td>17.299254</td>\n",
       "      <td>10.530151</td>\n",
       "      <td>17.804554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.031534</td>\n",
       "      <td>18.236641</td>\n",
       "      <td>17.321772</td>\n",
       "      <td>23.341717</td>\n",
       "      <td>25.507479</td>\n",
       "      <td>-6.083631</td>\n",
       "      <td>-20.739405</td>\n",
       "      <td>-15.223138</td>\n",
       "      <td>-2.893024</td>\n",
       "      <td>16.497008</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.429696</td>\n",
       "      <td>-2.656703</td>\n",
       "      <td>-10.358425</td>\n",
       "      <td>16.450623</td>\n",
       "      <td>-22.032162</td>\n",
       "      <td>22.730402</td>\n",
       "      <td>2.092286</td>\n",
       "      <td>8.496325</td>\n",
       "      <td>27.627024</td>\n",
       "      <td>-23.033078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-22.874645</td>\n",
       "      <td>3.698369</td>\n",
       "      <td>-20.610707</td>\n",
       "      <td>6.313510</td>\n",
       "      <td>11.934285</td>\n",
       "      <td>11.623967</td>\n",
       "      <td>-29.311596</td>\n",
       "      <td>2.801635</td>\n",
       "      <td>-17.637064</td>\n",
       "      <td>-12.357982</td>\n",
       "      <td>...</td>\n",
       "      <td>-25.270801</td>\n",
       "      <td>-17.413254</td>\n",
       "      <td>-30.728475</td>\n",
       "      <td>1.437318</td>\n",
       "      <td>8.165583</td>\n",
       "      <td>21.963248</td>\n",
       "      <td>-6.926002</td>\n",
       "      <td>17.847203</td>\n",
       "      <td>50.869994</td>\n",
       "      <td>6.258697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>46.234998</td>\n",
       "      <td>74.337402</td>\n",
       "      <td>-16.575459</td>\n",
       "      <td>26.780416</td>\n",
       "      <td>4.814146</td>\n",
       "      <td>-13.088080</td>\n",
       "      <td>-42.409124</td>\n",
       "      <td>30.424017</td>\n",
       "      <td>4.664057</td>\n",
       "      <td>-15.892533</td>\n",
       "      <td>...</td>\n",
       "      <td>22.066719</td>\n",
       "      <td>-34.157310</td>\n",
       "      <td>-19.464928</td>\n",
       "      <td>-52.149147</td>\n",
       "      <td>-60.026301</td>\n",
       "      <td>20.138790</td>\n",
       "      <td>44.375847</td>\n",
       "      <td>43.633056</td>\n",
       "      <td>-26.840464</td>\n",
       "      <td>17.367882</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 96 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0          1          2          3          4          5   \\\n",
       "0  18.892710  41.648096  33.155665   4.310987  42.732992  -9.864838   \n",
       "1 -23.343609  12.173132  -0.257286 -17.855058   7.816336  -6.990792   \n",
       "2  -0.031534  18.236641  17.321772  23.341717  25.507479  -6.083631   \n",
       "3 -22.874645   3.698369 -20.610707   6.313510  11.934285  11.623967   \n",
       "4  46.234998  74.337402 -16.575459  26.780416   4.814146 -13.088080   \n",
       "\n",
       "          6          7          8          9   ...         86         87  \\\n",
       "0 -10.634421 -11.840693   5.617005   1.020409  ...  20.647821 -21.506388   \n",
       "1 -16.245950 -12.547119   3.286317  -2.855047  ...  49.590098  11.299516   \n",
       "2 -20.739405 -15.223138  -2.893024  16.497008  ...  -2.429696  -2.656703   \n",
       "3 -29.311596   2.801635 -17.637064 -12.357982  ... -25.270801 -17.413254   \n",
       "4 -42.409124  30.424017   4.664057 -15.892533  ...  22.066719 -34.157310   \n",
       "\n",
       "          88         89         90         91         92         93  \\\n",
       "0 -24.173498  14.336091 -16.272346  25.371137  25.488638  39.202739   \n",
       "1 -22.147376 -50.868811  -0.627945  -2.702794 -11.935560  17.299254   \n",
       "2 -10.358425  16.450623 -22.032162  22.730402   2.092286   8.496325   \n",
       "3 -30.728475   1.437318   8.165583  21.963248  -6.926002  17.847203   \n",
       "4 -19.464928 -52.149147 -60.026301  20.138790  44.375847  43.633056   \n",
       "\n",
       "          94         95  \n",
       "0   3.772958   9.927760  \n",
       "1  10.530151  17.804554  \n",
       "2  27.627024 -23.033078  \n",
       "3  50.869994   6.258697  \n",
       "4 -26.840464  17.367882  \n",
       "\n",
       "[5 rows x 96 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#questions_1_tfidf_weighted_word2vec\n",
    "data3_q1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>86</th>\n",
       "      <th>87</th>\n",
       "      <th>88</th>\n",
       "      <th>89</th>\n",
       "      <th>90</th>\n",
       "      <th>91</th>\n",
       "      <th>92</th>\n",
       "      <th>93</th>\n",
       "      <th>94</th>\n",
       "      <th>95</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>18.696167</td>\n",
       "      <td>37.228157</td>\n",
       "      <td>45.303721</td>\n",
       "      <td>5.520344</td>\n",
       "      <td>33.628753</td>\n",
       "      <td>-3.229875</td>\n",
       "      <td>-4.983830</td>\n",
       "      <td>-13.457677</td>\n",
       "      <td>3.857630</td>\n",
       "      <td>4.640148</td>\n",
       "      <td>...</td>\n",
       "      <td>23.768250</td>\n",
       "      <td>-20.440072</td>\n",
       "      <td>-25.349166</td>\n",
       "      <td>12.486474</td>\n",
       "      <td>-19.637780</td>\n",
       "      <td>16.586846</td>\n",
       "      <td>30.426209</td>\n",
       "      <td>43.345757</td>\n",
       "      <td>5.500876</td>\n",
       "      <td>2.262348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-12.684361</td>\n",
       "      <td>36.306758</td>\n",
       "      <td>30.951682</td>\n",
       "      <td>-8.703764</td>\n",
       "      <td>16.983735</td>\n",
       "      <td>-17.617952</td>\n",
       "      <td>-16.031544</td>\n",
       "      <td>2.547901</td>\n",
       "      <td>-4.663870</td>\n",
       "      <td>6.987566</td>\n",
       "      <td>...</td>\n",
       "      <td>63.486983</td>\n",
       "      <td>0.217166</td>\n",
       "      <td>-24.077537</td>\n",
       "      <td>-5.160563</td>\n",
       "      <td>-23.868522</td>\n",
       "      <td>4.316975</td>\n",
       "      <td>2.170839</td>\n",
       "      <td>41.508875</td>\n",
       "      <td>10.780357</td>\n",
       "      <td>21.472071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.850462</td>\n",
       "      <td>41.656700</td>\n",
       "      <td>11.439506</td>\n",
       "      <td>-2.610053</td>\n",
       "      <td>31.287295</td>\n",
       "      <td>-0.030191</td>\n",
       "      <td>-33.339810</td>\n",
       "      <td>9.737094</td>\n",
       "      <td>-8.313165</td>\n",
       "      <td>-18.748285</td>\n",
       "      <td>...</td>\n",
       "      <td>-10.409620</td>\n",
       "      <td>-35.576205</td>\n",
       "      <td>-20.263144</td>\n",
       "      <td>39.945732</td>\n",
       "      <td>-22.443190</td>\n",
       "      <td>12.674661</td>\n",
       "      <td>0.095201</td>\n",
       "      <td>21.343025</td>\n",
       "      <td>24.578321</td>\n",
       "      <td>-7.446970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.718443</td>\n",
       "      <td>15.163344</td>\n",
       "      <td>8.306139</td>\n",
       "      <td>33.179072</td>\n",
       "      <td>17.315456</td>\n",
       "      <td>2.685051</td>\n",
       "      <td>-16.807440</td>\n",
       "      <td>0.021019</td>\n",
       "      <td>-10.302241</td>\n",
       "      <td>-14.506871</td>\n",
       "      <td>...</td>\n",
       "      <td>6.468350</td>\n",
       "      <td>-21.470736</td>\n",
       "      <td>0.546877</td>\n",
       "      <td>40.373335</td>\n",
       "      <td>-9.026011</td>\n",
       "      <td>18.770623</td>\n",
       "      <td>-1.556399</td>\n",
       "      <td>1.348674</td>\n",
       "      <td>18.060066</td>\n",
       "      <td>17.892235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9.795965</td>\n",
       "      <td>29.631456</td>\n",
       "      <td>29.330271</td>\n",
       "      <td>0.375853</td>\n",
       "      <td>15.920613</td>\n",
       "      <td>7.280994</td>\n",
       "      <td>-18.368975</td>\n",
       "      <td>6.835820</td>\n",
       "      <td>11.165791</td>\n",
       "      <td>7.979550</td>\n",
       "      <td>...</td>\n",
       "      <td>17.493541</td>\n",
       "      <td>-11.318888</td>\n",
       "      <td>-11.371670</td>\n",
       "      <td>14.175069</td>\n",
       "      <td>-32.553833</td>\n",
       "      <td>6.133714</td>\n",
       "      <td>20.308695</td>\n",
       "      <td>17.383724</td>\n",
       "      <td>12.348764</td>\n",
       "      <td>-12.703654</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 96 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0          1          2          3          4          5   \\\n",
       "0  18.696167  37.228157  45.303721   5.520344  33.628753  -3.229875   \n",
       "1 -12.684361  36.306758  30.951682  -8.703764  16.983735 -17.617952   \n",
       "2   1.850462  41.656700  11.439506  -2.610053  31.287295  -0.030191   \n",
       "3   5.718443  15.163344   8.306139  33.179072  17.315456   2.685051   \n",
       "4   9.795965  29.631456  29.330271   0.375853  15.920613   7.280994   \n",
       "\n",
       "          6          7          8          9   ...         86         87  \\\n",
       "0  -4.983830 -13.457677   3.857630   4.640148  ...  23.768250 -20.440072   \n",
       "1 -16.031544   2.547901  -4.663870   6.987566  ...  63.486983   0.217166   \n",
       "2 -33.339810   9.737094  -8.313165 -18.748285  ... -10.409620 -35.576205   \n",
       "3 -16.807440   0.021019 -10.302241 -14.506871  ...   6.468350 -21.470736   \n",
       "4 -18.368975   6.835820  11.165791   7.979550  ...  17.493541 -11.318888   \n",
       "\n",
       "          88         89         90         91         92         93  \\\n",
       "0 -25.349166  12.486474 -19.637780  16.586846  30.426209  43.345757   \n",
       "1 -24.077537  -5.160563 -23.868522   4.316975   2.170839  41.508875   \n",
       "2 -20.263144  39.945732 -22.443190  12.674661   0.095201  21.343025   \n",
       "3   0.546877  40.373335  -9.026011  18.770623  -1.556399   1.348674   \n",
       "4 -11.371670  14.175069 -32.553833   6.133714  20.308695  17.383724   \n",
       "\n",
       "          94         95  \n",
       "0   5.500876   2.262348  \n",
       "1  10.780357  21.472071  \n",
       "2  24.578321  -7.446970  \n",
       "3  18.060066  17.892235  \n",
       "4  12.348764 -12.703654  \n",
       "\n",
       "[5 rows x 96 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#questions_2_tfidf_weighted_word2vec\n",
    "data3_q2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NUMBER OF FEATURES IN NLP DATAFRAME: 17\n",
      "NUMBER OF FEATURES IN PREPROCESSED DATAFRAME: 12\n",
      "NUMBER OF FEATURES IN QUESTION1 W2V  DATAFRAME: 96\n",
      "NUMBER OF FEATURES IN QUESTION2 W2V  DATAFRAME: 96\n",
      "NUMBER OF FEATURES IN FINAL DATAFRAME: 221\n"
     ]
    }
   ],
   "source": [
    "print(\"NUMBER OF FEATURES IN NLP DATAFRAME:\", data1.shape[1])\n",
    "print(\"NUMBER OF FEATURES IN PREPROCESSED DATAFRAME:\", data2.shape[1])\n",
    "print(\"NUMBER OF FEATURES IN QUESTION1 W2V  DATAFRAME:\", data3_q1.shape[1])\n",
    "print(\"NUMBER OF FEATURES IN QUESTION2 W2V  DATAFRAME:\", data3_q2.shape[1])\n",
    "print(\"NUMBER OF FEATURES IN FINAL DATAFRAME:\", data1.shape[1]+data2.shape[1]+data3_q1.shape[1]+data3_q2.shape[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#storing_the_final_features_to_csv_file\n",
    "\n",
    "data3_q1['id']=data1['id']\n",
    "data3_q2['id']=data1['id']\n",
    "\n",
    "#merge_nlp_data_and_preprocessed_data_and_store_in_data1_\n",
    "data1 = data1.merge(data2, on='id',how='left')\n",
    "\n",
    "#merge_questions_1_tfidf_weighted_word2vec_and_questions_2_tfidf_weighted_word2vec_store_in_data2_\n",
    "data2 = data3_q1.merge(data3_q2, on='id',how='left')\n",
    "\n",
    "#now_merge_data1_and_data2_in_final_\n",
    "final  = data1.merge(data2, on='id',how='left')\n",
    "\n",
    "#save_data_in_csv_file\n",
    "final.to_csv('final_quora.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}