quora-eda.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import networkx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv('./data/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.369197853026293"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['is_duplicate'].sum() / len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "graph = networkx.MultiGraph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "graph.add_nodes_from(data['qid1'].unique())\n",
    "\n",
    "graph.add_nodes_from(data['qid2'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "graph.add_edges_from(data[data['is_duplicate'] == 1][['qid1', 'qid2']].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([     1,      3,      5, ..., 537928, 537930, 537932])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['qid1'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200803"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "networkx.number_connected_components(graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "networkx.nodes(graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "dic = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "404290it [01:13, 5517.98it/s]\n"
     ]
    }
   ],
   "source": [
    "for _, (qid1, qid2, q1, q2) in tqdm(data[['qid1', 'qid2', 'question1', 'question2']].iterrows()):\n",
    "    dic[qid1] = q1\n",
    "    dic[qid2] = q2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "pycharm": {},
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "53 268\n",
      "[\"How will Trump's presidency affect prospective international students from Syria?\", \"What would be the effect of Trump's presidency on international Master's students who plan to work in the US after completing the degree?\", 'How would a Trump presidency affect schools accessibility for international students?', 'I am an international student in the US - do I have to leave if Trump wins?', \"What effect will Donald Trump's victory have on international students looking to study in USA?\", 'Will international college students be negatively affected if Trump becomes president?', \"What does Donald Trump's win mean for Indian students in USA?\", \"What does Donald Trump's victory mean for the graduate students from Islamic countries?\", 'What happens to the Indian students trying to pursue Masters of PHD in USA, now that Trump is the president?', \"What does a Trump presidency mean for Indian students looking to pursue Master's degrees in USA?\", 'I am an Indian, planning to go to US for MS (a STEM course) this January. If Trump wins, how will that affect my future in US?', 'How is Trump becoming the president affect the Indians applying for an MS in the US (Mech)?', 'Will Donald Trump’s win in the elections have an impact on Indian students applying for a higher education in the US?', 'How will Trump’s presidency affect international students in the US?', 'How might Trump affect the status of foreign students at top universities in the US?', 'What would a Trump presidency mean for current international master’s students on an F1 visa?', 'How will a Trump presidency affect the students presently in US or planning to study in US?', 'How will Trump or Clinton presidency affect foreign students?', 'What will happen to Chinese students studying abroad in America now that Trump is president?', \"How does Trump's win affects Indian students planning for graduate studies in United States?\", 'How will it affect international students coming to the US for undergraduation now that Trump has become the President?', 'What will happen to international students in the US if Donald Trump becomes president?', 'How would it affect Indian students studying/working in the US if Trump becomes the President?', \"How does Donald Trump's win affects Indian students preparing for graduate studies in US?\", \"What are the advantages and disadvantages of international students in the US, now that Trump's officially the President of the United States?\", \"How will Trump's presidency affect the Indian students who are planning to study in the US?\", 'How will Donald Trump becoming a president affect the Indian students who are trying to pursue their masters in America?', 'Now that Donald Trump is President, will international students stop coming to US universities?', 'Should I (a prospective Indian student) stop applying to universities in the USA, now that Donald Trump has become the president?', 'How are international students from Hong Kong going to be affected after Donald Trump becomes president?', 'Will foreign students studying in the USA be unwelcomed after Donald Trump is elected as president?', 'Considering I am an international student, should my plan to study in the US be scrapped in a Trump Presidency?', 'If Trump is elected, will I be held from entering the US as an international student?', \"How bad can Trump's election as president be for students aspiring to study in US?\", 'How would Trump presidency affect Indian students in the US?', \"How will Trump's presidency affect Indian students who are planning to do a PhD in the US?\", 'What will be the impact of Donald Trump’s win on Indian students, who are planning to study in the United States?', 'Is there any harm for the international students from South Asian countries in America after Trump becoming the President?', 'Will the Trump factor affect the admission of international students in the USA?', 'What will happen if Donald trump wins, and its effect on Indian students who are studying in US?']\n"
     ]
    }
   ],
   "source": [
    "subgraphs = (graph.subgraph(c) for c in networkx.connected_components(graph))\n",
    "\n",
    "for sg in subgraphs:\n",
    "    nn = networkx.number_of_nodes(sg)\n",
    "    ne = networkx.number_of_edges(sg)\n",
    "    if ne != nn * (nn - 1) // 2:\n",
    "        print(nn, ne)\n",
    "        ids = networkx.nodes(sg)\n",
    "        print([dic[vertex] for vertex in ids][:40])\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[     1,      2],\n",
       "       [     3,      4],\n",
       "       [     5,      6],\n",
       "       ...,\n",
       "       [537928, 537929],\n",
       "       [537930, 537931],\n",
       "       [537932, 537933]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[['qid1', 'qid2']].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>What is the story of Kohinoor (Koh-i-Noor) Dia...</td>\n",
       "      <td>What would happen if the Indian government sto...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>How can I increase the speed of my internet co...</td>\n",
       "      <td>How can Internet speed be increased by hacking...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>Why am I mentally very lonely? How can I solve...</td>\n",
       "      <td>Find the remainder when [math]23^{24}[/math] i...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>Which one dissolve in water quikly sugar, salt...</td>\n",
       "      <td>Which fish would survive in salt water?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  qid1  qid2                                          question1  \\\n",
       "0   0     1     2  What is the step by step guide to invest in sh...   \n",
       "1   1     3     4  What is the story of Kohinoor (Koh-i-Noor) Dia...   \n",
       "2   2     5     6  How can I increase the speed of my internet co...   \n",
       "3   3     7     8  Why am I mentally very lonely? How can I solve...   \n",
       "4   4     9    10  Which one dissolve in water quikly sugar, salt...   \n",
       "\n",
       "                                           question2  is_duplicate  \n",
       "0  What is the step by step guide to invest in sh...             0  \n",
       "1  What would happen if the Indian government sto...             0  \n",
       "2  How can Internet speed be increased by hacking...             0  \n",
       "3  Find the remainder when [math]23^{24}[/math] i...             0  \n",
       "4            Which fish would survive in salt water?             0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "290654"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['qid1'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8461      50\n",
       "14110     48\n",
       "25984     47\n",
       "20628     47\n",
       "1749      47\n",
       "          ..\n",
       "415339     1\n",
       "411241     1\n",
       "413288     1\n",
       "409190     1\n",
       "524288     1\n",
       "Name: qid1, Length: 290654, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['qid1'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "299364"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['qid2'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "404290"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data['qid1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "404290"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data['qid2'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}