CompleteCode-Final.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The effect of preprocessing on short document clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas, numpy, textblob, string\n",
    "import numpy as np\n",
    "from sklearn import model_selection, preprocessing, linear_model, naive_bayes, metrics, svm, decomposition, ensemble\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.cluster import KMeans\n",
    "\n",
    "from nltk.stem import PorterStemmer, WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "import nltk\n",
    "from gensim.models import word2vec, KeyedVectors\n",
    "import gensim\n",
    "from gensim.scripts.glove2word2vec import glove2word2vec\n",
    "from sklearn.neighbors import KDTree"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Amazon_load_data():\n",
    "    data = open('corpus', encoding=\"utf8\").read()\n",
    "    labels, texts = [], []\n",
    "    for i, line in enumerate(data.split(\"\\n\")):\n",
    "        content = line.split()\n",
    "        labels.append(content[0])\n",
    "        texts.append(content[1:])\n",
    "\n",
    "    # create a dataframe using texts and lables\n",
    "    data = pandas.DataFrame()\n",
    "    data['text'] = texts\n",
    "    data['label'] = labels\n",
    "    texts1=[' '.join(line) for line in texts] \n",
    "    data['text']=texts1\n",
    "    data.loc[:,'label'].replace(['__label__1', '__label__2'], [0, 1], inplace=True)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Yelp_load_data():\n",
    "    data = pandas.read_csv('yelp.csv')\n",
    "    data = data.drop('Unnamed: 0', axis=1)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Dbpedia_load_data():\n",
    "    data = pandas.read_csv('dbpedia.csv')\n",
    "    data = data.drop('Unnamed: 0', axis=1)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quick info on datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Amazon Dataset\n",
    "df = Amazon_load_data()\n",
    "print(\"Amazon data:\")\n",
    "print(\"Shape: \", df.shape)\n",
    "count = df['text'].str.split().str.len()\n",
    "print(\"Average number of words in documents\", np.mean(count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# obtain the number of unique words in the complete dataframe, by joining all documents, counting the occurences and taking the length\n",
    "uniqueWords = list(set(\" \".join(df['text']).lower().split(\" \")))\n",
    "uniqueWords_count = len(uniqueWords)\n",
    "uniqueWords_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "5000 * 13.7794"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Yelp dataset\n",
    "df = Yelp_load_data()\n",
    "print(\"Yelp data:\")\n",
    "print(\"Shape: \", df.shape)\n",
    "count = df['text'].str.split().str.len()\n",
    "print(\"Average number of words in documents\", np.mean(count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# obtain the number of unique words in the complete dataframe, by joining all documents, counting the occurences and taking the length\n",
    "uniqueWords = list(set(\" \".join(df['text']).lower().split(\" \")))\n",
    "uniqueWords_count = len(uniqueWords)\n",
    "uniqueWords_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DBpedia dataset\n",
    "df = Dbpedia_load_data()\n",
    "print(\"DBpedia data:\")\n",
    "print(\"Shape: \", df.shape)\n",
    "count = df['text'].str.split().str.len()\n",
    "print(\"Average number of words in documents\", np.mean(count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# obtain the number of unique words in the complete dataframe, by joining all documents, counting the occurences and taking the length\n",
    "uniqueWords = list(set(\" \".join(df['text']).lower().split(\" \")))\n",
    "uniqueWords_count = len(uniqueWords)\n",
    "uniqueWords_count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def textcleaning(trainDF, lower=0, punctuation=0, stemming=0, lemmatization=0, commonwords=0, rarewords=0, stopword=0):\n",
    "    if (lower):\n",
    "        # to Lowercase\n",
    "        trainDF['text'] = trainDF['text'].apply(lambda x: \" \".join(x.lower() for x in x.split()))\n",
    "    if (punctuation):\n",
    "        # remove punctuation\n",
    "        trainDF['text'] = trainDF['text'].str.replace('[^\\w\\s]','')\n",
    "    if (commonwords):\n",
    "        # remove common words\n",
    "        freq = pandas.Series(' '.join(trainDF['text']).split()).value_counts()[:10]\n",
    "        freq = list(freq.index)\n",
    "        trainDF['text'] = trainDF['text'].apply(lambda x: \" \".join(x for x in x.split() if x not in freq))\n",
    "    if (rarewords):\n",
    "        # rare words removal\n",
    "        freq = pandas.Series(' '.join(trainDF['text']).split()).value_counts()[-10:]\n",
    "        freq = list(freq.index)\n",
    "        trainDF['text'] = trainDF['text'].apply(lambda x: \" \".join(x for x in x.split() if x not in freq))\n",
    "    if (stemming):\n",
    "        # stemming\n",
    "        st = PorterStemmer()\n",
    "        trainDF['text'] = trainDF['text'].apply(lambda x: \" \".join([st.stem(word) for word in x.split()]))\n",
    "    if (lemmatization):\n",
    "        # lemmatization\n",
    "        wordnet_lemmatizer = WordNetLemmatizer()\n",
    "        trainDF['text'] = trainDF['text'].apply(lambda x: \" \".join([wordnet_lemmatizer.lemmatize(word, pos=\"v\") for word in x.split()]))\n",
    "    if (stopword):\n",
    "        # remove stopwords\n",
    "        stop = stopwords.words('english')\n",
    "        trainDF['text'] = trainDF['text'].apply(lambda x: \" \".join(x for x in x.split() if x not in stop))\n",
    "    return trainDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature extraction methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def wordtfidf(text):\n",
    "    # word level tf-idf\n",
    "    tfidf_vect = TfidfVectorizer(analyzer='word', token_pattern=r'\\w{1,}', max_features=5000)\n",
    "    tfidf_vect.fit(text)\n",
    "    text_tfidf =  tfidf_vect.transform(text)\n",
    "    return text_tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ngramtfidf(text):\n",
    "    # ngram level tf-idf \n",
    "    tfidf_vect_ngram = TfidfVectorizer(analyzer='word', token_pattern=r'\\w{1,}', ngram_range=(2,3), max_features=5000)\n",
    "    tfidf_vect_ngram.fit(text)\n",
    "    x_tfidf_ngram =  tfidf_vect_ngram.transform(text)\n",
    "    return x_tfidf_ngram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenizecorpus(x):\n",
    "    wpt = nltk.WordPunctTokenizer()\n",
    "    tokenized_corpus = [wpt.tokenize(document) for document in x]\n",
    "    return tokenized_corpus\n",
    "\n",
    "def emb_glove():\n",
    "    glove_input_file = \"glove.6B.300d.txt\"\n",
    "    word2vec_output_file = \"word2vec.txt\"\n",
    "    glove2word2vec(glove_input_file, word2vec_output_file)\n",
    "    glove_model = KeyedVectors.load_word2vec_format(word2vec_output_file, binary=False)\n",
    "    return glove_model\n",
    "\n",
    "def emb_W2V(corpus):\n",
    "    # Set values for various parameters\n",
    "    feature_size = 10    # Word vector dimensionality  \n",
    "    window_context = 10  # Context window size                                                                                    \n",
    "    min_word_count = 1   # Minimum word count                        \n",
    "    sample = 1e-3   # Downsample setting for frequent words\n",
    "    tokenized_corpus = corpus\n",
    "    w2v_model = word2vec.Word2Vec(tokenized_corpus, size=feature_size, \n",
    "                                  window=window_context, min_count = min_word_count,\n",
    "                                  sample=sample, iter=100)\n",
    "    return w2v_model\n",
    "\n",
    "def average_word_vectors(words, model, vocabulary, num_features):\n",
    "    feature_vector = np.zeros((num_features,),dtype=\"float64\")\n",
    "    nwords = 0.\n",
    "\n",
    "    for word in words:\n",
    "        if word in vocabulary: \n",
    "            nwords = nwords + 1.\n",
    "            feature_vector = np.add(feature_vector, model[word])\n",
    "    \n",
    "    if nwords:\n",
    "        feature_vector = np.divide(feature_vector, nwords)\n",
    "        \n",
    "    return feature_vector\n",
    "    \n",
    "   \n",
    "def averaged_word_vectorizer(corpus, model, num_features):\n",
    "    vocabulary = set(model.wv.index2word)\n",
    "    features = [average_word_vectors(tokenized_sentence, model, vocabulary, num_features)\n",
    "                    for tokenized_sentence in corpus]\n",
    "    return np.array(features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K-means and evaluation metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def k_means(x, y, k, repeat = 1):\n",
    "    arr_a = list()\n",
    "    arr_b = list()\n",
    "    arr_c = list()\n",
    "    arr_f = list()\n",
    "    arr_g = list()\n",
    "    arr_h = list()\n",
    "    arr_i = list()\n",
    "    arr_j = list()\n",
    "    for i in range(repeat):\n",
    "        km = KMeans(n_clusters=k)\n",
    "        kmeans = km.fit(x)\n",
    "        predictions = kmeans.labels_\n",
    "        a = metrics.accuracy_score(predictions, y)\n",
    "        b = metrics.silhouette_score(x, predictions)\n",
    "        c = metrics.adjusted_rand_score(y, predictions)\n",
    "        f = metrics.silhouette_score(x, y)\n",
    "        g = metrics.adjusted_mutual_info_score(y, predictions, average_method='arithmetic')\n",
    "        h = metrics.normalized_mutual_info_score(y, predictions, average_method='arithmetic')\n",
    "        i = metrics.homogeneity_score(y, predictions)\n",
    "        j = metrics.completeness_score(y, predictions)\n",
    "        arr_a.append(a)\n",
    "        arr_b.append(b)\n",
    "        arr_c.append(c)\n",
    "        arr_f.append(f)\n",
    "        arr_g.append(g)\n",
    "        arr_h.append(h)\n",
    "        arr_i.append(i)\n",
    "        arr_j.append(j)\n",
    "    print(kmeans.cluster_centers_)        \n",
    "    print(\"accuracy: \\t\", np.mean(arr_a), \"std: \", np.std(arr_a))\n",
    "    print(\"ASW: \\t\\t\", np.mean(arr_b), \"std: \", np.std(arr_b))\n",
    "    print(\"ARI: \\t\\t\", np.mean(arr_c), \"std: \", np.std(arr_c))\n",
    "    print(\"true ASW: \\t\", np.mean(arr_f), \"std: \", np.std(arr_f))\n",
    "    print(\"AMI: \\t\\t\", np.mean(arr_g), \"std: \", np.std(arr_g))\n",
    "    print(\"NMI: \\t\\t\", np.mean(arr_h), \"std: \", np.std(arr_h))\n",
    "    print(\"H: \\t\\t\", np.mean(arr_i), \"std: \", np.std(arr_i))\n",
    "    print(\"C: \\t\\t\", np.mean(arr_j), \"std: \", np.std(arr_j))\n",
    "    means = [np.mean(arr_a), np.mean(arr_b), np.mean(arr_c), np.mean(arr_f), np.mean(arr_g), np.mean(arr_h), np.mean(arr_i), np.mean(arr_j)]\n",
    "    stds = [np.std(arr_a), np.std(arr_b), np.std(arr_c), np.std(arr_f), np.std(arr_g), np.std(arr_h), np.std(arr_i), np.std(arr_j)]\n",
    "    return means, stds, predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Normalization levels\n",
    "## Available settings:\n",
    "1. lower\n",
    "2. punctuation\n",
    "3. stemming\n",
    "4. lemmatization\n",
    "5. commonwords\n",
    "6. rarewords\n",
    "7. stopwords\n",
    "\n",
    "### Levels:\n",
    "- first level: no settings\n",
    "- second level: lower & punctuation\n",
    "- third level: second level & common words & rare words\n",
    "- fourth level: third level & stopwords\n",
    "- fifth level: third level & stemming\n",
    "- sixth level: third level & lemmatization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = Amazon_load_data()\n",
    "df = textcleaning(df, lower=0, punctuation=0, stemming=0, lemmatization=0, commonwords=0, rarewords=0, stopword=0)\n",
    "x = df['text']\n",
    "y = df['label']\n",
    "\n",
    "tfidf_vect = TfidfVectorizer(analyzer='word', token_pattern=r'\\w{1,}', max_features=5000)\n",
    "tfidf_vect.fit(x)\n",
    "text_tfidf =  tfidf_vect.transform(x)\n",
    "print (\"WordLevel TF-IDF: \")\n",
    "x_tfidf = text_tfidf\n",
    "\n",
    "km = KMeans(n_clusters=2)\n",
    "kmeans = km.fit(x_tfidf)\n",
    "print(\"Done\")\n",
    "\n",
    "print(\"Top terms per cluster:\")\n",
    "order_centroids = km.cluster_centers_.argsort()[:, ::-1]\n",
    "terms = tfidf_vect.get_feature_names()\n",
    "for i in range(2):\n",
    "    print(\"Cluster %d:\" % i, end='')\n",
    "    for ind in order_centroids[i, :10]:\n",
    "        print(' %s' % terms[ind], end='')\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = Amazon_load_data()\n",
    "df = textcleaning(df, lower=1, punctuation=1, stemming=0, lemmatization=1, commonwords=1, rarewords=1, stopword=1)\n",
    "x = df['text']\n",
    "y = df['label']\n",
    "\n",
    "tfidf_vect = TfidfVectorizer(analyzer='word', token_pattern=r'\\w{1,}', max_features=5000)\n",
    "tfidf_vect.fit(x)\n",
    "text_tfidf =  tfidf_vect.transform(x)\n",
    "print (\"WordLevel TF-IDF: \")\n",
    "x_tfidf = text_tfidf\n",
    "\n",
    "km = KMeans(n_clusters=2)\n",
    "kmeans = km.fit(x_tfidf)\n",
    "print(\"Done\")\n",
    "\n",
    "print(\"Top terms per cluster:\")\n",
    "order_centroids = km.cluster_centers_.argsort()[:, ::-1]\n",
    "terms = tfidf_vect.get_feature_names()\n",
    "for i in range(2):\n",
    "    print(\"Cluster %d:\" % i, end='')\n",
    "    for ind in order_centroids[i, :10]:\n",
    "        print(' %s' % terms[ind], end='')\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = Amazon_load_data()\n",
    "df = textcleaning(df, lower=0, punctuation=0, stemming=0, lemmatization=0, commonwords=0, rarewords=0, stopword=0)\n",
    "x = df['text']\n",
    "y = df['label']\n",
    "\n",
    "tfidf_vect_ngram = TfidfVectorizer(analyzer='word', token_pattern=r'\\w{1,}', ngram_range=(2,3), max_features=5000)\n",
    "tfidf_vect_ngram.fit(x)\n",
    "x_tfidf_ngram =  tfidf_vect_ngram.transform(x)\n",
    "print (\"ngram TF-IDF: \")\n",
    "\n",
    "km = KMeans(n_clusters=2)\n",
    "kmeans = km.fit(x_tfidf_ngram)\n",
    "print(\"Done\")\n",
    "\n",
    "print(\"Top terms per cluster:\")\n",
    "order_centroids = km.cluster_centers_.argsort()[:, ::-1]\n",
    "terms = tfidf_vect_ngram.get_feature_names()\n",
    "for i in range(2):\n",
    "    print(\"Cluster %d:\" % i, end='')\n",
    "    for ind in order_centroids[i, :10]:\n",
    "        print(' %s' % terms[ind], end='')\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = Amazon_load_data()\n",
    "df = textcleaning(df, lower=1, punctuation=1, stemming=1, lemmatization=0, commonwords=1, rarewords=1, stopword=1)\n",
    "x = df['text']\n",
    "y = df['label']\n",
    "\n",
    "tfidf_vect_ngram = TfidfVectorizer(analyzer='word', token_pattern=r'\\w{1,}', ngram_range=(2,3), max_features=5000)\n",
    "tfidf_vect_ngram.fit(x)\n",
    "x_tfidf_ngram =  tfidf_vect_ngram.transform(x)\n",
    "print (\"ngram TF-IDF: \")\n",
    "\n",
    "km = KMeans(n_clusters=2)\n",
    "kmeans = km.fit(x_tfidf_ngram)\n",
    "print(\"Done\")\n",
    "\n",
    "print(\"Top terms per cluster:\")\n",
    "order_centroids = km.cluster_centers_.argsort()[:, ::-1]\n",
    "terms = tfidf_vect_ngram.get_feature_names()\n",
    "for i in range(2):\n",
    "    print(\"Cluster %d:\" % i, end='')\n",
    "    for ind in order_centroids[i, :10]:\n",
    "        print(' %s' % terms[ind], end='')\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = Yelp_load_data()\n",
    "df = textcleaning(df, lower=0, punctuation=0, stemming=0, lemmatization=0, commonwords=0, rarewords=0, stopword=0)\n",
    "x = df['text']\n",
    "y = df['label']\n",
    "\n",
    "wpt = nltk.WordPunctTokenizer()\n",
    "tokenized_corpus = [wpt.tokenize(document) for document in x]\n",
    "\n",
    "w2v_model = word2vec.Word2Vec(tokenized_corpus, size=10, \n",
    "                                  window=10, min_count = 1,\n",
    "                                  sample=1e-3, iter=100)\n",
    "\n",
    "vocabulary = set(w2v_model.wv.index2word)\n",
    "\n",
    "def average_word_vectors(words, model, vocabulary, num_features):\n",
    "    feature_vector = np.zeros((num_features,),dtype=\"float64\")\n",
    "    nwords = 0.\n",
    "\n",
    "    for word in words:\n",
    "        if word in vocabulary: \n",
    "            nwords = nwords + 1.\n",
    "            feature_vector = np.add(feature_vector, model[word])\n",
    "    \n",
    "    if nwords:\n",
    "        feature_vector = np.divide(feature_vector, nwords)\n",
    "        \n",
    "    return feature_vector\n",
    "\n",
    "features = [average_word_vectors(tokenized_sentence, w2v_model, vocabulary, 10)\n",
    "                    for tokenized_sentence in tokenized_corpus]\n",
    "\n",
    "# word embeddings\n",
    "print(\"W2V:\")\n",
    "w2v_feature_array = np.array(features)\n",
    "km = KMeans(n_clusters=2)\n",
    "kmeans = km.fit(w2v_feature_array)\n",
    "print(\"Done\")\n",
    "\n",
    "word_vectors = w2v_feature_array\n",
    "idx = km.fit_predict(word_vectors);\n",
    "centroid_map = dict(zip(w2v_model.wv.index2word, idx))\n",
    "\n",
    "def get_top_words(index2word, k, centers, wordvecs):\n",
    "    tree = KDTree(wordvecs);\n",
    "#Closest points for each Cluster center is used to query the closest 20 points to it.\n",
    "    closest_points = [tree.query(np.reshape(x, (1, -1)), k=k) for x in centers];\n",
    "    closest_words_idxs = [x[1] for x in closest_points];\n",
    "#Word Index is queried for each position in the above array, and added to a Dictionary.\n",
    "    closest_words = {};\n",
    "    for i in range(0, len(closest_words_idxs)):\n",
    "        closest_words['Cluster #' + str(i)] = [index2word[j] for j in closest_words_idxs[i][0]]\n",
    "#A DataFrame is generated from the dictionary.\n",
    "    df = pandas.DataFrame(closest_words);\n",
    "    df.index = df.index+1\n",
    "    return df\n",
    "\n",
    "centers = km.cluster_centers_\n",
    "top_words = get_top_words(w2v_model.wv.index2word, 10, centers, word_vectors)\n",
    "top_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = Yelp_load_data()\n",
    "df = textcleaning(df, lower=1, punctuation=1, stemming=0, lemmatization=0, commonwords=1, rarewords=1, stopword=1)\n",
    "x = df['text']\n",
    "y = df['label']\n",
    "\n",
    "print(\"Glove:\")\n",
    "model=emb_glove()\n",
    "glove_feature_array = averaged_word_vectorizer(corpus=tokenizecorpus(x), model = model, num_features=300)\n",
    "km = KMeans(n_clusters=2)\n",
    "kmeans = km.fit(glove_feature_array)\n",
    "print(\"Done\")\n",
    "\n",
    "word_vectors = glove_feature_array\n",
    "idx = km.fit_predict(word_vectors);\n",
    "centroid_map = dict(zip(model.wv.index2word, idx))\n",
    "\n",
    "def get_top_words(index2word, k, centers, wordvecs):\n",
    "    tree = KDTree(wordvecs);\n",
    "#Closest points for each Cluster center is used to query the closest k points to it.\n",
    "    closest_points = [tree.query(np.reshape(x, (1, -1)), k=k) for x in centers];\n",
    "    closest_words_idxs = [x[1] for x in closest_points];\n",
    "#Word Index is queried for each position in the above array, and added to a Dictionary.\n",
    "    closest_words = {};\n",
    "    for i in range(0, len(closest_words_idxs)):\n",
    "        closest_words['Cluster #' + str(i)] = [index2word[j] for j in closest_words_idxs[i][0]]\n",
    "#A DataFrame is generated from the dictionary.\n",
    "    df = pandas.DataFrame(closest_words);\n",
    "    df.index = df.index+1\n",
    "    return df\n",
    "\n",
    "centers = km.cluster_centers_\n",
    "top_words = get_top_words(model.wv.index2word, 10, centers, word_vectors)\n",
    "top_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results Amazon data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def use_amazondata(r=1, lower=0, punctuation=0, stemming=0, lemmatization=0, commonwords=0, rarewords=0, stopword=0):\n",
    "    measures = ['accuracy', 'ASW', 'ARI', 'true ASW', 'AMI', 'NMI', 'H', 'C']\n",
    "    #total_eval = pandas.DataFrame(index = measures)\n",
    "    \n",
    "    df = Amazon_load_data()\n",
    "    df = textcleaning(df, lower=lower, punctuation=punctuation, stemming=stemming, lemmatization=lemmatization, commonwords=commonwords, rarewords=rarewords, stopword=stopword)\n",
    "    x = df['text']\n",
    "    y = df['label']\n",
    "    \n",
    "    print (\"WordLevel TF-IDF: \")\n",
    "    x_tfidf = wordtfidf(x)\n",
    "    col = ['mean_tfidf', 'std_tfidf']\n",
    "    evalsdf = pandas.DataFrame(index = measures, columns=col)\n",
    "    evalsdf['mean_tfidf'], evalsdf['std_tfidf'], df['preds_1'] = k_means(x_tfidf, y, 2, repeat = r)\n",
    "    #total_eval = pandas.concat([total_eval, evalsdf], axis=1)\n",
    "    \n",
    "    \n",
    "    print (\"N-Gram Vectors: \")\n",
    "    x_tfidf_ngram = ngramtfidf(x)\n",
    "    col = ['mean_tfidf_ngram', 'std_tfidf_ngram']\n",
    "    evalsdf = pandas.DataFrame(index = measures, columns=col)\n",
    "    evalsdf['mean_tfidf_ngram'], evalsdf['std_tfidf_ngram'], df['preds_2'] = k_means(x_tfidf_ngram, y, 2, repeat = r)\n",
    "    #total_eval = pandas.concat([total_eval, evalsdf], axis=1)\n",
    "    \n",
    "    # word embeddings\n",
    "    print(\"W2V:\")\n",
    "    w2v_feature_array = averaged_word_vectorizer(corpus=tokenizecorpus(x), model=emb_W2V(tokenizecorpus(x)), num_features=10)\n",
    "    col = ['mean_w2v', 'std_w2v']\n",
    "    evalsdf = pandas.DataFrame(index = measures, columns=col)\n",
    "    evalsdf['mean_w2v'], evalsdf['std_w2v'], df['preds_3'] = k_means(w2v_feature_array, y, 2, repeat=r)\n",
    "    #total_eval = pandas.concat([total_eval, evalsdf], axis=1)\n",
    "    \n",
    "    print(\"Glove:\")\n",
    "    glove_feature_array = averaged_word_vectorizer(corpus=tokenizecorpus(x), model=emb_glove(), num_features=300)\n",
    "    col = ['mean_glove', 'std_glove']\n",
    "    evalsdf = pandas.DataFrame(index = measures, columns=col)\n",
    "    evalsdf['mean_glove'], evalsdf['std_glove'], df['preds_4'] = k_means(glove_feature_array, y, 2, repeat=r)\n",
    "    #total_eval = pandas.concat([total_eval, evalsdf], axis=1)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#level 1 amazon data\n",
    "df_1 = use_amazondata(r=1, lower=0, punctuation=0, stemming=0, lemmatization=0, commonwords=0, rarewords=0, stopword=0)\n",
    "df_1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1.to_csv('AMZ_1_labels.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#level 2 amazon data\n",
    "df_2 = use_amazondata(r=1, lower=1, punctuation=1, stemming=0, lemmatization=0, commonwords=0, rarewords=0, stopword=0)\n",
    "df_2.to_csv('AMZ_2_labels.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_3 = use_amazondata(r=1, lower=1, punctuation=1, stemming=0, lemmatization=0, commonwords=1, rarewords=1, stopword=0)\n",
    "df_3.to_csv('AMZ_3_labels.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_4 = use_amazondata(r=1, lower=1, punctuation=1, stemming=0, lemmatization=0, commonwords=1, rarewords=1, stopword=1)\n",
    "df_4.to_csv('AMZ_4_labels.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# level 5\n",
    "df_5 = use_amazondata(r=1, lower=1, punctuation=1, stemming=1, lemmatization=0, commonwords=1, rarewords=1, stopword=1)\n",
    "df_5.to_csv('AMZ_5_labels.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# level 6\n",
    "df_6 = use_amazondata(r=1, lower=1, punctuation=1, stemming=0, lemmatization=1, commonwords=1, rarewords=1, stopword=1)\n",
    "df_6.to_csv('AMZ_6_labels.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results Yelp Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = Yelp_load_data()\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def use_yelpdata(r=1, lower=0, punctuation=0, stemming=0, lemmatization=0, commonwords=0, rarewords=0, stopword=0):\n",
    "    measures = ['accuracy', 'ASW', 'ARI', 'true ASW', 'AMI', 'NMI', 'H', 'C']\n",
    "    total_eval = pandas.DataFrame(index = measures)\n",
    "    \n",
    "    df = Yelp_load_data()\n",
    "    df = textcleaning(df, lower=lower, punctuation=punctuation, stemming=stemming, lemmatization=lemmatization, commonwords=commonwords, rarewords=rarewords, stopword=stopword)\n",
    "    x = df['text']\n",
    "    y = df['label']\n",
    "    \n",
    "    print (\"WordLevel TF-IDF: \")\n",
    "    x_tfidf = wordtfidf(x)\n",
    "    col = ['mean_tfidf', 'std_tfidf']\n",
    "    evalsdf = pandas.DataFrame(index = measures, columns=col)\n",
    "    evalsdf['mean_tfidf'], evalsdf['std_tfidf'] = k_means(x_tfidf, y, 2, repeat = r)\n",
    "    total_eval = pandas.concat([total_eval, evalsdf], axis=1)\n",
    "    \n",
    "    print (\"N-Gram Vectors: \")\n",
    "    x_tfidf_ngram = ngramtfidf(x)\n",
    "    col = ['mean_tfidf_ngram', 'std_tfidf_ngram']\n",
    "    evalsdf = pandas.DataFrame(index = measures, columns=col)\n",
    "    evalsdf['mean_tfidf_ngram'], evalsdf['std_tfidf_ngram'] = k_means(x_tfidf_ngram, y, 2, repeat = r)\n",
    "    total_eval = pandas.concat([total_eval, evalsdf], axis=1)\n",
    "    \n",
    "    # word embeddings\n",
    "    print(\"W2V:\")\n",
    "    w2v_feature_array = averaged_word_vectorizer(corpus=tokenizecorpus(x), model=emb_W2V(tokenizecorpus(x)), num_features=10)\n",
    "    col = ['mean_w2v', 'std_w2v']\n",
    "    evalsdf = pandas.DataFrame(index = measures, columns=col)\n",
    "    evalsdf['mean_w2v'], evalsdf['std_w2v'] = k_means(w2v_feature_array, y, 2, repeat=r)\n",
    "    total_eval = pandas.concat([total_eval, evalsdf], axis=1)\n",
    "    \n",
    "    print(\"Glove:\")\n",
    "    glove_feature_array = averaged_word_vectorizer(corpus=tokenizecorpus(x), model=emb_glove(), num_features=300)\n",
    "    col = ['mean_glove', 'std_glove']\n",
    "    evalsdf = pandas.DataFrame(index = measures, columns=col)\n",
    "    evalsdf['mean_glove'], evalsdf['std_glove'] = k_means(glove_feature_array, y, 2, repeat=r)\n",
    "    total_eval = pandas.concat([total_eval, evalsdf], axis=1)\n",
    "    \n",
    "    return total_eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#level 1 yelp\n",
    "level1yelp = use_yelpdata(r=10, lower=0, punctuation=0, stemming=0, lemmatization=0, commonwords=0, rarewords=0, stopword=0)\n",
    "level1yelp.to_csv('YELP_level1.csv', encoding='utf-8')\n",
    "level1yelp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#level 2\n",
    "level2yelp = use_yelpdata(r=10, lower=1, punctuation=1, stemming=0, lemmatization=0, commonwords=0, rarewords=0, stopword=0)\n",
    "level2yelp.to_csv('YELP_level2.csv', encoding='utf-8')\n",
    "level2yelp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#level 3\n",
    "level3yelp = use_yelpdata(r=10, lower=1, punctuation=1, stemming=0, lemmatization=0, commonwords=1, rarewords=1, stopword=0)\n",
    "level3yelp.to_csv('YELP_level3.csv', encoding='utf-8')\n",
    "level3yelp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#level 4\n",
    "level4yelp = use_yelpdata(r=10, lower=1, punctuation=1, stemming=0, lemmatization=0, commonwords=1, rarewords=1, stopword=1)\n",
    "level4yelp.to_csv('YELP_level4.csv', encoding='utf-8')\n",
    "level4yelp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#level 5\n",
    "level5yelp = use_yelpdata(r=10, lower=1, punctuation=1, stemming=1, lemmatization=0, commonwords=1, rarewords=1, stopword=1)\n",
    "level5yelp.to_csv('YELP_level5.csv', encoding='utf-8')\n",
    "level5yelp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#level 6\n",
    "level6yelp = use_yelpdata(r=10, lower=1, punctuation=1, stemming=0, lemmatization=1, commonwords=1, rarewords=1, stopword=1)\n",
    "level6yelp.to_csv('YELP_level6.csv', encoding='utf-8')\n",
    "level6yelp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results DBpedia data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = Dbpedia_load_data()\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def use_dbpediadata(r=1, lower=0, punctuation=0, stemming=0, lemmatization=0, commonwords=0, rarewords=0, stopword=0):\n",
    "    measures = ['accuracy', 'ASW', 'ARI', 'true ASW', 'AMI', 'NMI', 'H', 'C']\n",
    "    total_eval = pandas.DataFrame(index = measures)\n",
    "    \n",
    "    df = Dbpedia_load_data()\n",
    "    df = textcleaning(df, lower=lower, punctuation=punctuation, stemming=stemming, lemmatization=lemmatization, commonwords=commonwords, rarewords=rarewords, stopword=stopword)\n",
    "    x = df['text']\n",
    "    y = df['label']\n",
    "    \n",
    "    print (\"WordLevel TF-IDF: \")\n",
    "    x_tfidf = wordtfidf(x)\n",
    "    col = ['mean_tfidf', 'std_tfidf']\n",
    "    evalsdf = pandas.DataFrame(index = measures, columns=col)\n",
    "    evalsdf['mean_tfidf'], evalsdf['std_tfidf'] = k_means(x_tfidf, y, 2, repeat = r)\n",
    "    total_eval = pandas.concat([total_eval, evalsdf], axis=1)\n",
    "    \n",
    "    print (\"N-Gram Vectors: \")\n",
    "    x_tfidf_ngram = ngramtfidf(x)\n",
    "    col = ['mean_tfidf_ngram', 'std_tfidf_ngram']\n",
    "    evalsdf = pandas.DataFrame(index = measures, columns=col)\n",
    "    evalsdf['mean_tfidf_ngram'], evalsdf['std_tfidf_ngram'] = k_means(x_tfidf_ngram, y, 2, repeat = r)\n",
    "    total_eval = pandas.concat([total_eval, evalsdf], axis=1)\n",
    "    \n",
    "    # word embeddings\n",
    "    print(\"W2V:\")\n",
    "    w2v_feature_array = averaged_word_vectorizer(corpus=tokenizecorpus(x), model=emb_W2V(tokenizecorpus(x)), num_features=10)\n",
    "    col = ['mean_w2v', 'std_w2v']\n",
    "    evalsdf = pandas.DataFrame(index = measures, columns=col)\n",
    "    evalsdf['mean_w2v'], evalsdf['std_w2v'] = k_means(w2v_feature_array, y, 2, repeat=r)\n",
    "    total_eval = pandas.concat([total_eval, evalsdf], axis=1)\n",
    "    \n",
    "    print(\"Glove:\")\n",
    "    glove_feature_array = averaged_word_vectorizer(corpus=tokenizecorpus(x), model=emb_glove(), num_features=300)\n",
    "    col = ['mean_glove', 'std_glove']\n",
    "    evalsdf = pandas.DataFrame(index = measures, columns=col)\n",
    "    evalsdf['mean_glove'], evalsdf['std_glove'] = k_means(glove_feature_array, y, 2, repeat=r)\n",
    "    total_eval = pandas.concat([total_eval, evalsdf], axis=1)\n",
    "    \n",
    "    return total_eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#level 1 dbpedia data\n",
    "level1dbpedia = use_dbpediadata(r=10, lower=0, punctuation=0, stemming=0, lemmatization=0, commonwords=0, rarewords=0, stopword=0)\n",
    "level1dbpedia.to_csv('DBPEDIA_level1.csv', encoding='utf-8')\n",
    "level1dbpedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#level 2\n",
    "level2dbpedia = use_dbpediadata(r=10, lower=1, punctuation=1, stemming=0, lemmatization=0, commonwords=0, rarewords=0, stopword=0)\n",
    "level2dbpedia.to_csv('DBPEDIA_level2.csv', encoding='utf-8')\n",
    "level2dbpedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#level 3\n",
    "level3dbpedia = use_dbpediadata(r=10, lower=1, punctuation=1, stemming=0, lemmatization=0, commonwords=1, rarewords=1, stopword=0)\n",
    "level3dbpedia.to_csv('DBPEDIA_level3.csv', encoding='utf-8')\n",
    "level3dbpedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#level 4\n",
    "level4dbpedia = use_dbpediadata(r=10, lower=1, punctuation=1, stemming=0, lemmatization=0, commonwords=1, rarewords=1, stopword=1)\n",
    "level4dbpedia.to_csv('DBPEDIA_level4.csv', encoding='utf-8')\n",
    "level4dbpedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#level 5\n",
    "level5dbpedia = use_dbpediadata(r=10, lower=1, punctuation=1, stemming=1, lemmatization=0, commonwords=1, rarewords=1, stopword=1)\n",
    "level5dbpedia.to_csv('DBPEDIA_level5.csv', encoding='utf-8')\n",
    "level5dbpedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# level 6\n",
    "level6dbpedia = use_dbpediadata(r=10, lower=1, punctuation=1, stemming=0, lemmatization=1, commonwords=1, rarewords=1, stopword=1)\n",
    "level6dbpedia.to_csv('DBPEDIA_level6.csv', encoding='utf-8')\n",
    "level6dbpedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}