tokenizer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tokenize\n",
    "import pydriller as pyd\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "import ast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for commit in pyd.RepositoryMining(path_to_repo=path, since=tf_dt1, to=tf_dt2, only_authors=Authors).traverse_commits():\n",
    "    for modified_file in commit.modifications:\n",
    "        if modified_file.filename.endswith(\".py\"):\n",
    "            tf_source = tf_source.append({'commit_ID': commit.hash, 'Author': commit.author.name,\n",
    "                                          'msg': commit.msg,'Commit_before': modified_file.source_code_before,\n",
    "                                          'Commit_after': modified_file.source_code}, ignore_index=True)\n",
    "                #print(\"Code\", commit.hash, commit.author.name, commit.msg,commit.author_date,\n",
    "                        # modified_file.filename, flush=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1282,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FuncParser_onlycall(ast.NodeVisitor):\n",
    "    def visit_Import(self, node): \n",
    "        ast.NodeVisitor.generic_visit(self, node)\n",
    "    def visit_ImportFrom(self, node):\n",
    "        if(node.module != None):\n",
    "            n3= (node.module)\n",
    "            file_contents.append([n3])\n",
    "        ast.NodeVisitor.generic_visit(self, node)\n",
    "    def visit_alias(self, node): \n",
    "        n1 =  (node.name)\n",
    "        file_contents.append([n1])\n",
    "        if(node.asname != None):\n",
    "            n2 = (node.asname)\n",
    "            file_contents.append([n2])\n",
    "        ast.NodeVisitor.generic_visit(self, node)\n",
    "    def visit_Call(self, node): \n",
    "        attrib = node.func\n",
    "        if(attrib.__class__.__name__ == 'Attribute'):\n",
    "            file_contents.append([attrib.attr])\n",
    "    def generic_visit(self, node):\n",
    "        ast.NodeVisitor.generic_visit(self, node)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1423,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_word = {'ctx','value','name','values','kw_defaults','kwonlyargs','kwarg','n','body','args', 'defaults', 'vararg', 'arguments'}\n",
    "class FuncParser(ast.NodeVisitor):\n",
    "    def visit_Load(self, node): pass\n",
    "    def visit_Store(self, node): pass\n",
    "    def visit_Str(self, node): pass\n",
    "    def visit_Num(self, node): pass\n",
    "    def visit_Module(self, node): \n",
    "        ast.NodeVisitor.generic_visit(self, node)\n",
    "    def visit_arguments(self, node): \n",
    "        ast.NodeVisitor.generic_visit(self, node)\n",
    "    def visit_Expr(self, node): \n",
    "        ast.NodeVisitor.generic_visit(self, node)\n",
    "    def visit_Assign(self, node): \n",
    "        ast.NodeVisitor.generic_visit(self, node)\n",
    "    def visit_FunctionDef(self,node):\n",
    "        file_contents.append([node.__class__.__name__])\n",
    "        file_contents.append([node.name])\n",
    "        ast.NodeVisitor.generic_visit(self, node)\n",
    "    def visit_Call(self, node): \n",
    "        attrib = node.func\n",
    "        if(attrib.__class__.__name__ == 'Attribute'):\n",
    "            file_contents.append([attrib.attr])\n",
    "        elif (attrib.__class__.__name__ == 'Name'):\n",
    "            file_contents.append([attrib.id])\n",
    "    def visit_arg(self, node): \n",
    "        ast.NodeVisitor.generic_visit(self, node)\n",
    "    def visit_BinOp(self,node):\n",
    "        temp = (node.op)\n",
    "        op = \"BinOp: \"+temp.__class__.__name__ +\"()\"\n",
    "        file_contents.append([op])\n",
    "    def visit_Import(self, node): \n",
    "        ast.NodeVisitor.generic_visit(self, node)\n",
    "    def visit_ImportFrom(self, node):\n",
    "        if(node.module != None):\n",
    "            n3= (node.module)\n",
    "            file_contents.append([n3])\n",
    "        ast.NodeVisitor.generic_visit(self, node)\n",
    "    def visit_alias(self, node): \n",
    "        n1 =  (node.name)\n",
    "        file_contents.append([n1])\n",
    "        if(node.asname != None):\n",
    "            n2 =  (node.asname)\n",
    "            file_contents.append([n2])\n",
    "        ast.NodeVisitor.generic_visit(self, node)\n",
    "    def visit_Name(self, node):\n",
    "       # temp =\"Name:\"+node.id\n",
    "        #file_contents.append([node.id])\n",
    "        ast.NodeVisitor.generic_visit(self, node)\n",
    "    def visit_Attribute(self, node): \n",
    "        #temp1 =\"Name:\"+node.attr\n",
    "        file_contents.append([node.attr])\n",
    "        ast.NodeVisitor.generic_visit(self, node)\n",
    "    def generic_visit(self, node):\n",
    "        file_contents.append([node.__class__.__name__])\n",
    "        #for f in node._fields:\n",
    "           # if(not f in stop_word):     \n",
    "              #  file_contents.append([f])\n",
    "        #ast.iter_fields(node)\n",
    "        ast.NodeVisitor.generic_visit(self, node)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "226301"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#First we need to put the 'commit_before' and 'commit_after' into two different text file\n",
    "#becuase AST accept a string which is a read line by line of a code file. but while we store the commit in a cell of dataframe, it is not able to be read line by line\n",
    "text_before=tf_source['Commit_before'][1]\n",
    "text_after=tf_source['Commit_after'][1]\n",
    "before = open(\"before-commit.py\", \"w+\")\n",
    "after = open(\"after-commit.py\", \"w+\")\n",
    "before.write(text_before)\n",
    "after.write(text_after)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1424,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tokenize.open(\"co.py\") as sf:  # need the tokenize.open for source files and not a string\n",
    "    source_file_before = sf.read()\n",
    "\n",
    "ast_before = ast.parse(source_file_before)\n",
    "bf_obj = FuncParser()\n",
    "bf_tree = ast.parse(ast_before)\n",
    "file_contents = []\n",
    "bf_obj.visit(bf_tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1289,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Module\n",
      "FunctionDef\n",
      "arguments\n",
      "Assign\n",
      "If\n",
      "Return\n",
      "arg\n",
      "Name\n",
      "Num\n",
      "BoolOp\n",
      "Assign\n",
      "If\n",
      "Name\n",
      "Store\n",
      "Or\n",
      "Name\n",
      "Call\n",
      "Name\n",
      "Call\n",
      "Compare\n",
      "Assign\n",
      "Load\n",
      "Load\n",
      "Name\n",
      "Name\n",
      "Name\n",
      "Store\n",
      "Name\n",
      "Call\n",
      "Name\n",
      "Eq\n",
      "Num\n",
      "Name\n",
      "Call\n",
      "Load\n",
      "Load\n",
      "Load\n",
      "Load\n",
      "Attribute\n",
      "Load\n",
      "Store\n",
      "Name\n",
      "Name\n",
      "Name\n",
      "Load\n",
      "Load\n",
      "Load\n",
      "Load\n"
     ]
    }
   ],
   "source": [
    "for node in ast.walk(bf_tree):\n",
    "    print(node.__class__.__name__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1308,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Module(body=[ImportFrom(module='sklearn.feature_extraction.text', names=[alias(name='TfidfTransformer', asname=None)], level=0), Assign(targets=[Name(id='X_train_tokens', ctx=Store())], value=Call(func=Attribute(value=Name(id='data', ctx=Load()), attr='drop', ctx=Load()), args=[List(elts=[Str(s='commit_Author'), Str(s='commit_msg')], ctx=Load())], keywords=[keyword(arg='axis', value=Num(n=1))])), Assign(targets=[Name(id='tfidf_transformer', ctx=Store())], value=Call(func=Name(id='TfidfTransformer', ctx=Load()), args=[], keywords=[keyword(arg='use_idf', value=NameConstant(value=True))])), Expr(value=Call(func=Attribute(value=Name(id='tfidf_transformer', ctx=Load()), attr='fit_transform', ctx=Load()), args=[Name(id='X_train_tokens', ctx=Load())], keywords=[]))])\n"
     ]
    }
   ],
   "source": [
    "print(ast.dump(bf_tree))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1425,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['FunctionDef'],\n",
       " ['dict_keys_to_ordered_list'],\n",
       " ['If'],\n",
       " ['BoolOp'],\n",
       " ['Or'],\n",
       " ['isinstance'],\n",
       " ['list'],\n",
       " ['If'],\n",
       " ['Compare'],\n",
       " ['Eq'],\n",
       " ['try_sort'],\n",
       " ['BinOp: Add()'],\n",
       " ['Return']]"
      ]
     },
     "execution_count": 1425,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_contents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "with tokenize.open(\"after-commit.py\") as sf:  # need the tokenize.open for source files and not a string\n",
    "    source_file_after = sf.read()\n",
    "\n",
    "after = ast.parse(source_file_after)\n",
    "ast_after = ast.parse(source_file_after)\n",
    "aft_obj = FuncParser()\n",
    "aft_tree = ast.parse(ast_after)\n",
    "file_contents = []\n",
    "aft_obj.visit(aft_tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "diff = tokens_after.subtract(tokens_before)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>token</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [index, token]\n",
       "Index: []"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diff_token = diff[(diff.select_dtypes(include=['number']) != 0).any(1)]\n",
    "diff_token=diff_token.fillna(0)\n",
    "diff_token= diff_token.abs()\n",
    "diff_token = diff_token.reset_index()\n",
    "diff_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_author_token = pd.DataFrame(columns=['Author','msg', 'tokens', 'counts'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_author_token = df_author_token.append({'Author':tf_source['Author'][1],'msg':tf_source['msg'][1]\n",
    "                                         ,'tokens':diff_token['index'],'counts':diff_token['token']},ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [],
   "source": [
    "x=df_author_token['tokens'][0]\n",
    "y=df_author_token['counts'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Import:combinations': 1, 'Import:parameterized': 2, 'Import:test_util': 3, 'ImportFrom:absl.testing': 4}\n"
     ]
    }
   ],
   "source": [
    "vocab = {}\n",
    "i = 1\n",
    "for word in x:\n",
    "    if word in vocab:\n",
    "        continue\n",
    "    else:\n",
    "        vocab[word]=i\n",
    "        i+=1\n",
    "print(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Allen Lavoie', 0, 0, 0, 0]"
      ]
     },
     "execution_count": 237,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "one = [df_author_token['Author'][0]]+[0]*len(vocab)\n",
    "one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Allen Lavoie', 0.0, 0.0, 0.0, 0.0]"
      ]
     },
     "execution_count": 238,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# here I try to count the number of each vocab in diff_token of each commit per developer\n",
    "i=0\n",
    "for word in x:\n",
    "    one[vocab[word]]+=y[i]\n",
    "    i+=1\n",
    "    \n",
    "one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1426,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create a new list to repeat all above steps in a loop function\n",
    "df_author_token = pd.DataFrame(columns=['Author','msg', 'tokens', 'counts'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1427,
   "metadata": {},
   "outputs": [],
   "source": [
    "i=0\n",
    "error=0\n",
    "vocab = {}\n",
    "j = 1\n",
    "for row in tf_source.iterrows():\n",
    "    if tf_source['Commit_before'][i] is not None:\n",
    "        text_before=tf_source['Commit_before'][i]\n",
    "    if tf_source['Commit_after'][i] is not None:\n",
    "        text_after=tf_source['Commit_after'][i]\n",
    "    before = open(\"before-commit.py\", \"w+\")\n",
    "    after = open(\"after-commit.py\", \"w+\")\n",
    "    try:\n",
    "        before.write(text_before)\n",
    "    except ValueError:\n",
    "        #print(\"Oops!  Error in writing to the before file #\",i)\n",
    "        error+=1\n",
    "    try:\n",
    "        after.write(text_after)\n",
    "    except ValueError:\n",
    "        #print(\"Oops!  Error in writing to the After file #\",i)\n",
    "        error+=1\n",
    "    #this for commit before\n",
    "    with tokenize.open(\"before-commit.py\") as sf:  # need the tokenize.open for source files and not a string\n",
    "        try:\n",
    "            source_file_before = sf.read()\n",
    "            ast_before = ast.parse(source_file_before)\n",
    "        except ValueError:\n",
    "            error+=1\n",
    "            #print(\"Oops!  Error in before commit #\",i)       \n",
    "    bf_obj = FuncParser()\n",
    "    bf_tree = ast.parse(ast_before)\n",
    "    file_contents = []\n",
    "    bf_obj.visit(bf_tree)\n",
    "    #counting tokens in file before\n",
    "    dtobj_bfore = pd.DataFrame(file_contents, columns=['token'])\n",
    "    tokens_before =pd.DataFrame(dtobj_bfore['token'].value_counts())\n",
    "    #doing the same for file after\n",
    "    with tokenize.open(\"after-commit.py\") as sf:  # need the tokenize.open for source files and not a string\n",
    "        try:\n",
    "            source_file_after = sf.read()\n",
    "            ast_after = ast.parse(source_file_after)\n",
    "        except ValueError:\n",
    "            error+=1\n",
    "            #print(\"Oops!  Error in after commit #\",i)\n",
    "    aft_obj = FuncParser()\n",
    "    aft_tree = ast.parse(ast_after)\n",
    "    file_contents = []\n",
    "    aft_obj.visit(aft_tree)\n",
    "    #counting\n",
    "    dtobj_after = pd.DataFrame(file_contents, columns=['token'])\n",
    "    tokens_after =pd.DataFrame(dtobj_after['token'].value_counts())\n",
    "    #calculating the difference\n",
    "    diff = tokens_after.subtract(tokens_before)\n",
    "    diff_token = diff[(diff.select_dtypes(include=['number']) != 0).any(1)]\n",
    "    diff_token=diff_token.fillna(0)\n",
    "    diff_token= diff_token.abs()\n",
    "    diff_token = diff_token.reset_index()\n",
    "    df_author_token = df_author_token.append({'Author':tf_source['Author'][i],'msg':tf_source['msg'][i]\n",
    "                                         ,'tokens':diff_token['index'],'counts':diff_token['token']},ignore_index=True)\n",
    "    i+=1\n",
    "    for word in diff_token['index']:\n",
    "        if word in vocab:\n",
    "            continue\n",
    "        else:\n",
    "            vocab[word]=j\n",
    "            j+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1428,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7039, 4)"
      ]
     },
     "execution_count": 1428,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_author_token.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1429,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24694"
      ]
     },
     "execution_count": 1429,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#number of vocab\n",
    "j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1430,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1432,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating an empty matrix in size of all commits and number of tokens and try to import the number of each token per commit and developer into a new dataset\n",
    "vocab_range = len(vocab)+1\n",
    "cv_token = pd.DataFrame(0, index=np.arange(len(df_author_token)), columns=vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1433,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_token['commit_Author'] =0\n",
    "cv_token['commit_msg']=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    " #vocab['orelse']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x=0\n",
    "for row in df_author_token.iterrows():\n",
    "    y=0\n",
    "    temp_token = df_author_token['tokens'][x]\n",
    "    temp_count = df_author_token['counts'][x]\n",
    "    for word in temp_token:\n",
    "        if temp_count[y] == 0:\n",
    "            cv_token[word][x]+=1\n",
    "            y+=1\n",
    "        else:\n",
    "            cv_token[word][x]+=temp_count[y]\n",
    "            y+=1\n",
    "    cv_token['commit_Author'][x]=df_author_token['Author'][x]\n",
    "    cv_token['commit_msg'][x]=df_author_token['msg'][x]\n",
    "    x+=1\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1636,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_token =  pd.read_csv('tf_calls_tokens.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1637,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_tokens  = cv_token.drop(['commit_Author','commit_msg'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1640,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7039, 2062)"
      ]
     },
     "execution_count": 1640,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_token_all.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1638,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_token_all=X_train_tokens.loc[:, (X_train_tokens.astype(bool).sum(axis=0)>4)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1641,
   "metadata": {},
   "outputs": [],
   "source": [
    "test=train_token_all.columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1624,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['absl.testing', 'combinations', 'generate', ...,\n",
       "       '_readAndCheckGraphExecutionTracesFile', '_readAndCheckGraphsFile',\n",
       "       '_readAndCheckSourceFilesAndStackFrames'], dtype=object)"
      ]
     },
     "execution_count": 1624,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1642,
   "metadata": {},
   "outputs": [],
   "source": [
    "#list_all_vocab = list(train_token_all.columns.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1643,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_transformer = TfidfTransformer()\n",
    "X_train_tfidf=tfidf_transformer.fit_transform(train_token_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1644,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_train_tfidf=  pd.DataFrame(X_train_tfidf.toarray(),columns=test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1647,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_train_tfidf['commit_Author'] = cv_token['commit_Author']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1649,
   "metadata": {},
   "outputs": [],
   "source": [
    "#all_train_tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1136,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_train_tfidf[all_train_tfidf>=0.0001] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1137,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_train_tfidf[all_train_tfidf<0.0001] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7039, 2062)"
      ]
     },
     "execution_count": 1142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_train_tfidf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1620,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_train_tfidf =  pd.read_csv('tf_calls_tokens.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1650,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_train_tfidf_cl= all_train_tfidf.loc[:, (all_train_tfidf.astype(bool).sum(axis=0)>5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1651,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7039, 1759)"
      ]
     },
     "execution_count": 1651,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_train_tfidf_cl.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_token_cl=X_train_tokens.loc[:, (X_train_tokens.astype(bool).sum(axis=0)>5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X_train_tokens  = cv_token.drop(['commit_Author','commit_msg'], axis=1)\n",
    "from gensim.models import LogEntropyModel\n",
    "from math import log, e"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1652,
   "metadata": {},
   "outputs": [],
   "source": [
    "Author_groupby=all_train_tfidf_cl.groupby(['commit_Author']).sum().reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1660,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_tokens  = Author_groupby.drop(['commit_Author'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1661,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(31, 1758)"
      ]
     },
     "execution_count": 1661,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_tokens.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1662,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_token_cl=X_train_tokens.loc[:, (X_train_tokens.astype(bool).sum(axis=0)>2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1663,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(31, 1141)"
      ]
     },
     "execution_count": 1663,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_token_cl.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1664,
   "metadata": {},
   "outputs": [],
   "source": [
    "#list_cl_vocab = list(train_token_cl.columns.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1569,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_transformer = TfidfTransformer()\n",
    "X_train_tfidf=tfidf_transformer.fit_transform(train_token_cl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1571,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.003860732433599646"
      ]
     },
     "execution_count": 1571,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_tfidf.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1572,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt_train_tfidf=  pd.DataFrame(X_train_tfidf.toarray(),columns=list_cl_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1678,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt_all = pd.concat([train_token_cl, dt_train_tfidf_all])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 557,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mlxtend.frequent_patterns import apriori, association_rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1768,
   "metadata": {},
   "outputs": [],
   "source": [
    "frequent_itemsets = frequent_itemsets.reset_index()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 450,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pandas_entropy(column, base):\n",
    "    vc = pd.Series(column).value_counts(normalize=True, sort=False)\n",
    "    return -(vc * np.log(vc)/np.log(base)).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 451,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     1.347426\n",
       "1     3.290522\n",
       "2     1.003908\n",
       "3     0.392948\n",
       "4     0.478578\n",
       "5     2.345980\n",
       "6     1.137267\n",
       "7     0.413622\n",
       "8     0.404613\n",
       "9     1.785008\n",
       "10    5.123702\n",
       "11    0.212164\n",
       "12    0.490156\n",
       "13    2.657800\n",
       "14    1.858087\n",
       "15    0.067392\n",
       "16    0.190851\n",
       "17    1.601309\n",
       "18    0.735731\n",
       "19    1.136205\n",
       "20    0.792648\n",
       "21    0.972891\n",
       "22    0.347572\n",
       "23    0.859877\n",
       "24    1.741171\n",
       "25    0.407716\n",
       "26    0.327281\n",
       "27    1.079233\n",
       "28    1.023904\n",
       "29    0.873709\n",
       "30    1.524509\n",
       "dtype: float64"
      ]
     },
     "execution_count": 451,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#model = LogEntropyModel(train_token_cl)\n",
    "train_token_cl.apply(lambda x: pandas_entropy(x,2), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1003,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(num_cm,columns=['num-cm'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1007,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "All_LDA_tokens = cv_token.drop(['commit_Author', 'commit_msg'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "from sklearn.feature_extraction.text import TfidfTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "LDA_model_ast = LatentDirichletAllocation(n_components=5,random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7039, 9243)"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_transformer = TfidfTransformer()\n",
    "\n",
    "X_train_tfidf = tfidf_transformer.fit_transform(All_LDA_tokens)\n",
    "X_train_tfidf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LatentDirichletAllocation(batch_size=128, doc_topic_prior=None,\n",
       "                          evaluate_every=-1, learning_decay=0.7,\n",
       "                          learning_method='batch', learning_offset=10.0,\n",
       "                          max_doc_update_iter=100, max_iter=10,\n",
       "                          mean_change_tol=0.001, n_components=5, n_jobs=None,\n",
       "                          perp_tol=0.1, random_state=42, topic_word_prior=None,\n",
       "                          total_samples=1000000.0, verbose=0)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LDA_model_ast.fit(All_LDA_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "test=All_LDA_tokens.columns.values\n",
    "#list_vocab = list(vocab)\n",
    "#len(list_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9243"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "AST_domain_results = LDA_model_ast.transform(All_LDA_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_token['domain'] = AST_domain_results.argmax(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "AST_Author_Domain=cv_token[['commit_Author','domain']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7039"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(AST_Author_Domain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "AST_Author_groupby=AST_Author_Domain.groupby(['domain']).size().reset_index(name='count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_per=((AST_Author_groupby)/len(AST_Author_Domain))*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}