git_issue_data_collection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests \n",
    "from github import Github\n",
    "from github import RateLimitExceededException\n",
    "from github import UnknownObjectException\n",
    "from datetime import datetime, timedelta\n",
    "import pickle\n",
    "import time\n",
    "import itertools\n",
    "import os\n",
    "import calendar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_roles = pd.read_csv('..\\dev_name_users.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect Issue dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GitIssueScrapper:\n",
    "\n",
    "    def __init__(self):\n",
    "        auth_git = #put your github authentication key here\n",
    "        self.g = Github(auth_git)\n",
    "\n",
    "    def issue_info_extracter(self, users):\n",
    "\n",
    "        #users_iter = iter(users)\n",
    "\n",
    "        for u in users:\n",
    "            try:\n",
    "                user = self.g.get_user(u)\n",
    "                try:\n",
    "                    name = user.name\n",
    "\n",
    "                except:\n",
    "                    name = \"none\"\n",
    "                    \n",
    "                try:\n",
    "                    u_id = user.login\n",
    "\n",
    "                except:\n",
    "                    u_id = \"none\"\n",
    "\n",
    "\n",
    "                try:\n",
    "                    reps = user.get_repos()\n",
    "                except:\n",
    "                    reps = \"none\"\n",
    "\n",
    "                all_issue_repo={}\n",
    "                counter=0\n",
    "                for repository in reps:\n",
    "                    full_issue={}\n",
    "                    cnt=0\n",
    "                    try:\n",
    "                        repo_parent = repository.parent\n",
    "                    except:\n",
    "                        repo_parent = \"None\"\n",
    "\n",
    "         \n",
    "\n",
    "                    try:\n",
    "                        if repo_parent:\n",
    "                            mention_issues=repository.parent.get_issues(mentioned=user,state='all')\n",
    "                            create_issues=repository.parent.get_issues(creator=user,state='all')\n",
    "                            assigne_issues=repository.parent.get_issues(assignee=user,state='all')\n",
    "\n",
    "                        else:\n",
    "                            mention_issues=repository.get_issues(mentioned=user,state='all')\n",
    "                            create_issues=repository.get_issues(creator=user,state='all')\n",
    "                            assigne_issues=repository.get_issues(assignee=user,state='all')\n",
    "                    except:\n",
    "                        issues = []\n",
    "\n",
    "                    for issue in mention_issues:\n",
    "                        full_issue[cnt] = {'title':issue.title,'body':issue.body}\n",
    "                        cnt += 1\n",
    "    \n",
    "                    for issue in create_issues:\n",
    "                        full_issue[cnt] = {'title':issue.title,'body':issue.body}\n",
    "                        cnt += 1\n",
    "    \n",
    "                    for issue in assigne_issues:\n",
    "                        full_issue[cnt] = {'title':issue.title,'body':issue.body}\n",
    "                        cnt += 1\n",
    "\n",
    "                    all_issue_repo[counter] = full_issue\n",
    "                    counter += 1\n",
    "\n",
    "                    \n",
    "                        \n",
    "    \n",
    "\n",
    "\n",
    "                \n",
    "                data = {}\n",
    "\n",
    "                data['name'] = name\n",
    "                data['u_id'] = u_id\n",
    "                data['issues'] = all_issue_repo\n",
    "\n",
    "                    \n",
    "        \n",
    "                pickle.dump(data, open(('../' + u_id + '.pickle'), 'wb'))\n",
    "            \n",
    "\n",
    "                print(u_id, \"-------------------> done\")\n",
    "\n",
    "            except RateLimitExceededException:\n",
    "                now = datetime.now()\n",
    "                current_time = now.strftime(\"%H:%M:%S\")\n",
    "                search_rate_limit = self.g.get_rate_limit().search\n",
    "                print('current time:{}'.format(current_time))\n",
    "                #time.sleep(sleep_time)\n",
    "                time.sleep(3600)\n",
    "                continue\n",
    "            except UnknownObjectException:\n",
    "                continue\n",
    "                \n",
    "    def get_issue_details(self):\n",
    "        # users = self.git.search_users('', location=self.location, language=self.language)\n",
    "        #users = self.git.search_users('', location='montreal')\n",
    "        self.issue_info_extracter(user_roles)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "milindhg -------------------> done\n",
      "behnammodi -------------------> done\n",
      "morganjunlin -------------------> done\n",
      "crisgon -------------------> done\n",
      "cswanghan -------------------> done\n",
      "nacarty -------------------> done\n",
      "highflyer910 -------------------> done\n",
      "chiragkyal -------------------> done\n",
      "abodesigner -------------------> done\n",
      "janikgar -------------------> done\n",
      "bdetweiler -------------------> done\n",
      "ronilaukkarinen -------------------> done\n",
      "liuwenzhuang -------------------> done\n",
      "sravi4701 -------------------> done\n",
      "joelloyd -------------------> done\n",
      "JoabMendes -------------------> done\n",
      "ronapelbaum -------------------> done\n",
      "stevetarver -------------------> done\n",
      "current time:09:36:33\n",
      "BlueMona -------------------> done\n",
      "iamtekeste -------------------> done\n",
      "alexpnt -------------------> done\n",
      "lumue -------------------> done\n",
      "bradleybossard -------------------> done\n",
      "SethClydesdale -------------------> done\n",
      "alicj -------------------> done\n",
      "yessky -------------------> done\n",
      "egig -------------------> done\n",
      "cwalsh -------------------> done\n",
      "yongsen -------------------> done\n",
      "Ritesh-Kumar-4946 -------------------> done\n",
      "current time:10:47:20\n",
      "PratikLunagaria -------------------> done\n",
      "naieem -------------------> done\n",
      "sanjogshrestha -------------------> done\n",
      "celis -------------------> done\n",
      "mesarikaya -------------------> done\n",
      "dnlsyfq -------------------> done\n",
      "DanielB3 -------------------> done\n",
      "janogale -------------------> done\n",
      "Rhadow -------------------> done\n",
      "current time:11:58:04\n",
      "juanignaciosl -------------------> done\n",
      "membersheep -------------------> done\n",
      "moki-daniel -------------------> done\n",
      "sdlambert -------------------> done\n",
      "sambgordon -------------------> done\n",
      "james-work-account -------------------> done\n",
      "josephmtinangi -------------------> done\n"
     ]
    }
   ],
   "source": [
    "gg = GitIssueScrapper()\n",
    "gg.get_issue_details()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}