Dev2vec_data_load_clean.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import os\n",
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Dev2vec:repos dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#the directory of pickle files\n",
    "usr_dir = r'..\\users'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load files in pickle format. There is a seperate pickle file per each developer.\n",
    "def open_file(file_path):\n",
    "    with open(file_path, \"rb\") as picklefile:\n",
    "        reader = pickle.load(picklefile)\n",
    "    return(reader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Seprate lines in text with \\n only (replace other newline tags with \\n)\n",
    "def newline(text):\n",
    "    return '\\n'.join([p for p in re.split(r'\\\\n|\\\\r|\\\\n\\\\n|\\\\r\\\\n|\\r\\n', text) if len(p) > 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#merge a list of readme files in a single string which are seperated with \\n \n",
    "def merg_RM(RM):\n",
    "    return '\\n '.join(RM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#merge to texts in one single text\n",
    "def merg_text(txt1,txt2):\n",
    "    return txt1 +\"\\n \"+txt2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "col_name= ['Name','u_id','bio','Repo_Name','ReadMe','tags','topics','textual_all']\n",
    "Corpus = pd.DataFrame(columns = col_name, dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "#loop over all files in a directory and collect required fields\n",
    "for filename in os.listdir(usr_dir):\n",
    "    file_path = os.path.join(usr_dir, filename)\n",
    "    reader = open_file(file_path) \n",
    "    bio   = newline(str(reader['bio']))\n",
    "    Name = str(reader['name'])\n",
    "    try:\n",
    "        u_id= str(reader['u_id'])\n",
    "    except:\n",
    "        u_id =\"none\"\n",
    "        \n",
    "    RM = []\n",
    "    repo_name=[]\n",
    "    repo_tags=[]\n",
    "    repo_topics=[]\n",
    "    repo_text=[]\n",
    "    if(len(reader['repository'])>4):\n",
    "        for row in range(0,len(reader['repository'])):\n",
    "            repo_name_tmp = str(reader['repository'][row]['repo_name'])\n",
    "            repo_RM_tmp = newline(str(reader['repository'][row]['readme']))\n",
    "            repo_tags_tmp = str(reader['repository'][row]['topics'])\n",
    "            repo_topics_tmp=newline(str(reader['repository'][row]['about']))\n",
    "        \n",
    "            repo_name.append(repo_name_tmp)\n",
    "            RM.append(repo_RM_tmp)\n",
    "            repo_tags.append(repo_tags_tmp)\n",
    "            repo_topics.append(repo_topics_tmp)   \n",
    "        \n",
    "            repo_text_tmp = merg_text(merg_text(repo_tags_tmp,repo_topics_tmp),merg_text(repo_name_tmp,repo_RM_tmp))\n",
    "            repo_text.append(repo_text_tmp)\n",
    "        \n",
    "        \n",
    "        \n",
    "        text_all = merg_RM(repo_text) \n",
    "        info_all = merg_text(bio,text_all)\n",
    "    \n",
    "    \n",
    "    Corpus = Corpus.append({'Name':Name,'u_id':u_id,'bio':bio,'Repo_Name':repo_name,'ReadMe':RM,'tags':repo_tags,\n",
    "                            'topics':repo_topics,'textual_all':info_all},ignore_index=True)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### We want to seperate single-label developers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt1=Corpus[['Name','u_id','textual_all']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt2 = pd.read_csv('..\\git_data_single_label.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt_count_lb = dt2['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Frontend         598\n",
       "Mobile           380\n",
       "Backend          197\n",
       "DataScientist    158\n",
       "DevOps           127\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt_count_lb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "hatches = ['XX','..','++','o','-']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVwAAADqCAYAAADu8v3wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO2dd3xUVfbAvycJhNCLgDSlptFEkKKwgqCAIiooTVGxt1UR3PWHBZTVxe6qaxcLuoo0FRuia8VFAamSSJMaitRAqCHn98d9A8MwMylM5r1J7vfzuZ9kzjv3vTvJvDP3nXvuOaKqWCwWi6X4iXN7ABaLxVJasAbXYrFYooQ1uBaLxRIlrMG1WCyWKGENrsVisUQJa3AtFoslSiS4PQCLxVI05s2bVyshIeE1oAV28uQF8oAlubm517Vt23ZLMAVrcC2WGCUhIeG1k08+Oa1mzZo74uLibEC9y+Tl5cmff/6ZvmnTpteAvsF07LeipUiISLyIzBeRT5zXjUTkZxFZLiITRaRskD5lReQNEVksIgtFpGvAsVdEZJmIZIpIf0f+VxFZIiKf+c4pIp1F5KkovVUv06JmzZrZ1th6g7i4OK1Zs+YuzBNHcJ0ojsdSsrgDyPB7/SjwtKo2A3YA1wbpcz2AqrYEzgWeFBHfZ/BeYIuqJgPpwHeO/DqgFTAf6CkiAtwPjI3s24lJ4qyx9RbO/yOkXbUG11JoRKQ+cAHwmvNagHOAyY7KW8DFQbqmA18DqOoWYCfQzjl2DfBP51ieqm7161cGKA8cAoYCn6nqjgi+JUsRiY+Pb5uamprua7///vtxTzZF4aGHHqq1e/fuiNmnevXqtdy4caPrLlTXB2CJSZ4B/gZUcl7XAHaqaq7zej1QL0i/hcBFIvI+0ABoCzQQkWXO8bGOm2ElcJuqbgaeAGYDvwGzgA+BXhF/RyWAevXqtczKyoqIwQOoUaNG7tatWxeG00lMTMzLzMxcGur4oUOHKFOmTKGv/fLLL9e+/vrrt1eqVCmv0J09jJ3hWgqFiPTBPPrP8xcHUQ32qDseY4znYoz2T0Au5ou/PjBLVU8H/ocxtKjqBFVto6pXAHcBzwK9RWSyiDzt55Io9WRlZZVV1XmBbfr06cuqVq2aO3369GX+vwfT9dfftm1bkSZkzz77bI3evXs3Puecc5p26dIlOS8vjxtvvLF+s2bNmicnJ6e/+uqr1QA++eSTSu3bt0/p1atX40aNGjXv27dvo7y8PP7xj3/U2rJlS5mzzz47uUOHDskAU6dOrXzaaaelpqenp/Xu3bvxrl274sB8yQwfPrxuenp6WnJycvr8+fPLAWzatCn+rLPOapaWlpY+ZMiQU72SpMt+WC2F5Sygr4isBt7HuBKeAaqKiO8GrQ9kBXZU1VxVHa6qp6nqRUBVYDmwDdgLTHNUJwGn+/cVkbrAGar6EXAfMBA4AHSP7NsrWXzyySeVhg4d2njChAmr+vTps7tPnz67J0yYsGro0KGNP/nkk0rh9Aty/gMHDsT53AnnnntuE5/8119/rfjee+/9MXv27GVvv/121cWLFydlZGT89vXXXy974IEH6q9Zs6YMQEZGRtK///3vdStWrPht7dq1iTNnzqx43333balVq9ah7777btnPP/+8bOPGjQmPPPJIne+//37Z0qVLM04//fS9Y8eOre271kknnZS7dOnSjGuuuebPcePG1Qa455576nbq1GlPRkbG0r59++7cuHFjxGb+J4I1uJZCoar/p6r1VbUhMAj4r6peDnwDXOqoXQV8FNhXRMqLSAXn93OBXFVdqmb6MR3o6qh2BwIfU8diFssAkjAz6DyMb9fi4G9EA42tTx7K6IbSD4fPpZCZmbl05syZK33yLl26ZNeuXfswwA8//FBpwIAB2xMSEmjQoEFuhw4d9vz444/lAVq2bJnTpEmTQ/Hx8TRv3nzvypUrjzOM3377bYWVK1eWa9++fWpqamr6+++/X2Pt2rVH9IYMGbIDoH379nvXrVuXCDB79uxK11xzzTaAQYMG7apcufLhgv8Viw/rw7VEir8D74vIPzARBa8DiEhfoJ2qPgDUAmaISB6wAbMA5t9/gog8A/wJDPMdEJE2AKo63xG9DiwG1gEPFuebijX8Z6fhjKe/0S2IfmEpX778Ed9ruMf5xMTEIwfj4+PJzc09zj2lqnTu3Dl7+vTpfwQ7R7ly5RQgISFB/fvHxXlvPum9EVliBlX9VlX7OL+vUtX2qtpUVS9T1QOO/GPH2KKqq1U1RVXTVLWHqq7xO9caVf2LqrZS1e6qutbv2HxVvdbv9TOq2lxVe/muYzFMmDBh1YUXXph84YUXJudnPH1Gt6D6ReXss8/ePXny5Oq5ublkZWUl/PLLLxW7dOmSE65PhQoVDvv8tF27ds2ZO3duxSVLliQC7N69O27RokWJ4fp37Nhx9/jx42sAfPDBB5Wzs7PjI/V+TgRrcC0WS7EydOjQnc2bN9+XlpbWvGvXrskPPvjg+lNOOSU3XJ+rrrpqa+/evZt16NAhuW7durkvv/zy6kGDBjVOTk5Ob9u2berixYvLhes/bty4rFmzZlVMT09PmzFjRpU6deocjOy7KhrildU7i8VSOBYuXLi6devWR+KV69Sp03LTpk0RWxyqW7fuwQ0bNiyO1PlKCwsXLjypdevWDYMdsz5ci6WEsHHjRmscPY51KVgsFkuUsAbXYrFYooQ1uBaLxRIlrMG1WCyWKGENrsVisUQJa3AtFkuREZG2F198cSPf60OHDlGtWrXW3bp1axqu31133VX3gQceqB0oX716dZlevXo1BrPVOL/zxBo2LMxiKSFEOj1jQeJwk5KS8n7//fekPXv2SMWKFXXatGmVa9eufaio12zYsOGhL774okCJc2IRa3BLOE5ZmvpAbUyiF19L8vsZj8nWlROkbQfWqaondupYQuNLz+gv27NnjwBUrFhRCysXkbYFuW737t13TZo0qeqwYcN2vPfee9X79++//aeffqoIsHnz5vjLL7+84dq1axOTkpLyXnnllTUdOnTYB7Bo0aLyHTt2TN64cWPZ22+/fdOIESO2/v7772X79OnTbPny5b/5XyM7Ozvu2muvPSUjIyPp8OHDcu+992ZdccUVOwv3F3Ifa3BLACJSGZPOsAVwCnCq8/MU4CRgI7AJY1TLYErWxANfAn9wNOtWhSDtZOBkEckGlmGSg69xWibwq6pGfP+9pWj4Zpq+3xs0aNAKYN26dYsKKy/oNYcOHbp99OjRdQYOHLgzIyOj/LXXXrvNZ3D/9re/1W3duvXer776auXHH39c6aqrrmrkS1iekZGRNG/evIzdu3fHt2nTJr1///67Ql1j1KhRdbp165Y9adKk1Vu3bo1v165dWt++fbMrV64cUwnKrcGNMUSkEsa4tsWUp2mPMax7gf8CP2Oyda11WpaqHvbr3xWTShHgWVX9Np/r+fSrAf/B1Ctr6Fz3aqCViKwB5jhtLrBAVfef4Fu1xAgdOnTYt379+sRXX321eo8ePY4xmr/88kulKVOmrADo27fv7htuuCFh27Zt8QC9e/feWbFiRa1YsWJup06dsn/44YcK7du33xvsGt9++23lGTNmVH322WdPBjhw4ICsWLGi7Omnnx5TnzNrcD2OU9HgdKA3cD7QEpOacB7GoPZyjh3GJO4OaUQd4zkJ6OaIJonIZYXRBy5T1Ql+OmWA5sAZQB/gISBJRBYBXwCfA3P8jb6l+PB3EVSsWFF9M9UTkReEXr167Rw9enSDL7/88vctW7YcsSvBcrWIiDo/A+Uhz6+qTJ48eUXr1q1jOjucjVLwICJSXUQGicg7mBnlB5gZ5gPASaraCVOw8Vqgn6p+7RjNyzBGtGuQc3blqMH8NlL6qnpIVRdgKjecCfQDqmOq8CZjDO6fIjJBRC51ZuiWKOHMII+zeoWV58fNN9+8dcSIEVnt27ff5y/v2LHj7jfeeKMGmKiDatWq5VavXj0P4PPPP6+6d+9e2bRpU/zs2bMrde7cOWTKxm7dumU/+eSTtfPyjAdh1qxZSYUdoxewBtcjiEiSiAwQkY8xftXLMUUTb8QUa/xYVWeq6v5AY+g7RyijGG19Vd2nql+p6mDgEkx1hq2YkucbRGSKiPQSEU/kKLWcOE2aNDl0//33bwmUP/roo1m//vpr+eTk5PR777233ptvvnkkiXibNm1yunfv3qxDhw5pI0eO3NiwYcOQ0Q3jxo3Lys3NldTU1PRmzZo1v++++4IVKfU8Nj2jizjlxdsCt2LKis8FJgDT/Bei/A2aIzrOGAac18v6FTG+4LWYmmZvAm+q6opgfS2hCUzPGOmwsDp16hzMysqyGcgKiU3P6DFEJBG4Avg/TBTAe0BLVV0fTF9VvxWRyzi62NUt3GJXrOiLSAtMKZ1ZIpIJvApMVNUix3GWZmzuWu9jXQpRREQqishdmNCqa4EmmNCrd0MZ25KMqi5R1RFAA0zl32uAZSJys4iEzehvscQi1uBGARE5SUQexPhmOwBjgGaY1f9uhFi48uvflaPRAiVOX1UPquo0VT0HGAIMBjaKyN12kc1SkrAGtxgRkWoi8gRmw0AdzCr+i8A/iXC0QEnRBxKBNGAE0AZYJSIPWsNrKQlYg1sMiEiCiNwCrMa4DVqo6g1APTwcXeAx/fGqOgTohNlosUxErnLiki2WmMRGKUQYEekBPA1sAd4FHsWb0QIxpS8i7YHXAAGuU9Wfg52zNBEYpWDxBuGiFOxsIUKISFMR+Qh4Gbgf6KGq4zHG5BunhTQ+cMzMz+ofr/sLcBrwMfCNiEwSkbqhzm2JDvHx8W1TU1PTU1JS0tPT09NmzpxZoSjn6d+/f8M33nijWqTH1759+5Tvv/++fKTPW1RsWNgJ4jzi3gmMAh4DBqhqTG8/9CqqmiciMzF/675AdxH5P+AVtY9q1K5du9WWLVvKROp8BUnPmJiYmOdLRjNlypTKo0aNqn/uuef+HqkxlDTsDPcEEJGGwLeY3VQdVPUxf2Pr9up/Cdfv6YjvAD4RkTqh+pUWtmzZUkZV5/m34cOHbxw+fPjGosgLu4li165d8VWqVMl1fo/r1KlTcnp6elpycnL6O++8U9Wn9/zzz9dITk5OT0lJSfdPXu7jjjvuqNu/f/+Ghw8f5ocffih/xhlnpDRv3jytc+fOzdasWVMGzMz15ptvrteyZcu0hg0btvjiiy8qgsl41qdPn8bJycnpF1xwQeP9+/eHTtDgAnaGWwScHWLXYGa0CcAlqroyQKcrAT5JJ/g/aMIYq18k/X6O7BdgvojcqqpTKMXcddddx7hZnn766aBfRIWVh+LAgQNxqamp6QcOHJCtW7eW+eyzz5YBlC9fPu/TTz9dUb169byNGzcmdOjQIXXIkCE7f/3113JPPPFEnf/973+ZderUyd28efMx27tvuumm+tnZ2fGTJk1afejQIbn99ttP+fTTT1fUrVs399VXX602cuTIepMmTVoNkJubK4sXL86YOHFilYceeqhur169lj3xxBO1kpKS8pYtW7b0559/TjrrrLPSC/N+ihs7wy0kInIyMB2zHfds4CJgYgyt/pdE/QswWcrGicjbIlIFS1TwuRT++OOP36ZNm7Z82LBhjfLy8sjLy5M777yzfnJycnq3bt2St2zZUnb9+vUJM2bMqHzhhRfuqFOnTi5A7dq1j2SRGzduXJ1du3bF/+c//1kTFxfHokWLEpcvX550zjnnJKempqY//vjjdbKyso64TC677LIdAGeeeWbO+vXrywL8+OOPFYcOHboNTNrI5OTkoOke3cLOcAuBiHTEZOl6E5Ol66AjPzIzc1RDrrb7bXO1+hHWB4Zikv1kikhPVS1wEu2SwlNPPZUVKXlhZ7s9evTI2bFjR8LGjRsTpkyZUmXbtm0JixcvzkhMTNR69eq13LdvX5yqHknPGMhpp52Ws2jRovKbN2+Or1279mFVlaZNm+5bsGBBZjD9cuXKKUBCQgKHDx8+4jqQMGke3cbOcAuIiAzDrJDfpKr3qV/JGS+t5pdy/c8xSYBeBb52XA6WKDF//vxyeXl51K5dO3fXrl3xJ5100qHExESdPn16JZ8/uFevXtkff/xx9U2bNsWDKcHj69+rV6/sESNGbOrZs2ezHTt2xLVq1Wr/9u3bE7766qsKYJKOz507N+yW786dO+955513qgPMmTOn3LJlyzwToQB2hpsvYhJsP4VZpDlbVTNcHpIlf/6L+XKcKiItgbGqGlOlWGIFnw8XTJLwF198cXVCQgLXXXfd9t69ezdt0aJFWvPmzfc2atRoP0C7du32jxgxYmOXLl1S4+LitEWLFnunTJmy2ne+a665Zkd2dnZcr169mn799dfL33///ZW33377Kbt3744/fPiw3HzzzZvbtWsXssrDyJEjtwwaNKhRcnJyevPmzfe2bNkyZI5dN7AbH8IgIidhXAinAVeo6ich9LrikQ0CVv+oPqbm2lQgC7haVfcE6x+rBG58cCMszHI8Nj1jERCRppgii5MwCzITPbQ6b/ULoI8xut2AlzApIC/QEpyVbfPmzaXOZx1rWB9uEEQkHRNfO05V/66q/8Wbq/NWPx99TC6GazBJz38UkePiPi2WaGFdCgGIyOnAp8DdqvpOwLGuxMCjtNUPqf8hsAc4R1WXBesTS9hcCt4knEvBGlw/RORMYBomEmFaCJ2uFLDSgdX3nj7QGBgL9FTVJeH6eh1rcL2J9eEWABE5B1Md9wpV/cLt8ViKB1UdLyL7gJkicr6qznd7TJbSg/XhAiLyF8yjZwIQMuTE7xHVi7kFrH4B9VX1PeAW4AvHhWSxRIVSb3DFFDKcDAzCBM3HRGUEq39i+o7L6CZgupgkRJYi4EvP2LRp0+YpKSnpY8aMqX348OGwfX7//feyL730UvX8zn348GGuvvrqBs2aNWuenJyc3qJFi7TMzMyQCXXefffdKqNGjTq5CG+DrVu3xo8bN66m7/Xq1avL9OrVq3FB9QuMqpbahileuBa43E/WFfgT6BpOZvVLhj5wG6YEUg23P4+FbQsWLFitqnN9rU6dOgcAjVSrW7fuAf/zB2tJSUmHfb+vX79+QadOnbLvvPPOrHB9pk+f/nvXrl135nful156aVXPnj235+bmzlXVuStWrFi4ZcuW+fn1K0rLzMxc1LRp032R0Hf+L8FtjtsfGrcaUA1YgolGCHlThrt5rX6J0d8J/A9IcvtzWZgWaHB9hrJfv35b+/XrtxXQHj167NAQRqNz5867wukDGqqvr/kbXFWd+9tvvy2qUqVK7uHDh+dmZmYuOv3003enpaXlpKWl5Xz55ZcZqjq3VatWeypWrJibkpKyd8yYMWtD6Y0ePXrdlVdeuTnYdSdNmrQsLS0tJzk5eW/Hjh2zVXXuv/71rz+GDh26RVXnbtiwYcF55523o3nz5jnNmzfPmTFjRoaqzh0+fHjWpZdeuvWMM87YXa9evQNjx45dq6pzL7jggu2JiYl5KSkpe2+44YZN/gZ1zpw5S1q0aJGTkpKyt1mzZnsXLVq0OFDff2zW4B5/w5UDvsOU5pYQOl05+m0f8ua1+iVCvxsmTncKEO/257OgLZjB7dev31bfa58R1RDGMj/9ohhcVZ1bqVKl3LVr1y7Izs7+NScnZ56qzl20aNHi5s2b52iQGW4ovRUrViysW7fugZSUlL3XXXfdph9//PE3dYxp7dq1D2ZkZCxS1bmbNm2arwEGt0+fPtu++OKLTFWdu2zZskWNGjXap47BPe200/bs3bt3XlZW1oIqVark7t+/f17gjNX/9ZVXXrn5hRdeWKWqc/ft2zdv9+7dx+n7t3AGt9RFKYiIYLJ9bQLuUufus5RqFBiGSX7zFCapuaWI+G6pgwcPyrXXXnvq0qVLk+Li4lizZk1iMP1Qek2aNDm0YsWKJdOnT6/09ddfVz7//PNT3n777ZU5OTlx7du3352amnoQjk3x6GPWrFmVly9fnuR7vWfPnvgdO3bEAZx33nk7k5KSNCkpKbd69eqH1q9fH9YOdurUKeeJJ56os379+rKDBg3a0bJlywNF/duUOoOLWZ3ujVlMCZrQJGB1G0JsK7X6JUcfs6DWD/hZRK5U1beD9fU6U6dOrdG/f/8jv3fp0iU7lG737t13Fka/ICxdurRsfHw89erVyx05cmTdWrVqHZoyZcofeXl5JCUltQ3W5+GHH64dSi8pKUkHDBiQPWDAgOzatWsfmjp1atXzzjsvW/JJwaiqzJ07N6NixYrHTagSExOPyOLj48nNzQ17sptuuml7ly5dcqZNm1ald+/eyS+88MLqlJSUIhndUhWlICJnAKMxRvddL66eW3339DFJivoDT4rJMhZzdOnSJXvq1Kk1pk6dWqNHjx47v/nmm+WhdGfMmLGyMPr5kZWVlXD99defOmzYsC1xcXHs2rUrvk6dOofi4+N54YUXaviiF6pUqXJ4z5498b5+ofR+/PHH8qtXry4DJmJh8eLFSaeeeurBbt265fz888+VfBELgVUjADp37pz96KOP1vK9/umnn5ICdfypUqXK4ZycnKD2cOnSpWXT0tIO3HfffVvOO++8nQsWLEgKpx8Wt/1Q0WqYRbI/gP561Ifn2dVzq++ePnA5JnKhstuf23AtmA9Xg/gUi9oKcr64uDhNSUnZ26RJk33Jycl777///nW+qIJFixYtbtas2d5WrVrtueWWWzb6/L379++f17Fjx+zk5OS9Y8aMWRtKb9KkScvS09NzmjZtuq9p06b7Lr300q0+X+/EiROXpaam7k1OTt7bqVOnXRrgw83Kylpw/vnnb2/WrNnexo0b7xs8ePAWdXy4999//zrf+Js2bbovMzNzkTp+36ZNm+4LXDS755571jdp0mRfSkrK3s6dO+/y+Yz99f3/JuF8uKVia6/jt/0IWKWqd/rJu+Ltvf9W3z39O4FcR9+TN0ng1t569eq1LGzhx3DY9IxFo9TnUhCRu4FLgS7qV6nBOdYVD+/9t/qu6q8EHlKP+nNtLgVvEs7glngfroi0Bu4GBgQaW4slHx7G+HND7jiyWApDiTa4YsrjvAX8XVXXBDnelRjY+2/1XdN/DBOb+4LktyxusRSAEm1wgQeBUzGLZccQS6vnVt9V/f5AI2BwYB+LpbCUWIMrpkTO34Ak4GP/myzw5vLJQ92UVr/U69cCnhaRfBOuWCzhKJGLZs7j3xfATCAZuA44CAwAsvHWarjVjw39D4GpqnpNMB03sItm3qTURSmIyEDgFaC+qu4WU6Psc0x2MMF7q+FW3/v652MM8/mq+l043WjhBYMbHx/ftlmzZvtyc3MlPj5eBw8evO3+++/fHB9/3F6EAjFjxoyKI0eObLBnz544gFtvvXXzyJEjY+pLpVRVfBCRysALQB4mjhJVXSoiqcBWjIvhHRHpqqor3BupJcbYC5QHJohIY1XNdXtAgbgRh5uYmJiXmZm5FGDDhg0Jl112WeNdu3bFP/3001mFvd7atWsTrr766kaTJk1a2blz570bN25M6NGjR7P69esfGjRo0K6ivg8vURJ9uHdg3Al1VXWfT+j8fhJmAWQb8LuIPB9s9TkGVs+tvnv6W4ChoXTdJCsrq6yqzotUK6zxrlevXu5rr722+o033qiVl5dHbm4uN954Y/0WLVqkJScnpz/++OMnAVxwwQWNJ06cWMXXr3///g3ffPPNqk8++WStgQMHbuvcufNegDp16uQ+8sgj6x9//PGTfXpDhgw5pW3btikNGzZs8d5771UBmDt3brmWLVumpaampicnJ6cvXrw4aJIcL1CiDK6IVAVuB8b4G1sfqrpPVdeoamuMX/c6YLOY4pG+c3QlNlbPrb4L+pjP10Mi4tmb2k3S09MP5uXlsWHDhoRnnnnmpCpVqhxesmRJxsKFCzPeeuutmpmZmWUHDhy4feLEidUA9u/fL7Nmzap86aWX7srIyEhq167dXv/zde7cee+KFSuO5EFYt25d4i+//PL79OnTl995552n7t27V5577rmat9xyy+bMzMylixYtymjUqJFn4+1LlMHF3Ayfqmq+CThU9Q1MfoUFwI8iMlVEehBbq+dWP8r6qvoTsBC4AUtQfOtCX331VeUPPvigRmpqanqbNm3SduzYkbB06dJyl1566a6ffvqp8r59+2Ty5MlV2rdvv7tixYqqqojIcYtK/g+h/fv33x4fH0/Lli0PNGjQ4MCCBQvKderUKefJJ5+sc++99568fPnyssEyhHmFEuPDdWa3fwU6BciT4IhLIZj8PBE5D1Oxty9mK+e3gedX1W9FxHdT5ru6bfVLtP59wOciMl5VcwLPVZrxT8+oqvLkk0+u7d+//3EpHzt27Lh76tSplSdOnFht8ODB2wHS0tL2zZkzp8Lll19+xF87a9as8k2aNDly7wZ6AEUkaPrEvn377i7Gt1lkSozBBUZg3s8Gn8Axquuc3xv4jG4Q+ZciUgOTmPxBEekN9FbVnf4X8LspC7S6bfVLpr6qLhCR7zBPVP8MdT43uOuuu+q6de3A9IznnnvurhdffLFmnz59dicmJuqiRYsSGzZseKhy5cp5gwYN2v7666+ftHjx4gqTJk1aDTBixIg/O3bsmDpgwIAdZ5555r5NmzbFjxo1qv6oUaOOLMBNnTq12m233bYtMzMzcd26dYmtW7fe70uf2Lx58y2rVq1KXLBgQZI1uMWIM7sdDhwIoRLqEeOIXFUPi8gNwAVAW2CLiNytqv+K6GAtJYUHgFki8oKqlogV9KJw4MCBuNTU1HRfWNjAgQO3jR49ejPA8OHDt65evTqxZcuWaaoq1atXP/TZZ5+tBLjkkkuyb7rppkY9evTYWa5cOQU49dRTD40fP/6PG264oWFOTk6cqsrNN9+8eciQIUf+vk2bNj3Qvn37lG3btpV55pln1pQvX14nTJhQfdKkSTUSEhK0Zs2ah/75z38WOkIiWpSIOFwReRBoCNwUxnVQKDnmsfHvwAqgl6qu9vfpOTpeCMi3+u7pz8SsGTwT7HhxU9rSM/bv379hnz59dg0bNmyH22MJR4mOw3Vmt7cC7QONJxxvUGHPlkIAACAASURBVAshv1dEXsCEmK0UkcnAOfjdgP4+v8CbMtiCi9UvcfrtgMYi8qyGKNcUTbxkHC3BKQlRClcAX6nqqkifWFU3qGpLjEG/BLNLbY/f8W/xyOq51XdF/xLMVvGeWIqdKVOmrPb67DY/Yt6lICLzgHtUdaafrEhuhHByEamEqRrRFRPRcLmqHnb0uuKtR12rHyV9EbkGU7bpgmB9ixMvbO21HE+JzaUgJrn4j8DJvvAc/wgEIGhkwgnK+2EM7n5gsKpOd+Rd8dbef6sfBX3nc7IGOFOjvFV84cKFq1q2bLkjLi4udm/iEkZeXp4sXry4WuvWrYMmrY91l8KNmJwJwfxn+UYmFFH+ObATiAc+EpFvndmvpRTifBGPx7idos2SP//8s0peXp5Nju4B8vLy5M8//6wCLAmlE7MzXBEpi4m5PVtVlwYci7hLIZgcaA5MB2pgZrx9HblnH4GtfuT1RaQRMAeoo6qHgp2jOJg3b16thISE14AWxP7kqSSQByzJzc29rm3btluCaqgHyj0XpWEy8X/rgXEI8DhwGPPNVh+PlPu2+tHTB2YD57n9ebTN2831ARR54PAJcJXb4/AbT0MgA5MS8mGvGAKrHx19zE7H8W5/Dm3zdnN9AEUaNNQFdgAV/GRJQFIQ3ajKMds9DwLrgWt8N2W4m9evb1erH7P6A4HdQBm37w/bvNtcH0CRBg23ARP8XidhkotvDTB+bslrY3y6eZiNE+q0rgV4b12tfszqL8K6FWwL02LV0X4x8FUQeXFFJhRWno3ZILEb6ELwKApLyeMHTN08iyU4blv8wjbMduSdQO0AuSdcCoFyoAwwBWN09wM9w7y3rnj3kdnq56+/DdgOJLh9n9jmzeb6AAo9YJPvdqHb4yjCuM90bs7DwLggx4+7wcPd9Fbfs/o7gDPc/rzZ5s3m+gAKPWCTFu8Jt8dRxLEL8BxmtrsMZ5buUcNh9YumPxW42+3Pmm3ebK4PoNADhu8x6RLBo26E/ORAU2AVJoRsfKib10//yA0e7ma3+u7rY3y4n7l9n9jmzeb6AAo1WKiEWYyqgPciE4oiz8HMdjcC6fm89654d3Xe6h/VOxnjVoh3+36xzXst1qIU/gLM0WPrSEU7AiGS8n2YhZb1wGIReUXk+LLtlthBVTcBm4FWbo/F4kHctviFacDTwL1+rz3hIoiEHBgC7MUY4G4Bel3xyCOz1S+Q/ofAnW7fL7Z5r7k+gEIN1sQ5nuP2OIrx/SVikuHkYbYuJwa7wcPd9FbfE/rZwIduf55s815zfQAFHqhZ4d8J1HR7LFF4r2djZrr7nJu3axAdLxoaq29kVwO/u/05ss17zfUBFHigcAqQhQce/6Mhx3zBvIyZ7f4C1AjS58hNH844WP3o6gNlMZtcEt2+b2zzVnN9AAUeqClf/iXeizQobvl2TDKcQ8DfgvxduuKB1Xmrf6w+JnNcC7fvG9u81WIpSqE5kOn8HokIgViR5wG7gEeAR0RkhYg0DdHf4h0ygHS3B2HxGG5b/II24FXgJjzyyO+GHJOF7FfM9uDniZFH7NKoD/wDGOP2fWObt5rrAyjwQOFboIfb4/BCwyzK7Me4GW71k4c0EsGOWf3i08eE+X3g9mfFNm811wdQ4IGazQGnuj0OrzTMzHcGxuUwFSfxtduGxuofOXYDMZhkybbiba4PoECDhHKYGV1cgNz1x3y35cB5mHC5PUB/R3bEIIQzDn7nsPqR198GbHb73rHNW831ARRokKakzh5/Y4M3IwrclO/DzHZnAVWdm97zq/klWP9cjK9d3L5/bPNOi5UohcqYzFrB8FJEgZvyPUAHTLzyFuDSEHqW6HAI85mt6vZALB7CbYtfkIYxJHOCyD3zaO8lOWaF/DCwBlPc0AuP2KVRfy3QzK37xjbvNdcHUKBBmsezr9weRyw1oB6w2DG8E0MZiWAGJJxRsfoF1wdmA2e6/VmwzTvN9QEUaJDm8XiK2+OIxYZZLT+A8fPu8KJhKqn6mEREF7n9GbDNO831ARRokHAN8Ibzuyce22NJjknc/h1mUW0fcE44Y+LX74iO1S+8PvAmMMzNe8c2bzXXB1CgQcJw4BncjwSIdfkuTOn2w3hnNb/E6gOvADe4ff/Y5p0WS1EK2X6vvRYhECvyg5gSMBOd10+LSKUQupYTJxdIcHsQFu8gqqHuTe8gIo8DW1X1URFJAlDVfQE6Vl4IuYi0Az4FqmGSAmVxPNWB1sBC57Xv9+1BdK3+8fppwMuq+kiI/pZSRqwY3LHAIVV9yO2xlCREpALGxeCro/Ybxz5JCMfPloPJrH5wWSPgRfu5tfiIlcedPUANtwdRAnkLYxwE499NB94DrlbVQ24OrCQgIi8QenZsKYXEig83B+PHPQYRSfI9Klt5oeVNgUuAyZhZbhVMRrb+wC4RuTywn6XQxGMWKC0WIHYM7h7gMn/j4fy+Dlhn5UWSLwFWYhJlLwH2qeo5QAXn+DsiskhETsVSVMpjKjFbLEDsGNwcTAxpMLwWCRAL8gRMReDBzuvvgCQRaauqh4H/OfI4YJWIvCQiEuQ8lvBUw7oULH7EyqJZb+BOVe0ZIPfEin8MylcCWaraRUTGOOIrMMUP/wtcBXyhqr1F5CrgBUwylitV9WMsBUJEfgLuVtVZbo/F4g1ixeD+BXhYVbu4PZZYR0QGYhbGnsb4bkc7h2ZgclbEAR9icuvmOX3KABOAAcAcoK+qbo7y0GMOEcnA/B2Xuj0WizeIFZfCHqCi24OIdRy3wIsYn+3ugMM/Oz8vUdVLfMYWQFUPqeogoCVwErBBRB6zboZ8qY51KVj8iJUZbmPgG1U91YOP57Ek/zswCqjitwFijKM/RkRmA3mqeiZhEJHbgccwMbuDVPW/4fRLI86X0QGgkqoecHs8Fm8QK3G4a4HaIlIVWAEgIg38jIZv5d3Kw8urAY8FGmM/HgM+EJF4Z/EsKKr6rIi8igkp+0pEVmO2C1vDcpQKwF5rbC3+xIRLQVVzgdVAY58olKqVh5RXwCx8jQrRB1Wd6ujcGkrHT3efql4AdMTUnLsbsD72o1THfGYtliPEygwXYBlwKtAAjn1kVtV9ImLlIeQY/3dZYLrm70P6L3Az8Gw+er5r/gLUFZH7gQeAFKCfIy+1iMhQTJY2i+UIsWZwk1V1WrCDoR6TrRwwUQkKXOQXBuZjNIDf+tcGoLeIVFHVXcGuEeK6Y0XkOUyEw2wR+Qy4LIz7oqTTFLOxxGI5Qky4FByWAcluDyLWEJHmmITj8QXskoVJUj5KROJE5EMRUad9ISLhPjPZmJLtAvQCdorIbScy/himCc56g8XiI5ZmuMuBof4Cj0YCeE3+AbAA+NjRGROgQ6Dc2c57JSbdYE9MkhswGyKmYHIwBOOzIPrPisidmFIzv4XoVxJpCrzk9iAs3iKWDO4xM1wPrPjHgnwTprxOMmYnWUEZDVyNSVb+lqpe7ZwTjBENRc8Q+tuAxSLyATBUS3gmMhFJAJpj4p0tliPEkkshC6jghIb546VIAK/Jk4DPVbVQj7aquhbjGogIqtoB6MdRN0M4o10SSAM2qGrE/oaWkkHMzHBVVUXkN8xj7ndeiwTwmhy4FuNLHRLmzxqOxUAH4Cq/BbWrgC/C9PkolL6qfigi1TA73caLyN1AH1VdXcTxeZn2mC3QFssxxMROMx8i8iSwXVUfdnssXsZZ2NqNqeDwmSP25Ux4MEA9P/n/gE7O78fkWAhx3U8xM9mQ+iLSEFNCPB14A7i+AOFqMYOIvAQsVdUChdZZSg+x5FIA+BHo7PYgYoCnMU8vn5/geQ4Du1VVnHZJKGMLoKp5qto7P31VXa2qLTF+4sEYN0O/ExyrlzgDO8O1BCFmXAoOPwJviEgFVc3xP+DRCAE35DUxO8VGquozfnIc/TEB+uHk7YAeIiLFMQNV1Qki8j7wJjBZROZhMpFtjPS1ooWIlAdSMZEhFssxxJTBVdU/RWQ3sF5E6nokEsCL8v3+xvYEmIeJPBjA0dLqEcWJWLhcRB7GuBnWi8jPwJfFcb0o0AT4rRRv+LCEIV+Xgog0EJFvRCRDRH4TkTv8jv1VRH535I+F6N/L0VkhIvf4yV8XkYViyrhMFpGKfudcIiKfiUhZR9ZZRJ5yun5K6C8KL0UIuCE/FVPJIVJJwhWYDYyM0PlCX0h1qao2Ae4A2jrXbFTc1y0GGmI+oxbL8ahq2AbUAU53fq+EiYdNB7oBXwGJzrFaQfrGY7Y3Nsbs5V8IpDvHKvvpPQXc4/y+EPNF8DBwIWalfQZQzTneB5OqMfBaSUBSaZZj/IabgTFB9McURQ5cgPHllsnvsxKp5ry3jzFllb7z/e9joTmf3zPdHodt3mz5uhTU+NM2Or/vFpPFvh5wPTBOnfRzqrolSPf2wApVXQXg+OsuwqzgZjsycW4w/9laGUwBvkOY3WWfqeoO59i3wPuBflz1Vu6CqMtFpAtmZijAaDk+N3hgzoQCyTHRC7nAncDjwcYSaZz31ldE2gHTgC0iMlZVH4rG9YuKE55Xj6PJ3H3y4cB1mM/4YmCYqu73O/40ZgID5nNfS1WrOseuAu5zjv1DVd8SkURMCF594AVVfcHRfQV4UVXnF9NbtJwghYpScMJ52mA+UMlAFxH5WUS+E5EzgnSph+NrdFjvyHznewOzGyoVeM4RP4F5jK0JzMLEcr7g66Oqe4C5HP2AWgzvAH8U07lXATf5XkghcywUVR8zY6/vXP8+EdkgIh0j+L4izfnADPXLJSwi9YDbgXaq2gLz1DfIv5OqDlfV01T1NMx9MNXpWx3zxdcBM3kZ7cQy98T411sBNzi6rYE4a2y9TYEXzRwf6xRMMcdsMdsXq2HyoZ6BSVzdWFX9Z6rBSrAcOa6qw0QkHvMhGwi8oaoTMPWzEJHRmDSBvUXkSozxHoH5QA4Ska+d83glQsAt+bUYw1QXk1oRLVw0Qli5iEwD5otIDVXdRvCcCYXNsVBY/U8ws7+fROQLTCaynBD93eIi4O0g8gRMVeRDmPeQFeYcgzn6dNETmKmq2wFEZCbObj3MU6H//TsWvy9Fizcp0AxXTBHBKcC7apJUg5mtTlXDLxh/20kBXdfj5K91qE/Ah82ZDUwE+gdcsy5whqp+hHmkGoipKNAdk26wj3P+dT5D5PTzrdSXJvnLmP/NZooBVV2Iqc11vyM6kjNBTd6Et4CLw5wiEvp9VLU75v/fDtguJimOJxCRWsCZmEiLI6jqBsxT21qMa26XqgaNwBCTNKgRJicxhH5CnInJc/Ez8JiI9AXmqWo4Q27xAAWJUhDgdSBDVZ/yO/QhJu0fIpKMWRTbGtB9DtBMRBo5EQeDgI/F0NTv/BcCmQF9x3L0Bvf5ePOA8qr6J2YxpSzeiRBwS/4A5u9yfQj9SDGZgEdhN1DVb4DamM0dT4jIKhFp6fKwwITOfRI463ZcABdhDGldTD6QUImEBgGT/VwSQZ8QVTVXVYeoahtgEsa//qSIPOVE/PSNxBuyRJ6CuBTOwixcLRYRXzD3KGA8Zk/8EuAgcJWqqjMzfU1Vz1fVXDH5UGdgfFfjVfU3x3/3lohUxnyoFuI8CgOISBsAP3/U65jFhnUc3YL6NlAZON//EVs9ltOgOOXO4sldwCNa/LWzHgRuFJEUwuRMCEFE9R231T1itnp/DCwUkSnAEHUvE9nlQLBFvR7AH84kARGZipkJvxNEdxDHljdaD3T1e10fs2jszy2YJ4BOmPtwIGY7dqRCAy0RJKZyKfjjGJssTMjaGrfH4wYi8i7m0dw/eqCoORMKIr8dsyFhCAXImeA3zgLlWDgB/QsxM3AFblHV8cH0igsRaQL8BNRTU3/P/1gHzOTkDExi9zeBuar6XIBeCmZi0si3DuIsms0DTnfUfgXa+vl0q2HyHZ8H9MUsrt0HLFDVVpF/p5YTJdZyKRzBmdF9gJlZlDpEpA5mgSXcTDHSLMRsvS1QzgQfUdCfjilkOR54VUSWOkYwWlwOfBBobJ2x/Yz5MvgV85QWB7wiIg8FPPoPBt73X3R2DOtYjGtuDvCQz9g6PIAJFVOMsW7nXOPVSL45S+SI2RkugIh0wmSbSsPccF6KHChu+Q+YTSQ1AuRjHP0xkZY7Y8kBuju+VM/hLDx9DLTAzCZv0DAl3yNwvTKYsLW+akOyLPkQU7kUgjAb4xvuwtHYRS/lNCgueUdM1rTXI/nHzA/Hj7wJM7PypMF13EutReRy4BVgiIh8DGQU0yWbA+utsbUUhJh1KcCRxZPXOLrg5pXIgeKWv4Wprrs+hH5x8gvQWeT4rWleQlXfxSyqTgYuw+z0qlgMl2pDlHbgWWKfWJ/hgqkgsBKzmrvS7ciB4pY7fr9mwL8K/6eKCAswMdBXcnRjgidxXAlDxWQi+xgTPvUccJe/r7SoiEh7TE7fj070XJbSQcwbXDW73p4HRqjqsCDHPZHrIILy1zExy3dC4XMjREiehwlH87TB9aGqmUCyiNyMSZR0pYgMUdUZJ3jqO4Dni9NHbClZxLRLwY9nMclOYjGdX4ERkbsw26mnuTyUOKCl+O14g+jlWCiqPmYDQnXgB+BzEfnBCb0qNM5nrRdR9qNbYpuYn+ECqOoOMXWk7sEE53stouCE5WJyTvwDeE5V7xWzLz+iORMKI8fktBiJCVvy4UaOhcLq/0dVLxaRtpgvrs0i8oiqjg5xjlA8APxbj2axs1jypUQYXIdngN9F5HFM9IJXIgoiIsf4qsE8ynuBz4FrONbgHsmBAEeMc7iS6K7pq+o84BQR+TswVkRuwGyu+CnM+XDOk4rxYzfLT9di8aekuBRQs3VyPDDcJwqlGmtyEamCMW4jIrHYEyEeABqKSG23B3IiqOqjQA1gCTBLRGaISIV8uj0IPKWqO4t9gJYSRUma4YLJyrQUk4h7k9sRBRGUvwdsUVXfLNd1VDVTRLZgFtFuccSu5lgoqr6q7gbOFbOZpDuwQ0Tu0WOTNQEgJu/sXzBfgBZLoYjpnWbBcFwKtVQ13KNmzCAmq9pyTLKTlX6HijNnQkHlvYBmvt1uUvw5E6Kpr5g45wvVpKf06XwKfKmqboXlWWKYkjbDBWMIlopIN69uPy0kUzE1xVbmp+gC3wEdRKSlqi52DFnvgnb2sr6YcvMfYRKvT8Mk7DkXaEroRTqLJSwlzuCq6h4R+Svwkoi0wvFTeyHSoLByEemGyQnwolvRCPnJRaQ/5kuuHyUIZ03gTBG5AFOBZAcmCfu1qnrQ1cFZYpYSs2jmj5oqERmYVHVeq85QYDkm5+8q4M8T+HMUN69iogFKJKr6KWZRbTGwLgKbJSylmBJpcB3+ismxEIcHIg0KKxdTp6wOJs7UyzwFlBORAj+6xyApQBNMVQeLpciUOJeCD1VdJyKPYOIlL/BApEGB5WKe25/CBOwH3fLrFVT1kIjMx1QB+dzt8UQa53/xEiYX7br89C2WcJRYg+vwLKY8UH8CSpp4KDdCMPk/MXXcNhCd3AgnKl8DXCwi4qE44UhxG+Z/8W+3B2KJfUqySwE1GfhvxBQbPNnt8RQEx4c7AvgekyQmFljs/Lwh2EHxeI6FUPoichpmg8flNkGNJRKU9BkuqvqLiLwKvCMiFwF5XohACCN/E1NKu4ej43sfYwL6ek1+OSZ71sscTyzkWDhGX8xus/eBO1R1RYjzWCyFosQbXIcHga8xRScPiQdyIwSTO37cS/FAOfIi8ANwtYhU0IBS4cRQjgU/ngP+p6r/CXMei6VQlGiXgg/HtTAYEwUQ6kvG9cgEYCKwSlUnhdD1MmuB/cD/uT2QE0VEhmBKmf/V7bFYShalZYaLqmaJyADMI3sNnPI0HopMaA90BNpH+r1HkeWYShD3Bcg9kTOhIPpiqv3+CzhXVfeEOYfFUmhKjcEFUNUvReQ5YIqInK2q+x25FyITJmJmu31EpI+f3EvRCPnKnWMNAkKo+mFyFPge2z/ERI6EwhV9MVnZPgQeVNUFYfpbLEWiVBlch3HA6cC/ReQ6L4QxichlQEO3xxEhsoExwLU+gZdzJvgQU+58MiY/hA0BsxQLpc7gqqqKyDDgf8DdwGP+x12KTHgR+A2Y7KGogyLJgQoYY3stMYKzueEVzCaTO7zwJWwpmZSKRbNAHN9cH2CkmMKCgDs5E0RkFKac94fF8V5d4B9AVRFp5/ZACsH9mCRBg228raU4KZUGF0BV1wBnAfeJyODAw6G6RVLuPMbeDzwN5IYebeygqruAZfj5dL2MiFwFDMPkvQ0MZ7NYIkqpcyn4o6rLRaQn8JWI7FXVj6IcmfA6cBBT/DImDFQBeQmzPdnTOP/7x4CuqrrJ7fFYSj6l2uACqOoSJyrgM8fozgyhF9HIBBGpAVwNfIIxtp6MOiiiXICyItJPVacGKnoB538+HrhEVTPcHo+ldFBqXQr+qOpcTKjQf0Skc5Qu+z5mRX9+lK4XTRSzq++1YDkKfIhLORNEpB/wOtBHVWedyBu1WApDqZ/h+lDVH50dRlNF5AJVnVNckQkiko4pVthdnTJAbkcXRFouIgsx+Qm8ljNhNnAK0FtVfw3Rz2IpFqzB9UNVZ4pJ/P2ZiNyICRUqjlwKHwALtWTUXAuKqk4TkYPAxWHUop0zoTHQEuiiqksK8j4slkhiDW4AqjpdRC7EFG8sg1nUCqpaFLmI9ALSgeQTGWeM8DUeKb/jhP+lAJ1UNdPt8VhKJ9bgBkFVZ4vImZjtn9/iZ3QjEJnwBrC8lKT8Gw2cLyJVnHCxQKKVM+E0oCpwpqp6sfqxpZRgDW4IVHW1iJwFTAI+EpHBqrrbOVbUyIRbgdrAySIyJkDNC9EFxSE/jElmc3fgAYo/Z8KVwBJMFrPTVXV7GF2LpdixUQphUNWdwPmYFfcfRKR+Uc/lbB99DBMyVZqIxxi+41DVPFXtraritEucPAdBKYy+iDQDfsLMgLtYY2vxAnaGmw9qiiTeCIwEZovIZar6P9/xQkQsPOX8fMiIvRldUAzyysBwEWmsqquIAn4xtg8AL9vcCBavYGe4BUANj2PKrk8TkX+ISNlguRHg+JwJIlIJU4zwfkIvqpVUsoGdmKobxYrzP3kEkwzoIlV9yRpbi5ewBrcQqOp04DSgNSaeM9V3KFQX5+fbwA5VfSqEXklnMdC3OC8gIq2AOUBzoJ3/U4jF4hWswS0kzp77vsALwJfAo8CpgZEJQANMgH0t4CLguuiP1jP8AFRyFiEjiogkiMj/AV9h3DYXq+rmSF/HYokE1odbBJzH1NdE5BvM7LWXiAxT1bV+Or7IhEnAVuB0ETkd70URRKsSxGaMTzVicbkikozZRZaDmdWuzaeLxeIqdoZ7AjgxnX/BzK7misg1/nv7nVjedpjtp6WdOcC5kciZ4PhqRwCzgHeB86yxtcQCdoZ7gjgJq/8pIp9jFmtuFZE7VPVH4D/A96rqn+Tc12+M/3lKuhyzaHYIYyR7UoScCWKSzlyMCa9bhtnIsDzEOSwWz2ENboRQ1QXOjHYwJuvYGowPt4O7I/MGqqoi8hOmBM9bFC1nwn+Bk4BbVfXLYh2wxVIMWJdCBHHCx/4DpGFqlO0D/k9Eark7Ms/wT6ANhfjciUhdTOn4bExKyzbW2FpiFWtwiwFVzVHVm4DGmJ1lGSLykJgy3KUWVf0ck5diKAXLgbAI+N3RbaCqL6tqiShFZCmdWJdCMeKEJ90hIk9jiiuuw2QgK+e1KIIoyv8A6gMXBCo6em0xuQ92Y8LunlTVjcF0A/qNxxQG3aKqLRxZdWAipgT9amCAqu4I0vcxZzxxwEzgDiAJk0ejCSYfxHRVvcfR/ytwI7AWE4Z2UEzi+n6qeld+Y7WUXuwMNwqo6mpVvQKzUeJJzM6rQRjXQ2n7H3wLVAKq+wRi6CUi/wWmAb8A9VR1ZEGMrcObQK8A2T3A16raDJMq8p7ATo7f/SygFaZy7xnA2c7hJ1Q1FeMGOUtEejvy6xz9+UBPJ0/G/cDYAo7VUkqxM9wooqpZmCrBD2FW6G8DzgReBp4AcrwWXVAcchG5DHhARP4JXIGpmnsIeByYqKqHKCSq+r2INAwQXwR0dX5/C2Ps/x7YFSgHlMW4f8oAm1V1L/CNc+6DIvIrZmbuowxQ3hn3UOCzYLNni8Wf0ja78gSqelBVJ6pqF8xj8CnATcCFInKJiFR0d4TFh5PMZhHmkXwpZtZ/E3Caqr5TFGMbhtq+GbLz87jFS2cL8DfARqfN0ICikiJSFbgQM0sG8+U4G6iJCXO7CrPz0GIJizW4LqOqC1X1BsxW4DeAW4AsEflcRG4BSsJCW1kRGSQiUzF+7P3AKKC+ql6rqt+7lWRGRJpiXDv1gXrAOSLyF7/jCcB7wLO+bGeqOkFV2zhuoruAZ4HeIjJZRJ4Ot6nDUrqxLgWPoCb37vPA884ssCdmVtUX2OXc+NOBOeFyxnoBEakGdMb4Qi8B6gDfYxawro3io/dmEamjqhtFpA6wJYjOJcBsVd0D4Gxg6eiMF0xdu+Wq+kxgRydk7QxVfVBEfgE6AQ9jCoTOjPzbscQ61uB6EFXNxqyQTxKReIwBuAh4B6glIuuBZkAZEamJydXgmyG6EY1QAZMv+ICIDMNsTpiL2fI8DPhFVffn976LgY8xj/vjnJ8fBdFZC1zv+JMF8yXxDICI/APzhBEq8dBYzGIZmKgGBfIwvl2L5TiswfU4ztbhWU77mzOrauO0zhgfcC1gA8bw+qgK7AEiFbdaAajmGxbQG2OMagCJmBmhrwbc/Aj7YvNFRN7DLJCd5HwhjcYY2g/EVGJeC1zm6LYDblLV64DJwDmYFJIKfKGmkGh94F4gE/jV+YJ5XlVfc87RBkBV5ztDeN05xzqikPvXEpuIzc8c+4hJcN4Kk6u3vfN7DUz9tBzMYlA8xne6ABOWlo3x4f/FObYYYzjLOj+bYb6Q84BGmF1zjmdm6QAAAndJREFUq4D1mM0IK53Xq4D1zheDxWIJgzW4JRhn8aY6xodaEzMjreq0KpiA/gOY3V8HgrSDwCbgD8fNYbFYTgBrcC0WiyVK2PAVi8ViiRLW4FosFkuUsAbXYrFYooQ1uBaLxRIlrMG1WCyWKGENrsVisUQJa3AtFoslSliDa7FYLFHCGlyLxWKJEtbgWiwWS5SwBtdisViihDW4FovFEiWswfUwIjJeRLaIyBI/2VgRWSQiC0TkSyc/bqj+lUVkg4g87ycb6PT/zSkP7pP/VUSWiMhnIlLWkXUWkaeK6/1ZLKUNa3C9zZscX/r7cVVtpaqnAZ8AD4TpPxb4zvdCRGpgKuN2V9XmQG0R6e4ctqW/LZZixhpcD6Oq3wPbA2T+eWkrcLS0zjGISFtMAvIv/cSNgWWq+qfz+iugv99xW/rbYilGbImdGEREHgauBHYB3YIcjwOexBjN7n6HVgCpItIQU7nhYkyFBzha+vs3TDmfDzl+dm2xWE4AO8ONQVT1XlVtALwL3BZE5RbM7HRdQL8dwM2Y6rk/AKtxap7Z0t8WS/Fjb6LY5j8c6xLw0Qm4TURWY2auV4rIOABVna6qHVS1E6Y22XL/jn6lvz8C7gMGYsrt+M+ULRZLEbAuhRhDRJqpqs9I9sVUlT0GVb3cT/9qoJ2q3uO8rqWqW0SkGmYmPCCguy39bbEUE9bgepgQpb/PF5EUjBFcA9zk6PqX/g7Hv0SktfP7Q6q6zO96tvS3xVKM2CKSFovFEiWsD9disViihDW4FovFEiWswbVYLJYoYQ2uxWKxRAlrcC0WiyVKWINrsVgsUcIaXIvFYokS1uBaLBZLlPh/O2Ts6a+a85QAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(5,4))\n",
    "patches=plt.pie(dt_count_lb.values,autopct='%.2f%%', pctdistance=1.22)\n",
    "for i in range(len(patches[0])):\n",
    "    patches[0][i].set(hatch = hatches[i], fill=False)\n",
    "plt.legend(dt_count_lb.index,bbox_to_anchor=(1.5,1.025),loc=\"upper right\")\n",
    "plt.savefig('jobroles.jpg', bbox_inches = 'tight',dpi=100)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_inner = pd.merge(left=dt1, right=dt2[['gh_login','label']], left_on='u_id', right_on='gh_login')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "txt_dt=merged_inner[['u_id','textual_all','label']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "txt_dt.to_csv(r'..\\dev2vec_repos_raw.csv', index=None, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24429870"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#number of words before cleaning\n",
    "txt_dt['textual_all'].apply(lambda x: len(x.split(' '))).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean Dev2vec:repos dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "from gensim.parsing.preprocessing import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "#store the textual_all in a sperate varaible\n",
    "documents=txt_dt['textual_all']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#seperte sentences with \\n\n",
    "def split_sentence(corpus):\n",
    "    sentences = [doc.split(\"\\n\") for doc in corpus]\n",
    "    return sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#flat a list of lists into a list\n",
    "def flatten_list(sentences):\n",
    "    flatList = [ item.lower() for elem in sentences for item in elem]\n",
    "    return flatList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tokenize all sentences\n",
    "def token_in_sentence(list_sentences):\n",
    "    list_tokens = [list(gensim.utils.tokenize(doc, lower=True)) for doc in list_sentences]\n",
    "    return list_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#build a bundle of actions to clean data\n",
    "def custom_filter():\n",
    "    cstm_filter = [remove_stopwords,split_alphanum, strip_numeric, strip_tags, strip_non_alphanum]\n",
    "    return cstm_filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cleaning data based on steps define in custom_filter\n",
    "def str_preprocessing(tokenize_sentence):\n",
    "    CUSTOM_FILTERS = custom_filter()\n",
    "    tokens_new = preprocess_string(\" \".join(tokenize_sentence), CUSTOM_FILTERS)\n",
    "    return tokens_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get the possible phrases in the list of tokens\n",
    "def get_bigram_token(model,tokens):\n",
    "    bigrams = model[tokens]\n",
    "    bi_tokens= [sent for sent in bigrams]\n",
    "    return bi_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "#split sentence in documents (Corpus['textual_all'])\n",
    "sentences_stream = split_sentence(txt_dt['textual_all'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tokenize all sentences\n",
    "tokenize_sentence=[]\n",
    "for row in sentences_stream:\n",
    "    tokenize_st = flatten_list(token_in_sentence(row))\n",
    "    tokenize_sentence.append(tokenize_st)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cleaning data with costum actions\n",
    "processing_tokens = [gensim.utils.simple_preprocess(sen) for sen in txt_dt['textual_all']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cleaning data with costum actions\n",
    "processing_tokens = [str_preprocessing(sen) for sen in tokenize_sentence]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package words to\n",
      "[nltk_data]     C:\\Users\\morad\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package words is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('words')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = set(nltk.corpus.words.words())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "english_tokens=[]\n",
    "for row in processing_tokens:\n",
    "    english_tokens.append([w for w in row if w.lower() in words or not w.isalpha()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_tokens=[]\n",
    "for row in english_tokens:\n",
    "    clean_tokens.append([w for w in row if len(w) > 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "flatten_tokenize=flatten_list(clean_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9135161"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of tokens after cleaning process\n",
    "len(flatten_tokenize)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### build a dataset for doctovec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "#build the data\n",
    "dev2vec_repos = {'text_all':clean_tokens,'tags':txt_dt['u_id'],'label':txt_dt['label']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev2vec_repo_dt = pd.DataFrame(dev2vec_repos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev2vec_repo_dt.to_csv(r'..\\dev2vec_repo_clean.csv', index=None, header=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load dev2vec:issues dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#the directory of pickle files for issues\n",
    "issue_dir = r'..\\issues'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "col_name= ['Name','u_id','bio','Repo_Name','ReadMe','tags','topics','textual_all']\n",
    "Issue_Corpus = pd.DataFrame(columns = col_name, dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#loop over all files in a directory and collect required fields\n",
    "issues=0\n",
    "error=0\n",
    "for filename in os.listdir(usr_dir):\n",
    "    usr_path = os.path.join(usr_dir, filename)\n",
    "    reader = open_file(usr_path)\n",
    "    try:\n",
    "        issue_path = os.path.join(issue_dir, filename)\n",
    "        issue_reader = open_file(issue_path)\n",
    "    except:\n",
    "        issue_reader=None\n",
    "        \n",
    "    bio   = newline(str(reader['bio']))\n",
    "    Name = str(reader['name'])\n",
    "    try:\n",
    "        u_id= str(reader['u_id'])\n",
    "    except:\n",
    "        u_id =\"none\"\n",
    "\n",
    "        \n",
    "   \n",
    "    repo_name=[]\n",
    "   \n",
    "    issue_text=[]\n",
    "    issue_txt_all=[]\n",
    "    \n",
    "    if(len(reader['repository'])>4):\n",
    "        for row in range(0,len(reader['repository'])):\n",
    "            repo_name_tmp = str(reader['repository'][row]['repo_name'])\n",
    "           \n",
    "        \n",
    "            repo_name.append(repo_name_tmp)  \n",
    "            \n",
    "           \n",
    "            \n",
    "            try:\n",
    "                if((issue_reader is not None) and (issue_reader['issues'][row])):\n",
    "                    for j in issue_reader['issues'][row]:\n",
    "                        issues=issues+1\n",
    "                        issue_title= str(issue_reader['issues'][row][j]['title'])\n",
    "                        issue_body= str(issue_reader['issues'][row][j]['body'])\n",
    "                        issue_temp=merg_text(issue_title,issue_body)\n",
    "                        issue_text.append(issue_temp)\n",
    "            except:\n",
    "                error=error+1\n",
    "            \n",
    "            issue_txt_all.append(issue_text)\n",
    "            \n",
    "            \n",
    "            \n",
    "                \n",
    "        \n",
    "      \n",
    "            \n",
    "        text_all = merg_RM(issue_txt_all) \n",
    "      \n",
    "    \n",
    "        Issue_Corpus = Issue_Corpus.append({'Name':Name,'u_id':u_id,'bio':bio,'Repo_Name':repo_name,\n",
    "                            'textual_all':text_all},ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1615"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Issue_Corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt3=Issue_Corpus[['Name','u_id','textual_all']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_inner_issue = pd.merge(left=dt3, right=dt2[['gh_login','label']], left_on='u_id', right_on='gh_login')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1306"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(merged_inner_issue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "issue_txt_dt=merged_inner_issue[['u_id','textual_all','label']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "issue_txt_dt.to_csv(r'..\\dev2vec_issues_raw.csv', index=None, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2593012"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#number of words before cleaning\n",
    "issue_txt_dt['textual_all'].apply(lambda x: len(x.split(' '))).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean dev2vec:issues data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#split sentence in documents (Corpus['textual_all'])\n",
    "issue_sentences_stream = split_sentence(issue_txt_dt['textual_all'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tokenize all sentences\n",
    "issue_tokenize_sentence=[]\n",
    "for row in issue_sentences_stream:\n",
    "    tokenize_st = flatten_list(token_in_sentence(row))\n",
    "    issue_tokenize_sentence.append(tokenize_st)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cleaning data with costum actions\n",
    "issue_processing_tokens = [str_preprocessing(sen) for sen in issue_tokenize_sentence]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "issue_english_tokens=[]\n",
    "for row in issue_processing_tokens:\n",
    "    issue_english_tokens.append([w for w in row if w.lower() in words or not w.isalpha()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "issue_clean_tokens=[]\n",
    "for row in issue_english_tokens:\n",
    "    issue_clean_tokens.append([w for w in row if len(w) > 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "flatten_tokenize=flatten_list(issue_clean_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "978445"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of tokens after cleaning process\n",
    "len(flatten_tokenize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#build the model\n",
    "dev2vec_issues = {'text_all':issue_clean_tokens,'tags':issue_txt_dt['u_id'],'label':issue_txt_dt['label']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev2vec_issues_dt = pd.DataFrame(dev2vec_issues)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev2vec_issues_dt.to_csv(r'..\\dev2vec_issue_clean.csv', index=None, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}