final_research.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "final_research.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "iBjcPYkGYKhS"
      },
      "source": [
        "### Mounting Google drive\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4L5XPSnxylKe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kWBHTB-xysfn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%cd '/content/drive/My Drive/five_class_entitydata'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "fGDt5kh-X7kO"
      },
      "source": [
        "### Imports\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SNSxFiLtD5pG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import spacy \n",
        "import re\n",
        "import json\n",
        "import random\n",
        "import logging\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.metrics import precision_recall_fscore_support\n",
        "from spacy.gold import GoldParse\n",
        "from spacy.scorer import Scorer\n",
        "from sklearn.metrics import accuracy_score"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "snew6xiTzqQe",
        "colab_type": "text"
      },
      "source": [
        "### Converting dataturks to spacy format\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4hpbzf4WEXCU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#converting dataturks annotated data to spacy format to be \n",
        "#used as training data\n",
        "\n",
        "def convert_dataturks_to_spacy(dataturks_JSON_FilePath):\n",
        "    try:\n",
        "        training_data = []\n",
        "        lines=[]\n",
        "        with open(dataturks_JSON_FilePath, 'r') as f:\n",
        "            lines = f.readlines()\n",
        "\n",
        "        for line in lines:\n",
        "            data = json.loads(line)\n",
        "            text = data['content']\n",
        "            entities = []\n",
        "            for annotation in data['annotation']:\n",
        "                #only a single point in text annotation.\n",
        "                point = annotation['points'][0]\n",
        "                labels = annotation['label']\n",
        "                # handle both list of labels or a single label.\n",
        "                if not isinstance(labels, list):\n",
        "                    labels = [labels]\n",
        "\n",
        "                for label in labels:\n",
        "                    #dataturks indices are both inclusive [start, end] but spacy is not [start, end)\n",
        "                    entities.append((point['start'], point['end'] + 1 ,label))\n",
        "\n",
        "\n",
        "            training_data.append((text, {\"entities\" : entities}))\n",
        "\n",
        "        return training_data\n",
        "    except Exception as e:\n",
        "        logging.exception(\"Unable to process \" + dataturks_JSON_FilePath + \"\\n\" + \"error = \" + str(e))\n",
        "        return None"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "LvYScw-hmBKe"
      },
      "source": [
        "### Cleaning data\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TyYU3wnbIAtm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "############################Removes leading and trailing white spaces from entity spans.############################\n",
        "# https://github.com/explosion/spaCy/issues/3558\n",
        "def trim_entity_spans(data: list) -> list:\n",
        "    \"\"\"Removes leading and trailing white spaces from entity spans.\n",
        "\n",
        "    Args:\n",
        "        data (list): The data to be cleaned in spaCy JSON format.\n",
        "\n",
        "    Returns:\n",
        "        list: The cleaned data.\n",
        "    \"\"\"\n",
        "    invalid_span_tokens = re.compile(r'\\s')\n",
        "\n",
        "    cleaned_data = []\n",
        "    for text, annotations in data:\n",
        "        entities = annotations['entities']\n",
        "        valid_entities = []\n",
        "        for start, end, label in entities:\n",
        "            valid_start = start\n",
        "            valid_end = end\n",
        "            while valid_start < len(text) and invalid_span_tokens.match(\n",
        "                    text[valid_start]):\n",
        "                valid_start += 1\n",
        "            while valid_end > 1 and invalid_span_tokens.match(\n",
        "                    text[valid_end - 1]):\n",
        "                valid_end -= 1\n",
        "            valid_entities.append([valid_start, valid_end, label])\n",
        "        cleaned_data.append([text, {'entities': valid_entities}])\n",
        "\n",
        "    return cleaned_data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "XT-Nvw0RYWA7"
      },
      "source": [
        "### Training the model\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_nWx5rMhEeL6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "################### Train Spacy NER.###########\n",
        "def train_spacy():\n",
        "    TRAIN_DATA = convert_dataturks_to_spacy(\"/content/drive/My Drive/five_class,\\\n",
        "    _entitydata/traindata_3withmyannotation.json\")\n",
        "    TRAIN_DATA=trim_entity_spans(TRAIN_DATA)\n",
        "    nlp = spacy.blank('en')  # create blank Language class\n",
        "    # create the built-in pipeline components and add them to the pipeline\n",
        "    # nlp.create_pipe works for built-ins that are registered with spaCy\n",
        "    # if 'tagger' not in nlp.pipe_names:\n",
        "    #      nlp.add_pipe(nlp.create_pipe('tagger'))\n",
        "    if 'ner' not in nlp.pipe_names:\n",
        "        ner = nlp.create_pipe('ner')\n",
        "        nlp.add_pipe(ner, last=True)\n",
        "\n",
        "       \n",
        "       \n",
        "    # add labels\n",
        "    for _, annotations in TRAIN_DATA:\n",
        "         for ent in annotations.get('entities'):\n",
        "            ner.add_label(ent[2])\n",
        "\n",
        "    # get names of other pipes to disable them during training\n",
        "    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']\n",
        "    with nlp.disable_pipes(*other_pipes):  # only train NER\n",
        "        optimizer = nlp.begin_training()\n",
        "        for itn in range(25):\n",
        "            print(\"Statring iteration \" + str(itn))\n",
        "            random.shuffle(TRAIN_DATA)\n",
        "            losses = {}\n",
        "            for text, annotations in TRAIN_DATA:\n",
        "                nlp.update(\n",
        "                    [text],  # batch of texts\n",
        "                    [annotations],  # batch of annotations\n",
        "                    drop=0.1,  # dropout - make it harder to memorise data\n",
        "                    sgd=optimizer,  # callable to update weights\n",
        "                    losses=losses)\n",
        "            print(losses)\n",
        "    return nlp"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ISjYoCkqFlHE",
        "colab_type": "code",
        "outputId": "97611aec-15b2-48cd-ac7f-7732558c7c72",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 830
        }
      },
      "source": [
        "nlp_=train_spacy()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Statring iteration 0\n",
            "{'ner': 16882.106997960484}\n",
            "Statring iteration 1\n",
            "{'ner': 10908.776434217287}\n",
            "Statring iteration 2\n",
            "{'ner': 6077.587373414839}\n",
            "Statring iteration 3\n",
            "{'ner': 8363.131303432065}\n",
            "Statring iteration 4\n",
            "{'ner': 4631.257164553054}\n",
            "Statring iteration 5\n",
            "{'ner': 5223.568889697201}\n",
            "Statring iteration 6\n",
            "{'ner': 3904.1670692492044}\n",
            "Statring iteration 7\n",
            "{'ner': 4233.118298977122}\n",
            "Statring iteration 8\n",
            "{'ner': 3326.9928887288224}\n",
            "Statring iteration 9\n",
            "{'ner': 4829.5860955535945}\n",
            "Statring iteration 10\n",
            "{'ner': 3626.4763339962747}\n",
            "Statring iteration 11\n",
            "{'ner': 3477.1512522536004}\n",
            "Statring iteration 12\n",
            "{'ner': 2976.898051461118}\n",
            "Statring iteration 13\n",
            "{'ner': 3416.6796133147454}\n",
            "Statring iteration 14\n",
            "{'ner': 4535.087774518935}\n",
            "Statring iteration 15\n",
            "{'ner': 2616.040203681598}\n",
            "Statring iteration 16\n",
            "{'ner': 2176.3353705488266}\n",
            "Statring iteration 17\n",
            "{'ner': 2250.3050472226787}\n",
            "Statring iteration 18\n",
            "{'ner': 2827.64227697353}\n",
            "Statring iteration 19\n",
            "{'ner': 2004.3714014867123}\n",
            "Statring iteration 20\n",
            "{'ner': 1944.1099728117474}\n",
            "Statring iteration 21\n",
            "{'ner': 3099.2789618953875}\n",
            "Statring iteration 22\n",
            "{'ner': 2007.6645720774557}\n",
            "Statring iteration 23\n",
            "{'ner': 2089.51422308577}\n",
            "Statring iteration 24\n",
            "{'ner': 1698.1333922446988}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "dB2Au5AkYixS"
      },
      "source": [
        "### Saving the Trained model\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gqo__mEd0_-u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# save model to output directory (with parcial cleaned data)\n",
        "def save_model(output_dir):\n",
        "      nlp_.to_disk(output_dir)\n",
        "      print(\"Saved model to\", output_dir)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Gquu5l71TKl",
        "colab_type": "code",
        "outputId": "2d837318-2267-46be-c283-20b04c4a4c6d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "output_dir='./model2'\n",
        "save_model(output_dir)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Saved model to ./model2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "gTACN2hIYrXI"
      },
      "source": [
        "### Loading the trained model instance\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ySM49vc027LO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        " ###################loading the saved model################################\n",
        " output_dir='./model2'\n",
        " nlp2 = spacy.load(output_dir)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "GLLqzC6RXHwF"
      },
      "source": [
        "### Testing\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MQoJ9EhJIfu1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "##############################preparing the testdata########################\n",
        "examples = convert_dataturks_to_spacy(\"3class_test_data.json\")\n",
        "examples=trim_entity_spans(examples)\n",
        "tp = 0\n",
        "tr = 0\n",
        "tf = 0\n",
        "\n",
        "ta = 0\n",
        "c = 0\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k2ARtiTSNrFK",
        "colab_type": "code",
        "outputId": "871d880c-44eb-47a8-d306-1ca74c85395f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 525
        }
      },
      "source": [
        "#################testing the model######################\n",
        "nlp_=nlp2\n",
        "for text, annot in examples:\n",
        "\n",
        "    f = open(\"resume\"+str(c)+\".txt\", \"w\")\n",
        "    doc_to_test = nlp_(text)\n",
        "    d = {}\n",
        "    for ent in doc_to_test.ents:\n",
        "        d[ent.label_] = []\n",
        "    for ent in doc_to_test.ents:\n",
        "        d[ent.label_].append(ent.text)\n",
        "        \n",
        "    if 'Skills' in d:\n",
        "      skills_=d['Skills']    \n",
        "      print(f'resume {str(c)} skills {skills_}')\n",
        "    # print(d.keys())\n",
        "\n",
        "    #---------------------------      \n",
        "    for i in set(d.keys()):\n",
        "\n",
        "        f.write(\"\\n\\n\")\n",
        "        f.write(i + \":\"+\"\\n\")\n",
        "        for j in set(d[i]):\n",
        "            f.write(j.replace('\\n', '')+\"\\n\")\n",
        "    #-----------------------------\n",
        "    d = {}\n",
        "    for ent in doc_to_test.ents:\n",
        "        d[ent.label_] = [0, 0, 0, 0, 0, 0]\n",
        "    for ent in doc_to_test.ents:\n",
        "        doc_gold_text = nlp_.make_doc(text)\n",
        "        gold = GoldParse(doc_gold_text, entities=annot.get(\"entities\"))\n",
        "        y_true = [ent.label_ if ent.label_ in x else 'Not ' +\n",
        "                  ent.label_ for x in gold.ner]\n",
        "        y_pred = [x.ent_type_ if x.ent_type_ ==\n",
        "                  ent.label_ else 'Not '+ent.label_ for x in doc_to_test]\n",
        "        if(d[ent.label_][0] == 0):\n",
        "            # f.write(\"For Entity \"+ent.label_+\"\\n\")\n",
        "            # f.write(classification_report(y_true, y_pred)+\"\\n\")\n",
        "            (p, r, f, s) = precision_recall_fscore_support(\n",
        "                y_true, y_pred, average='weighted')\n",
        "            a = accuracy_score(y_true, y_pred)\n",
        "            d[ent.label_][0] = 1\n",
        "            d[ent.label_][1] += p\n",
        "            d[ent.label_][2] += r\n",
        "            d[ent.label_][3] += f\n",
        "            d[ent.label_][4] += a\n",
        "            d[ent.label_][5] += 1\n",
        "    c += 1"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "resume 0 skills ['Angular', 'IBM Personality Insights', 'IBM Watson']\n",
            "resume 1 skills ['Machine Learning']\n",
            "resume 2 skills ['Jquery', 'Python', 'Python', 'Perl', 'Hadoop', 'Django', 'Jquery', 'Apache', 'Javascript', 'XML', 'CSS', 'HTML', 'Python', 'Python', 'Python', 'Python']\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "resume 3 skills ['Text mining']\n",
            "resume 4 skills ['machine learning (ML)', 'Python', 'Python', 'R', 'R', 'HDFS', 'Map Reduce', 'NLP -Text Mining']\n",
            "resume 5 skills ['Business/Data/Predictive/TextAnalytics', 'System Analyst', 'Devops', 'SQL', 'Machine Learning', 'SQL', 'VB', 'Python', 'DB2', 'IBM Power Systems', 'R', 'Watson Analytics', 'SPSS', 'Python', 'Java']\n",
            "resume 6 skills ['R', 'Python', 'ML', 'R', 'Python', 'Machine Learning', 'Python', 'R', 'Kafka', 'SQL', 'SQL', 'SQL', 'Python', 'Python', 'MySQL', 'Python', 'MySQL', 'JAVA SCRIPT', 'AJAX', 'XML', 'JAVA SCRIPT', 'AJAX']\n",
            "resume 7 skills ['Machine learning', 'Java/J2EE', 'Machine Learning', 'Python', 'PostgreSQL', 'MySQL']\n",
            "resume 8 skills ['Machine Learning Modelling', 'Python', 'MBA', 'Python', 'Python']\n",
            "resume 9 skills ['R', 'Image Processing', 'VC++', 'C++', 'Python', 'C++', 'Python', 'SW', 'C++', 'C', 'C/C++', 'Machine Learning', 'Machine Learning', 'Python']\n",
            "resume 10 skills ['Python', 'Javascript', 'AngularJS', 'HTML', 'CSS', 'WCDMA', 'CSS', 'Python', 'Python', 'Javascript', 'AngularJS', 'HTML', 'CSS', 'Python', 'Python', 'Python', 'Python']\n",
            "resume 11 skills ['Python/R', 'Deep Learning', 'SDN', 'BigData', 'Deep Learning', 'Machine Learning', 'Python', 'R', 'Cloud', 'Python', 'Machine Learning', 'Python']\n",
            "resume 12 skills ['data mining', 'Java', 'JavaScript', 'R-programming', 'data mining', 'Lotus Notes', 'MS Dynamics', 'Java', 'Lotus Notes', 'MS Dynamics', 'Bangalore']\n",
            "resume 13 skills ['R / Python', 'Data Analytics', 'Python', 'Hive', 'Machine Learning', 'COBOL', 'JCL', 'SQL']\n",
            "resume 14 skills ['Python', 'NLP', 'Python', 'Python', 'R', 'R', 'Python']\n",
            "resume 15 skills ['Pharmacy']\n",
            "resume 16 skills ['Python', 'PHP', 'AJAX', 'Cloud', 'Apache', 'Python']\n",
            "resume 17 skills ['Python', 'Machine Learning', 'Data Science', 'Python programming', 'Machine learning', 'Python', 'Python', 'Python', 'Python', 'Python', 'Jenkins']\n",
            "resume 18 skills ['R', 'Python', 'Tableau', 'Bayes', 'MS Excel', 'R', 'R', 'Python', 'Oracle', 'Python', 'Python', 'Python', 'HTML5']\n",
            "resume 20 skills ['Python', 'Python', 'XML', 'C#', 'JavaScript', 'CSS', 'Python', 'Kerala', 'Bangalore', 'Python', 'Python', 'Python', 'Python', 'Python']\n",
            "resume 21 skills ['IBM Watson', 'SQL Server', 'PL/SQL']\n",
            "resume 22 skills ['Machine Learning', 'Python', 'Azure Machine Learning', 'R', 'Python', 'R', 'Machine Learning', 'MS Excel', 'R-Programming', 'C-Programming', 'SQL Server', 'R- Programming Language', 'Azure Machine Learning', 'SQL', 'SAS', 'Azure Machine Learning', 'SAS', 'R-Programming', 'SQL', 'SAS', 'Tableau', 'SQL']\n",
            "resume 23 skills ['R', 'R', 'EMR', 'AWS', 'RDBMS', 'R Programming', 'Python']\n",
            "resume 24 skills ['R–programming', 'Machine Learning', 'MS Excel', 'R', 'Python', 'Machine Learning', 'MS Excel', 'Predictive Analytics', 'Linear Regression', 'Logistic Regression', 'Machine Learning']\n",
            "resume 25 skills ['Machine Learning', 'Microsoft Azure', 'Python', 'AWS']\n",
            "resume 26 skills ['SQL']\n",
            "resume 27 skills ['/SQL', 'PLSQL', 'SQL']\n",
            "resume 28 skills ['SAS', 'PL/SQL', 'R programming', 'SAS']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "wzAME085ozSv"
      },
      "source": [
        "### Validating the pridiction\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cd38QyKmNdjJ",
        "colab_type": "code",
        "outputId": "45897364-3dd9-417d-f9f7-8a707b537811",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 472
        }
      },
      "source": [
        "###########################validating the model##########################\n",
        "for i in d:\n",
        "    print(\"\\n For Entity \"+i+\"\\n\")\n",
        "    print(\"Accuracy : \"+str((d[i][4]/d[i][5])*100)+\"%\")\n",
        "    print(\"Precision : \"+str(d[i][1]/d[i][5]))\n",
        "    print(\"Recall : \"+str(d[i][2]/d[i][5]))\n",
        "    print(\"F-score : \"+str(d[i][3]/d[i][5]))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            " For Entity Name\n",
            "\n",
            "Accuracy : 99.88532110091744%\n",
            "Precision : 0.9988545291574397\n",
            "Recall : 0.9988532110091743\n",
            "F-score : 0.9987388618366561\n",
            "\n",
            " For Entity Location\n",
            "\n",
            "Accuracy : 99.77064220183486%\n",
            "Precision : 0.9977064220183486\n",
            "Recall : 0.9977064220183486\n",
            "F-score : 0.9977064220183486\n",
            "\n",
            " For Entity Skills\n",
            "\n",
            "Accuracy : 99.08256880733946%\n",
            "Precision : 0.9895468905067463\n",
            "Recall : 0.9908256880733946\n",
            "F-score : 0.9898169508022059\n",
            "\n",
            " For Entity Education\n",
            "\n",
            "Accuracy : 100.0%\n",
            "Precision : 1.0\n",
            "Recall : 1.0\n",
            "F-score : 1.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SPR0t30luyrq",
        "colab_type": "text"
      },
      "source": [
        "### matcher\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eziwDJ1Fu64i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "from pathlib import Path\n",
        "\n",
        "nlp_=nlp2\n",
        "\n",
        "def find_skills(text):\n",
        "  d = {}\n",
        "  docx=nlp_(text)\n",
        "  for ent in docx.ents:\n",
        "    d[ent.label_] = []\n",
        "  for ent in docx.ents:\n",
        "    d[ent.label_].append(ent.text)\n",
        "  if 'Skills' in d:\n",
        "    skills_=d['Skills']    \n",
        "    return skills_\n",
        "  else:\n",
        "    return None"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "T1AXwIq_ZS0G"
      },
      "source": [
        "### Creating job list\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2jhdeNPzvDBe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# create jobs list\n",
        "jobs=[]\n",
        "job_dir='/content/drive/My Drive/five_class_entitydata/jobs'\n",
        "pathlist = Path(job_dir).glob('**/*.txt')\n",
        "for path in pathlist:\n",
        "    with open (path, \"r\") as fileHandler:\n",
        "      job={\n",
        "          'name':path.name,\n",
        "           'skills':find_skills(''.join(fileHandler.readlines()))\n",
        "      }\n",
        "      jobs.append(job)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HU4CTnoIwYuu",
        "colab_type": "code",
        "outputId": "0eee5dcf-73ba-4189-db3d-4cdd7a00a155",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 167
        }
      },
      "source": [
        "print(jobs[1]['name'])\n",
        "print(jobs[1]['skills'])\n",
        "print(jobs[2]['name'])\n",
        "print(jobs[2]['skills'])\n",
        "print(jobs[3]['name'])\n",
        "print(jobs[3]['skills'])\n",
        "print(jobs[4]['name'])\n",
        "print(jobs[4]['skills'])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dataengineer.txt\n",
            "['J2EE', 'Oracle Fusion', 'Oracle Cloud', 'Salesforce', 'Devops Android', 'Business Analyst', 'UI Developer', 'DBAs', 'Embedded Systems', '.NET', 'Hadoop', 'SQL Developer', 'Big Data', 'Tableau', 'Networking', 'Etl', 'Informatica', 'Ios', 'Quality Analyst', 'Project Manager', 'Python']\n",
            "datascientist.txt\n",
            "['Data Science', 'Python', 'Machine Learning', 'SAS', 'Java', 'Scala', 'Hadoop', 'Hive', 'Bigdata', 'Programming', 'SQL server reporting', 'Msbi', 'Ssrs', 'Msbi', 'Sql', 'Artificial Intelligence', 'Pandas', 'Pyspark', 'Sklearn', 'Flask', 'Django', 'Map Reduce', 'Parametric Design', 'Modeling', 'Regression', 'Patterns', 'Data Mining', 'Text Mining', 'Oops', 'Deep Learning', 'Web Analytics', 'Time Series', 'Regression', 'Tensorflow', 'Azure', 'Linear Regression', 'Logistic Regression', 'Decision Tree', 'Random Forest', 'Data Structure', 'Computer Vision']\n",
            "javadeveloper.txt\n",
            "['SQL Server', 'IBM HTTP', 'IBM WebSphere', 'IHS', 'WAS', 'Java EE', 'SQL Server', '.NET core', 'C#', 'ASP.NET', 'Rdlc', 'Linq', 'Sql', 'Web Api', 'Mvc', 'Javascript', 'Web Services', 'Oracle', 'MS SQL']\n",
            "phpdeveloper.txt\n",
            "['PHP', 'PHP', 'Laravel', 'CodeIgniter', 'Symfony', 'Zend', 'Phalcon', 'CakePHP', 'Yii', 'FuelPHP', 'React', 'Vue', 'Angular', 'Ember', 'Backbone']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "4kLOR9AMZa8s"
      },
      "source": [
        "### Creating cv list\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aE4MW9iGvC73",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# create cvs list\n",
        "cvs=[]\n",
        "cv_dir='/content/drive/My Drive/five_class_entitydata/cv'\n",
        "pathlist = Path(cv_dir).glob('**/*.txt')\n",
        "for path in pathlist:\n",
        "    with open (path, \"r\") as files:\n",
        "      cv={\n",
        "          'name':path.name,\n",
        "           'skills':find_skills(''.join(files.readlines()))\n",
        "      }\n",
        "      cvs.append(cv)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-iDce6OGweJc",
        "colab_type": "code",
        "outputId": "5aeb9c51-79c1-4942-e79f-6997b51ee15d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 200
        }
      },
      "source": [
        "print(cvs[1]['name'])\n",
        "print(cvs[1]['skills'])\n",
        "print(cvs[2]['name'])\n",
        "print(cvs[2]['skills'])\n",
        "print(cvs[3]['name'])\n",
        "print(cvs[3]['skills'])\n",
        "print(cvs[4]['name'])\n",
        "print(cvs[4]['skills'])\n",
        "print(cvs[5]['name'])\n",
        "print(cvs[5]['skills'])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "r4.txt\n",
            "['MySQL', 'PostgreSQL', 'Microsoft Access', 'SQL Server', 'FileMaker', 'Oracle']\n",
            "r3.txt\n",
            "['MySQL', 'PostgreSQL', 'Microsoft Access', 'SQL Server', 'FileMaker', 'Oracle', 'RDBMS', 'dBASE']\n",
            "r2.txt\n",
            "['MySQL', 'PostgreSQL', 'Microsoft Access', 'SQL Server', 'FileMaker', 'Oracle', 'RDBMS', 'dBASE', 'Clipper', 'FoxPro']\n",
            "r8.txt\n",
            "['J2EE', 'Oracle Fusion', 'Oracle Cloud', 'Salesforce', 'Devops Android', 'Business Analyst', 'UI Developer', 'DBAs', 'Embedded Systems', '.NET', 'Hadoop', 'SQL Developer', 'Big Data', 'Tableau', 'Networking']\n",
            "r15.txt\n",
            "['Java', 'Scala', 'Hadoop', 'Hive', 'Bigdata', 'Programming', 'SQL server reporting', 'Artificial Intelligence', 'Pandas', 'Pyspark', 'Sklearn', 'Flask', 'Django', 'Map Reduce', 'Parametric Design', 'Modeling', 'Regression', 'Patterns', 'Data Mining', 'Text Mining', 'Oops', 'Deep Learning', 'Web Analytics', 'Time Series', 'Regression', 'Tensorflow', 'Azure', 'Linear Regression']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "4CCJcfbmZpDi"
      },
      "source": [
        "### Matching both list cv and jobs\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FuzQHEczvC32",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def job_match(text,cv=True):\n",
        "  skills=find_skills(text)\n",
        "  matched=[]\n",
        "  if cv:\n",
        "    for job in jobs:\n",
        "      nskill_job=len(job['skills'])\n",
        "      count=0\n",
        "      for skill in skills:\n",
        "        if skill in job['skills']:\n",
        "          count+=1\n",
        "      matched.append({\n",
        "          'name':job['name'],\n",
        "          'pct':count/nskill_job*100,\n",
        "          'job_skill':job['skills'],\n",
        "          'cv_skill':skills\n",
        "\n",
        "      })\n",
        "  else:\n",
        "    for cv in cvs:\n",
        "      nskill_cv=len(cv['skills'])\n",
        "      count=0\n",
        "      for skill in skills:\n",
        "        if skill in cv['skills']:\n",
        "          count+=1\n",
        "      matched.append({\n",
        "          'name':cv['name'],\n",
        "          'pct':count/nskill_cv*100,\n",
        "          'job_skill':cv['skills'],\n",
        "          'cv_skill':skills\n",
        "\n",
        "      })\n",
        "  return matched\n",
        "      \n",
        "      "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "MXQCS1tLZyxg"
      },
      "source": [
        "### Finding Most Matching Job\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kPV4xOUnvCyN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# find most matching jobs\n",
        "#######################reading the file from folder######################\n",
        "f = open('/content/drive/My Drive/five_class_entitydata/cv/r1.txt', 'r')\n",
        "text = f.read()\n",
        "match_jobs=job_match(text)\n",
        "match_jobs = sorted(match_jobs, key=lambda k: k['pct'],reverse=True) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sc-QS38mzL9b",
        "colab_type": "code",
        "outputId": "f6b43a00-ec65-41c1-d002-148ff334436f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 115
        }
      },
      "source": [
        "for i in range(3):\n",
        "  print(f\"cv matching with {match_jobs[i]['name']}\")\n",
        "  print(f\"{match_jobs[i]['pct']}\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cv matching with backenddeveloper.txt\n",
            "100.0\n",
            "cv matching with javadeveloper.txt\n",
            "10.526315789473683\n",
            "cv matching with dataengineer.txt\n",
            "0.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "pq-HucKtaLUi"
      },
      "source": [
        "### Finding Most Matching Resumes\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S5MfCbLsvNTO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# find most matching cv\n",
        "#######################reading the file from folder######################\n",
        "f = open('/content/drive/My Drive/five_class_entitydata/jobs/dataengineer.txt', 'r')\n",
        "text = f.read()\n",
        "match_cvs=job_match(text,cv=False)\n",
        "match_cvs = sorted(match_cvs, key=lambda k: k['pct'],reverse=True) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hZIekHlPx4Et",
        "colab_type": "code",
        "outputId": "e0fe73fb-56f6-42de-a9b5-3aaf8295d7e5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 342
        }
      },
      "source": [
        "for i in range(10):\n",
        "  print(f\"job matching with cv {match_cvs[i]['name']}\")\n",
        "  print(f\"{match_cvs[i]['pct']}\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "job matching with cv r8.txt\n",
            "100.0\n",
            "job matching with cv r9.txt\n",
            "100.0\n",
            "job matching with cv r7.txt\n",
            "100.0\n",
            "job matching with cv r6.txt\n",
            "100.0\n",
            "job matching with cv r10.txt\n",
            "100.0\n",
            "job matching with cv r13.txt\n",
            "5.555555555555555\n",
            "job matching with cv r12.txt\n",
            "5.128205128205128\n",
            "job matching with cv r11.txt\n",
            "4.878048780487805\n",
            "job matching with cv r15.txt\n",
            "3.571428571428571\n",
            "job matching with cv r14.txt\n",
            "3.125\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "sU6P3B_Ga90P"
      },
      "source": [
        "### Cleanups\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4-U5awHcf6Ox",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "##################################### delete produced resume files\n",
        "i=10\n",
        "while i < 30:\n",
        "  print (\"resume\"+str(i)+\".txt\")\n",
        "  if os.path.isfile(\"resume\"+str(i)+\".txt\"):\n",
        "    print (\"found\")\n",
        "    path = \"resume\"+str(i)+\".txt\" \n",
        "    os.remove(path)\n",
        "    print (\"deleted\")\n",
        "    print (\"..........\")\n",
        "  else:\n",
        "    print (\"not found\")\n",
        "  i+=1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7u_kWJ5u2Yhc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "###################deleting the saved model#################################\n",
        "#  !rm -rf model2\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "mfKE_4_2Y8Ij"
      },
      "source": [
        "### xxxxx\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vPVS91aJED0u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "###################loading the saved model################################\n",
        " output_dir='./model2'\n",
        " nlp2 = spacy.load(output_dir)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p2PHMpHsfWRP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#######################reading the file from folder######################\n",
        "f = open('/content/drive/My Drive/five_class_entitydata/feed1.txt', 'r')\n",
        "text = f.read()\n",
        "# text=\"im competent in java,c# and python\"\n",
        "# text=cleandata(text)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_xqx4SPJfaHW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "docx=nlp2(text)\n",
        "d = {}\n",
        "for ent in docx.ents:\n",
        "  d[ent.label_] = []\n",
        "for ent in docx.ents:\n",
        "  d[ent.label_].append(ent.text)\n",
        "if 'Skills' in d:\n",
        "  skills_=d['Skills']    \n",
        "  print(f'Dedected skills {skills_}')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bzH9fant5LOG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#########################viewving the results####################\n",
        "from spacy import displacy\n",
        "displacy.render(nlp_, style='ent',jupyter=True)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}