roberta_finetune_sts_biomedical_sentence_transformers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python36",
      "display_name": "Python 3.6",
      "language": "python"
    },
    "language_info": {
      "mimetype": "text/x-python",
      "nbconvert_exporter": "python",
      "name": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.6",
      "file_extension": ".py",
      "codemirror_mode": {
        "version": 3,
        "name": "ipython"
      }
    },
    "colab": {
      "name": "roberta_finetune_sts_biomedical_sentence_transformers.ipynb",
      "provenance": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "EX3eKJDE__No",
        "colab_type": "code",
        "outputId": "f94285c3-4471-484b-fda8-3fff1b16baef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KFQ4aEyP8c-d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "my_drive = '/content/drive/My Drive/'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "JufH3ASp0kMJ",
        "colab_type": "code",
        "outputId": "7ad26b60-3ac8-415c-f73d-55ba8a16e0e1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 355
        }
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (2.2.1)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (from transformers) (0.1.83)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from transformers) (4.28.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.17.4)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.9)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.21.0)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.35)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.10.32)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2019.11.28)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.8)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.14.0)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.9.4)\n",
            "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.2.1)\n",
            "Requirement already satisfied: botocore<1.14.0,>=1.13.32 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (1.13.32)\n",
            "Requirement already satisfied: python-dateutil<2.8.1,>=2.1; python_version >= \"2.7\" in /usr/local/lib/python3.6/dist-packages (from botocore<1.14.0,>=1.13.32->boto3->transformers) (2.6.1)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.14.0,>=1.13.32->boto3->transformers) (0.15.2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "y6hfRNET0kMZ",
        "colab_type": "code",
        "outputId": "ba25e1f5-27a1-4443-fe75-4ba97b4c7c8a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 474
        }
      },
      "source": [
        "!pip install -U sentence-transformers"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Processing /root/.cache/pip/wheels/12/a5/1c/03b7d87e027121fe1e23048007594e73f39a23e833658529c7/sentence_transformers-0.2.4.1-cp36-none-any.whl\n",
            "Requirement already satisfied, skipping upgrade: nltk in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (3.2.5)\n",
            "Requirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (1.17.4)\n",
            "Requirement already satisfied, skipping upgrade: scipy in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (1.3.3)\n",
            "Requirement already satisfied, skipping upgrade: scikit-learn in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (0.21.3)\n",
            "Requirement already satisfied, skipping upgrade: transformers==2.2.1 in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (2.2.1)\n",
            "Requirement already satisfied, skipping upgrade: tqdm in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (4.28.1)\n",
            "Requirement already satisfied, skipping upgrade: torch>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (1.3.1)\n",
            "Requirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.6/dist-packages (from nltk->sentence-transformers) (1.12.0)\n",
            "Requirement already satisfied, skipping upgrade: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sentence-transformers) (0.14.0)\n",
            "Requirement already satisfied, skipping upgrade: sentencepiece in /usr/local/lib/python3.6/dist-packages (from transformers==2.2.1->sentence-transformers) (0.1.83)\n",
            "Requirement already satisfied, skipping upgrade: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers==2.2.1->sentence-transformers) (0.0.35)\n",
            "Requirement already satisfied, skipping upgrade: requests in /usr/local/lib/python3.6/dist-packages (from transformers==2.2.1->sentence-transformers) (2.21.0)\n",
            "Requirement already satisfied, skipping upgrade: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers==2.2.1->sentence-transformers) (1.10.32)\n",
            "Requirement already satisfied, skipping upgrade: regex in /usr/local/lib/python3.6/dist-packages (from transformers==2.2.1->sentence-transformers) (2019.12.9)\n",
            "Requirement already satisfied, skipping upgrade: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.2.1->sentence-transformers) (7.0)\n",
            "Requirement already satisfied, skipping upgrade: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.2.1->sentence-transformers) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.2.1->sentence-transformers) (2.8)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.2.1->sentence-transformers) (2019.11.28)\n",
            "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.2.1->sentence-transformers) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers==2.2.1->sentence-transformers) (0.2.1)\n",
            "Requirement already satisfied, skipping upgrade: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers==2.2.1->sentence-transformers) (0.9.4)\n",
            "Requirement already satisfied, skipping upgrade: botocore<1.14.0,>=1.13.32 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers==2.2.1->sentence-transformers) (1.13.32)\n",
            "Requirement already satisfied, skipping upgrade: python-dateutil<2.8.1,>=2.1; python_version >= \"2.7\" in /usr/local/lib/python3.6/dist-packages (from botocore<1.14.0,>=1.13.32->boto3->transformers==2.2.1->sentence-transformers) (2.6.1)\n",
            "Requirement already satisfied, skipping upgrade: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.14.0,>=1.13.32->boto3->transformers==2.2.1->sentence-transformers) (0.15.2)\n",
            "Installing collected packages: sentence-transformers\n",
            "Successfully installed sentence-transformers-0.2.4.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nMR4D0-3AFg0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#!cp '/content/drive/My Drive/roberta_finetuned_stsb.zip' roberta_finetuned_stsb.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bD70FR0ZBCaT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#!unzip roberta_finetuned_stsb.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "tgKRiLBw0kMc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 62
        },
        "outputId": "0236aac3-8d47-4879-8ed6-3f3de6bf0607"
      },
      "source": [
        "\"\"\"\n",
        "This examples trains BERT for the STSbenchmark from scratch. It generates sentence embeddings\n",
        "that can be compared using cosine-similarity to measure the similarity.\n",
        "\"\"\"\n",
        "from torch.utils.data import DataLoader\n",
        "import math\n",
        "from sentence_transformers import SentenceTransformer,  SentencesDataset, LoggingHandler, losses, models\n",
        "from sentence_transformers.evaluation import EmbeddingSimilarityEvaluator\n",
        "from sentence_transformers.readers import STSDataReader\n",
        "import logging\n",
        "from datetime import datetime\n",
        "\n",
        "\n",
        "#### Just some code to print debug information to stdout\n",
        "logging.basicConfig(format='%(asctime)s - %(message)s',\n",
        "                    datefmt='%Y-%m-%d %H:%M:%S',\n",
        "                    level=logging.INFO,\n",
        "                    handlers=[LoggingHandler()])\n",
        "#### /print debug information to stdout"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "mqEWk1v90kMh",
        "colab_type": "code",
        "outputId": "0963b85b-ae5e-47c6-a979-5a6e6b338352",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 867
        }
      },
      "source": [
        "# Read the dataset\n",
        "train_batch_size = 16\n",
        "num_epochs = 4\n",
        "\n",
        "model_save_path = 'output/training_stsb_roberta-'+datetime.now().strftime(\"%Y-%m-%d_%H-%M-%S\")\n",
        "#sts_reader = STSDataReader('stsbenchmark', normalize_scores=True)\n",
        "# parameters for STSDataReader -> dataset_folder, s1_col_idx=5, s2_col_idx=6, score_col_idx=4, delimiter=\"\\t\", quoting=csv.QUOTE_NONE, normalize_scores=True, min_score=0, max_score=5\n",
        "sts_reader = STSDataReader('', s1_col_idx=0, s2_col_idx=1, score_col_idx=2, normalize_scores=True)\n",
        "\n",
        "# Use RoBERTa for mapping tokens to embeddings\n",
        "word_embedding_model = models.RoBERTa('roberta-base')\n",
        "\n",
        "# Apply mean pooling to get one fixed sized sentence vector\n",
        "pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(),\n",
        "                               pooling_mode_mean_tokens=True,\n",
        "                               pooling_mode_cls_token=True,\n",
        "                               pooling_mode_max_tokens=True)\n",
        "\n",
        "model = SentenceTransformer(modules=[word_embedding_model, pooling_model])"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-13 01:47:54 - https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-config.json not found in cache or force_download set to True, downloading to /tmp/tmpnn2ziw5x\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 473/473 [00:00<00:00, 282728.49B/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-13 01:47:55 - copying /tmp/tmpnn2ziw5x to cache at /root/.cache/torch/transformers/e1a2a406b5a05063c31f4dfdee7608986ba7c6393f7f79db5e69dcd197208534.9dad9043216064080cf9dd3711c53c0f11fe2b09313eaa66931057b4bdcaf068\n",
            "2019-12-13 01:47:55 - creating metadata file for /root/.cache/torch/transformers/e1a2a406b5a05063c31f4dfdee7608986ba7c6393f7f79db5e69dcd197208534.9dad9043216064080cf9dd3711c53c0f11fe2b09313eaa66931057b4bdcaf068\n",
            "2019-12-13 01:47:55 - removing temp file /tmp/tmpnn2ziw5x\n",
            "2019-12-13 01:47:55 - loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-config.json from cache at /root/.cache/torch/transformers/e1a2a406b5a05063c31f4dfdee7608986ba7c6393f7f79db5e69dcd197208534.9dad9043216064080cf9dd3711c53c0f11fe2b09313eaa66931057b4bdcaf068\n",
            "2019-12-13 01:47:55 - Model config {\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"finetuning_task\": null,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"is_decoder\": false,\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"num_labels\": 2,\n",
            "  \"output_attentions\": false,\n",
            "  \"output_hidden_states\": false,\n",
            "  \"output_past\": true,\n",
            "  \"pruned_heads\": {},\n",
            "  \"torchscript\": false,\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_bfloat16\": false,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "2019-12-13 01:47:55 - https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-pytorch_model.bin not found in cache or force_download set to True, downloading to /tmp/tmp1eui0dt6\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 501200538/501200538 [00:16<00:00, 30296141.89B/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-13 01:48:12 - copying /tmp/tmp1eui0dt6 to cache at /root/.cache/torch/transformers/228756ed15b6d200d7cb45aaef08c087e2706f54cb912863d2efe07c89584eb7.49b88ba7ec2c26a7558dda98ca3884c3b80fa31cf43a1b1f23aef3ff81ba344e\n",
            "2019-12-13 01:48:14 - creating metadata file for /root/.cache/torch/transformers/228756ed15b6d200d7cb45aaef08c087e2706f54cb912863d2efe07c89584eb7.49b88ba7ec2c26a7558dda98ca3884c3b80fa31cf43a1b1f23aef3ff81ba344e\n",
            "2019-12-13 01:48:14 - removing temp file /tmp/tmp1eui0dt6\n",
            "2019-12-13 01:48:14 - loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-pytorch_model.bin from cache at /root/.cache/torch/transformers/228756ed15b6d200d7cb45aaef08c087e2706f54cb912863d2efe07c89584eb7.49b88ba7ec2c26a7558dda98ca3884c3b80fa31cf43a1b1f23aef3ff81ba344e\n",
            "2019-12-13 01:48:18 - https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json not found in cache or force_download set to True, downloading to /tmp/tmpmuze0e_y\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 898823/898823 [00:00<00:00, 2102407.89B/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-13 01:48:19 - copying /tmp/tmpmuze0e_y to cache at /root/.cache/torch/transformers/d0c5776499adc1ded22493fae699da0971c1ee4c2587111707a4d177d20257a2.ef00af9e673c7160b4d41cfda1f48c5f4cba57d5142754525572a846a1ab1b9b\n",
            "2019-12-13 01:48:19 - creating metadata file for /root/.cache/torch/transformers/d0c5776499adc1ded22493fae699da0971c1ee4c2587111707a4d177d20257a2.ef00af9e673c7160b4d41cfda1f48c5f4cba57d5142754525572a846a1ab1b9b\n",
            "2019-12-13 01:48:19 - removing temp file /tmp/tmpmuze0e_y\n",
            "2019-12-13 01:48:19 - https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt not found in cache or force_download set to True, downloading to /tmp/tmpd92f1rhq\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 456318/456318 [00:00<00:00, 1337302.34B/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-13 01:48:20 - copying /tmp/tmpd92f1rhq to cache at /root/.cache/torch/transformers/b35e7cd126cd4229a746b5d5c29a749e8e84438b14bcdb575950584fe33207e8.70bec105b4158ed9a1747fea67a43f5dee97855c64d62b6ec3742f4cfdb5feda\n",
            "2019-12-13 01:48:20 - creating metadata file for /root/.cache/torch/transformers/b35e7cd126cd4229a746b5d5c29a749e8e84438b14bcdb575950584fe33207e8.70bec105b4158ed9a1747fea67a43f5dee97855c64d62b6ec3742f4cfdb5feda\n",
            "2019-12-13 01:48:20 - removing temp file /tmp/tmpd92f1rhq\n",
            "2019-12-13 01:48:20 - loading file https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json from cache at /root/.cache/torch/transformers/d0c5776499adc1ded22493fae699da0971c1ee4c2587111707a4d177d20257a2.ef00af9e673c7160b4d41cfda1f48c5f4cba57d5142754525572a846a1ab1b9b\n",
            "2019-12-13 01:48:20 - loading file https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt from cache at /root/.cache/torch/transformers/b35e7cd126cd4229a746b5d5c29a749e8e84438b14bcdb575950584fe33207e8.70bec105b4158ed9a1747fea67a43f5dee97855c64d62b6ec3742f4cfdb5feda\n",
            "2019-12-13 01:48:20 - Use pytorch device: cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "Kn7w48Ya0kMn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 186
        },
        "outputId": "47cb1a7e-0328-4295-877b-20ee3914d957"
      },
      "source": [
        "sts_reader = STSDataReader('', s1_col_idx=0, s2_col_idx=1, score_col_idx=2, normalize_scores=True)\n",
        "\n",
        "# Convert the dataset to a DataLoader ready for training\n",
        "logging.info(\"Read STSbenchmark train dataset\")\n",
        "train_data = SentencesDataset(sts_reader.get_examples(my_drive+'clinicalSTS2019.train_train60.csv'), model)\n",
        "train_dataloader = DataLoader(train_data, shuffle=True, batch_size=train_batch_size)\n",
        "train_loss = losses.CosineSimilarityLoss(model=model)\n",
        "\n",
        "\n",
        "logging.info(\"Read STSbenchmark dev dataset\")\n",
        "dev_data = SentencesDataset(examples=sts_reader.get_examples(my_drive+'clinicalSTS2019.train_validate20.csv'), model=model)\n",
        "dev_dataloader = DataLoader(dev_data, shuffle=False, batch_size=train_batch_size)\n",
        "evaluator = EmbeddingSimilarityEvaluator(dev_dataloader)\n"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Convert dataset:  14%|█▍        | 137/992 [00:00<00:00, 1366.55it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-13 01:50:07 - Read STSbenchmark train dataset\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Convert dataset: 100%|██████████| 992/992 [00:00<00:00, 1973.38it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-13 01:50:07 - Num sentences: 992\n",
            "2019-12-13 01:50:07 - Sentences 0 longer than max_seqence_length: 0\n",
            "2019-12-13 01:50:07 - Sentences 1 longer than max_seqence_length: 0\n",
            "2019-12-13 01:50:07 - Read STSbenchmark dev dataset\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Convert dataset: 100%|██████████| 331/331 [00:00<00:00, 2330.79it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-13 01:50:08 - Num sentences: 331\n",
            "2019-12-13 01:50:08 - Sentences 0 longer than max_seqence_length: 0\n",
            "2019-12-13 01:50:08 - Sentences 1 longer than max_seqence_length: 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "qB69vgWi0kMq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        },
        "outputId": "70beaaa8-a357-426d-dde9-32fc51fac856"
      },
      "source": [
        "# Configure the training. We skip evaluation in this example\n",
        "warmup_steps = math.ceil(len(train_data)*num_epochs/train_batch_size*0.1) #10% of train data for warm-up\n",
        "logging.info(\"Warmup-steps: {}\".format(warmup_steps))\n"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-13 01:50:11 - Warmup-steps: 25\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "eYRlcC2f0kMt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 780
        },
        "outputId": "b5fb568f-3dca-439d-942c-2d0bf3908d3e"
      },
      "source": [
        "# Train the model\n",
        "model.fit(train_objectives=[(train_dataloader, train_loss)],\n",
        "          evaluator=evaluator,\n",
        "          epochs=num_epochs,\n",
        "          evaluation_steps=1000,\n",
        "          warmup_steps=warmup_steps,\n",
        "          output_path=model_save_path)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch:   0%|          | 0/4 [00:00<?, ?it/s]\n",
            "Iteration:   0%|          | 0/62 [00:00<?, ?it/s]\u001b[A\n",
            "Iteration:   2%|▏         | 1/62 [00:00<00:32,  1.90it/s]\u001b[A\n",
            "Iteration:   3%|▎         | 2/62 [00:01<00:32,  1.85it/s]\u001b[A\n",
            "Iteration:   5%|▍         | 3/62 [00:01<00:31,  1.85it/s]\u001b[A\n",
            "Iteration:   6%|▋         | 4/62 [00:02<00:28,  2.05it/s]\u001b[A\n",
            "Iteration:   8%|▊         | 5/62 [00:02<00:27,  2.08it/s]\u001b[A\n",
            "Iteration:  10%|▉         | 6/62 [00:02<00:24,  2.25it/s]\u001b[A\n",
            "Iteration:  11%|█▏        | 7/62 [00:03<00:26,  2.09it/s]\u001b[A\n",
            "Iteration:  13%|█▎        | 8/62 [00:03<00:23,  2.27it/s]\u001b[A\n",
            "Iteration:  15%|█▍        | 9/62 [00:04<00:23,  2.28it/s]\u001b[A\n",
            "Iteration:  16%|█▌        | 10/62 [00:04<00:20,  2.49it/s]\u001b[A\n",
            "Iteration:  18%|█▊        | 11/62 [00:04<00:21,  2.37it/s]\u001b[A\n",
            "Iteration:  19%|█▉        | 12/62 [00:05<00:21,  2.35it/s]\u001b[A\n",
            "Iteration:  21%|██        | 13/62 [00:05<00:21,  2.31it/s]\u001b[A\n",
            "Iteration:  23%|██▎       | 14/62 [00:06<00:25,  1.86it/s]\u001b[A\n",
            "Iteration:  24%|██▍       | 15/62 [00:07<00:24,  1.95it/s]\u001b[A\n",
            "Iteration:  26%|██▌       | 16/62 [00:07<00:22,  2.01it/s]\u001b[A\n",
            "Iteration:  27%|██▋       | 17/62 [00:08<00:21,  2.06it/s]\u001b[A\n",
            "Iteration:  29%|██▉       | 18/62 [00:08<00:21,  2.01it/s]\u001b[A\n",
            "Iteration:  31%|███       | 19/62 [00:08<00:19,  2.22it/s]\u001b[A\n",
            "Iteration:  32%|███▏      | 20/62 [00:09<00:20,  2.06it/s]\u001b[A\n",
            "Iteration:  34%|███▍      | 21/62 [00:09<00:18,  2.16it/s]\u001b[A\n",
            "Iteration:  35%|███▌      | 22/62 [00:10<00:17,  2.28it/s]\u001b[A\n",
            "Iteration:  37%|███▋      | 23/62 [00:10<00:19,  2.00it/s]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-17-a8f1dbdc5a41>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m           \u001b[0mevaluation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m           \u001b[0mwarmup_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwarmup_steps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m           output_path=model_save_path)\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sentence_transformers/SentenceTransformer.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, train_objectives, evaluator, epochs, scheduler, warmup_steps, optimizer_class, optimizer_params, weight_decay, evaluation_steps, output_path, save_best_model, max_grad_norm, fp16, fp16_opt_level, local_rank)\u001b[0m\n\u001b[1;32m    332\u001b[0m                 \u001b[0mloss_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    333\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 334\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_train_objectives\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mmin_batch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdesc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"Iteration\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    335\u001b[0m                 \u001b[0midx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mnum_train_objectives\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    336\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tqdm/_tqdm.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1002\u001b[0m                             \u001b[0msp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__repr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1003\u001b[0m                             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpos\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1004\u001b[0;31m                                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmoveto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mabs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpos\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1005\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1006\u001b[0m                         \u001b[0;31m# If no `miniters` was specified, adjust automatically\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tqdm/_tqdm.py\u001b[0m in \u001b[0;36mmoveto\u001b[0;34m(self, n)\u001b[0m\n\u001b[1;32m   1240\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1241\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmoveto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1242\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_unicode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'\\n'\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mn\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0m_term_move_up\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1243\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflush\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1244\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/ipykernel/iostream.py\u001b[0m in \u001b[0;36mwrite\u001b[0;34m(self, string)\u001b[0m\n\u001b[1;32m    350\u001b[0m             \u001b[0mis_child\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_master_process\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    351\u001b[0m             \u001b[0;31m# only touch the buffer in the IO thread to avoid races\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 352\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpub_thread\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mschedule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_buffer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstring\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    353\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_child\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    354\u001b[0m                 \u001b[0;31m# newlines imply flush in subprocesses\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/ipykernel/iostream.py\u001b[0m in \u001b[0;36mschedule\u001b[0;34m(self, f)\u001b[0m\n\u001b[1;32m    188\u001b[0m                 \u001b[0mevent_id\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0murandom\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m16\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    189\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_events\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mevent_id\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 190\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_event_pipe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevent_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    191\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    192\u001b[0m             \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/zmq/sugar/socket.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, data, flags, copy, track, routing_id, group)\u001b[0m\n\u001b[1;32m    389\u001b[0m                                  copy_threshold=self.copy_threshold)\n\u001b[1;32m    390\u001b[0m             \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroup\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 391\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSocket\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mflags\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrack\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrack\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    392\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    393\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msend_multipart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg_parts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrack\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket.Socket.send\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket.Socket.send\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket._send_copy\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/zmq/backend/cython/checkrc.pxd\u001b[0m in \u001b[0;36mzmq.backend.cython.checkrc._check_rc\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "7DzTfukm0kMw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "##############################################################################\n",
        "#\n",
        "# Load the stored model and evaluate its performance on STS benchmark dataset\n",
        "#\n",
        "##############################################################################\n",
        "\n",
        "model = SentenceTransformer(model_save_path)\n",
        "test_data = SentencesDataset(examples=sts_reader.get_examples(\"clinicalSTS2019.train_test20.csv\"), model=model)\n",
        "test_dataloader = DataLoader(test_data, shuffle=False, batch_size=train_batch_size)\n",
        "evaluator = EmbeddingSimilarityEvaluator(test_dataloader)\n",
        "model.evaluate(evaluator)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wsee_LAZATg5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!zip -r output.zip output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LMTz9vGGAdJv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cp output.zip '/content/drive/My Drive/roberta_finetuned_sts_biomedical.zip'"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}