BERT_Fine_Tuning_Tutorial_with_PyTorch (update February 22).ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "5q2LY1BjyNVQ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 180
        },
        "outputId": "e00acf88-5997-470d-96b8-ba26ebfcb90e"
      },
      "outputs": [
        {
          "output_type": "error",
          "ename": "SystemError",
          "evalue": "GPU device not found",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mSystemError\u001b[0m                               Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-cd5bb073bb20>\u001b[0m in \u001b[0;36m<cell line: 7>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Found GPU at: {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0;32mraise\u001b[0m \u001b[0mSystemError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'GPU device not found'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mSystemError\u001b[0m: GPU device not found"
          ]
        }
      ],
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "# Get the GPU device name.\n",
        "device_name = tf.test.gpu_device_name()\n",
        "\n",
        "# The device name should look like the following:\n",
        "if device_name == '/device:GPU:0':\n",
        "    print('Found GPU at: {}'.format(device_name))\n",
        "else:\n",
        "    raise SystemError('GPU device not found')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "w71JPK6By1Um"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "\n",
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():\n",
        "\n",
        "    # Tell PyTorch to use the GPU.\n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WvvABQDT1k4U"
      },
      "outputs": [],
      "source": [
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "C_HpgToU10Fd"
      },
      "outputs": [],
      "source": [
        "##!pip install wget"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9Y1Sfyhw15Cx"
      },
      "outputs": [],
      "source": [
        "##import wget\n",
        "##import os\n",
        "\n",
        "##print('Downloading dataset...')\n",
        "\n",
        "# The URL for the dataset zip file.\n",
        "##url = 'https://nyu-mll.github.io/CoLA/cola_public_1.1.zip'\n",
        "\n",
        "# Download the file (if we haven't already)\n",
        "##if not os.path.exists('./cola_public_1.1.zip'):\n",
        "##    wget.download(url, './cola_public_1.1.zip')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gZ84CM3Q2P7I"
      },
      "outputs": [],
      "source": [
        "# Unzip the dataset (if we haven't already)\n",
        "##if not os.path.exists('./cola_public/'):\n",
        "##    !unzip cola_public_1.1.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UoxEXrZsbagH"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Load the dataset into a pandas dataframe.\n",
        "df = pd.read_csv(\"/content/preprocessed_data(210).csv\", names=['post_number','title','main_content','comments', 'word_count', 'URL', 'Category', 'Classification'])\n",
        "\n",
        "# Report the number of sentences.\n",
        "print('Number of training sentences: {:,}\\n'.format(df.shape[0]))\n",
        "\n",
        "# Display 10 random rows from the data.\n",
        "df.sample(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PbTb7wlBxG5o"
      },
      "outputs": [],
      "source": [
        "len(df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yAUdAC1WxOgw"
      },
      "outputs": [],
      "source": [
        "df = df.dropna(axis=0, how='any')\n",
        "len(df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_JOY18QSo-a0"
      },
      "outputs": [],
      "source": [
        "sentences = df.comments[1:].values\n",
        "labels = df.Category[1:].values\n",
        "print(labels.shape)\n",
        "print(sentences.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qGpWgGSTpRsI"
      },
      "outputs": [],
      "source": [
        "from transformers import BertTokenizer\n",
        "\n",
        "# Load the BERT tokenizer.\n",
        "print('Loading BERT tokenizer...')\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ie9OMZswphHp"
      },
      "outputs": [],
      "source": [
        "# Print the original sentence.\n",
        "print(' Original: ', sentences[1])\n",
        "\n",
        "# Print the sentence split into tokens.\n",
        "print('Tokenized: ', tokenizer.tokenize(sentences[1]))\n",
        "\n",
        "# Print the sentence mapped to token ids.\n",
        "print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentences[1])))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UFL3CbKwt4NW"
      },
      "outputs": [],
      "source": [
        "max_len = 0\n",
        "\n",
        "# For every sentence...\n",
        "for sent in sentences:\n",
        "\n",
        "    # Tokenize the text and add `[CLS]` and `[SEP]` tokens.\n",
        "    input_ids = tokenizer.encode(sent, add_special_tokens=True)\n",
        "\n",
        "    # Update the maximum sentence length.\n",
        "    max_len = max(max_len, len(input_ids))\n",
        "\n",
        "print('Max sentence length: ', max_len)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5UPFtU7zLinm"
      },
      "outputs": [],
      "source": [
        "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "# Convert the labels to a NumPy array\n",
        "\n",
        "labels_array = df.Category[1:].values.astype(float)\n",
        "\n",
        "# Convert the NumPy array to a tensor\n",
        "#labels = torch.tensor(labels_array)\n",
        "labels = torch.tensor(labels_array, dtype=torch.long)\n",
        "\n",
        "\n",
        "input_ids = []\n",
        "attention_masks = []\n",
        "labels_list = []\n",
        "\n",
        "# For every sentence...\n",
        "for sent in sentences:\n",
        "    # `encode_plus` will:\n",
        "    #   (1) Tokenize the sentence.\n",
        "    #   (2) Prepend the `[CLS]` token to the start.\n",
        "    #   (3) Append the `[SEP]` token to the end.\n",
        "    #   (4) Map tokens to their IDs.\n",
        "    #   (5) Pad or truncate the sentence to `max_length`\n",
        "    #   (6) Create attention masks for [PAD] tokens.\n",
        "    encoded_dict = tokenizer.encode_plus(\n",
        "                        sent,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                        max_length = 64,           # Pad & truncate all sentences.\n",
        "\n",
        "                        padding='max_length',  # Use the new padding argument\n",
        "                        truncation=True,  # Explicitly activate truncation\n",
        "                        #pad_to_max_length = True,\n",
        "                        return_attention_mask = True,   # Construct attn. masks.\n",
        "                        return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                   )\n",
        "\n",
        "    # Add the encoded sentence to the list.\n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "\n",
        "    # And its attention mask (simply differentiates padding from non-padding).\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "\n",
        "    #labels_list.append(label)\n",
        "\n",
        "# Convert the lists into tensors.\n",
        "input_ids = torch.cat(input_ids, dim=0)\n",
        "attention_masks = torch.cat(attention_masks, dim=0)\n",
        "\n",
        "#labels = torch.tensor(labels)\n",
        "#labels = torch.tensor(labels, dtype=torch.long)\n",
        "# Convert the labels array to a list\n",
        "#labels = torch.cat(labels_list, dim=0)\n",
        "#labels = torch.tensor(labels_list)\n",
        "#labels = torch.tensor(labels).clone().detach()\n",
        "\n",
        "\n",
        "\n",
        "# 將 labels 的大小從 (6089,) 擴展為 (6089, 1)\n",
        "#labels = labels.unsqueeze(1)\n",
        "\n",
        "# 使用 expand 將 labels 的大小擴展為 (6089, 256)\n",
        "#labels = labels.expand(-1, 256)\n",
        "\n",
        "\n",
        "\n",
        "print('Original: ', sentences[2])\n",
        "print('Token IDs:', input_ids[2])\n",
        "print(labels.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Mqo9PbIWOIQX"
      },
      "outputs": [],
      "source": [
        "print(input_ids.size())\n",
        "print(attention_masks.size())\n",
        "print(labels.size())\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qXkGLQV8Q4Oo"
      },
      "outputs": [],
      "source": [
        "########## if run this step the size become （6089，） dont run！！\n",
        "\n",
        "#labels_array = df.Category[1:].values.astype(float)\n",
        "\n",
        "# Convert the NumPy array to a tensor\n",
        "#labels = torch.tensor(labels_array)\n",
        "#print(labels.size())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bahVORyrijCT"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import TensorDataset, random_split\n",
        "\n",
        "\n",
        "#labels = df['Category'][1:].astype(float).astype(int).values\n",
        "#labels = df['Category'][1:].astype(int).values\n",
        "#labels = torch.tensor(labels).long()\n",
        "# 将标签转换为整数类型\n",
        "#labels = labels.long()\n",
        "# 将标签转换为整数类型\n",
        "#labels = torch.tensor(df['Category'][1:].values).long()\n",
        "\n",
        "# Combine the training inputs into a TensorDataset.\n",
        "dataset = TensorDataset(input_ids, attention_masks, labels)\n",
        "\n",
        "# Create a 90-10 train-validation split.\n",
        "\n",
        "# Calculate the number of samples to include in each set.\n",
        "train_size = int(0.9 * len(dataset))\n",
        "val_size = len(dataset) - train_size\n",
        "\n",
        "# Divide the dataset by randomly selecting samples.\n",
        "train_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n",
        "\n",
        "print('{:>5,} training samples'.format(train_size))\n",
        "print('{:>5,} validation samples'.format(val_size))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "k53XkpGFjTfR"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "\n",
        "# 检查标签中的不同类别值\n",
        "unique_labels = torch.unique(labels)\n",
        "print(unique_labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WCvrMQ78SYsP"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "# The DataLoader needs to know our batch size for training, so we specify it\n",
        "# here. For fine-tuning BERT on a specific task, the authors recommend a batch\n",
        "# size of 16 or 32.\n",
        "batch_size = 16\n",
        "\n",
        "# Create the DataLoaders for our training and validation sets.\n",
        "# We'll take training samples in random order.\n",
        "train_dataloader = DataLoader(\n",
        "            train_dataset,  # The training samples.\n",
        "            sampler = RandomSampler(train_dataset), # Select batches randomly\n",
        "            batch_size = batch_size # Trains with this batch size.\n",
        "        )\n",
        "\n",
        "# For validation the order doesn't matter, so we'll just read them sequentially.\n",
        "validation_dataloader = DataLoader(\n",
        "            val_dataset, # The validation samples.\n",
        "            sampler = SequentialSampler(val_dataset), # Pull out batches sequentially.\n",
        "            batch_size = batch_size # Evaluate with this batch size.\n",
        "        )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RGPI2Bv_Sdhm"
      },
      "outputs": [],
      "source": [
        "### 4. Train Our Classification Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uoNfNu22VhJd"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "\n",
        "\n",
        "unique_classes = df['Category'][1:].nunique()\n",
        "\n",
        "print(unique_classes)  # 輸出不同分類的數量"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IdiEO9O-ShkJ"
      },
      "outputs": [],
      "source": [
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "\n",
        "# Load BertForSequenceClassification, the pretrained BERT model with a single\n",
        "# linear classification layer on top.\n",
        "model = BertForSequenceClassification.from_pretrained(\n",
        "    \"bert-base-uncased\", # Use the 12-layer BERT model, with an uncased vocab.\n",
        "    num_labels = 37, # The number of output labels--2 for binary classification.\n",
        "                    # You can increase this for multi-class tasks.\n",
        "    output_attentions = False, # Whether the model returns attentions weights.\n",
        "    output_hidden_states = False, # Whether the model returns all hidden-states.\n",
        ")\n",
        "\n",
        "# Tell pytorch to run this model on the GPU.\n",
        "\n",
        "model.cuda()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xuGCiNMMWSDx"
      },
      "outputs": [],
      "source": [
        "# Get all of the model's parameters as a list of tuples.\n",
        "params = list(model.named_parameters())\n",
        "\n",
        "print('The BERT model has {:} different named parameters.\\n'.format(len(params)))\n",
        "\n",
        "print('==== Embedding Layer ====\\n')\n",
        "\n",
        "for p in params[0:5]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== First Transformer ====\\n')\n",
        "\n",
        "for p in params[5:21]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== Output Layer ====\\n')\n",
        "\n",
        "for p in params[-4:]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "E2i3z5-xWsfs"
      },
      "outputs": [],
      "source": [
        "# Note: AdamW is a class from the huggingface library (as opposed to pytorch)\n",
        "# I believe the 'W' stands for 'Weight Decay fix\"\n",
        "optimizer = AdamW(model.parameters(),\n",
        "                  lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
        "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
        "                )\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Jq8KTv99W4sO"
      },
      "outputs": [],
      "source": [
        "optimizer = torch.optim.AdamW(model.parameters(),\n",
        "                  lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
        "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
        "                )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "A0z2WMekXI6g"
      },
      "outputs": [],
      "source": [
        "from transformers import get_linear_schedule_with_warmup\n",
        "\n",
        "# Number of training epochs. The BERT authors recommend between 2 and 4.\n",
        "# We chose to run for 4, but we'll see later that this may be over-fitting the\n",
        "# training data.\n",
        "epochs = 4\n",
        "\n",
        "# Total number of training steps is [number of batches] x [number of epochs].\n",
        "# (Note that this is not the same as the number of training samples).\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "# Create the learning rate scheduler.\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer,\n",
        "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
        "                                            num_training_steps = total_steps)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wVy7NzjDXq9U"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "\n",
        "# Function to calculate the accuracy of our predictions vs labels\n",
        "def flat_accuracy(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1b9UiweLYuae"
      },
      "outputs": [],
      "source": [
        "import time\n",
        "import datetime\n",
        "\n",
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    # Round to the nearest second.\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "\n",
        "    # Format as hh:mm:ss\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oXeEO4bwY432"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "import numpy as np\n",
        "\n",
        "# This training code is based on the `run_glue.py` script here:\n",
        "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
        "\n",
        "# Set the seed value all over the place to make this reproducible.\n",
        "seed_val = 42\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "# We'll store a number of quantities such as training and validation loss,\n",
        "# validation accuracy, and timings.\n",
        "training_stats = []\n",
        "\n",
        "# Measure the total training time for the whole run.\n",
        "total_t0 = time.time()\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "# For each epoch...\n",
        "for epoch_i in range(0, epochs):\n",
        "\n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "\n",
        "    # Perform one full pass over the training set.\n",
        "\n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "\n",
        "    # Measure how long the training epoch takes.\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Reset the total loss for this epoch.\n",
        "    total_train_loss = 0\n",
        "\n",
        "\n",
        "\n",
        "    # Put the model into training mode. Don't be mislead--the call to\n",
        "    # `train` just changes the *mode*, it doesn't *perform* the training.\n",
        "    # `dropout` and `batchnorm` layers behave differently during training\n",
        "    # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
        "    model.train()\n",
        "\n",
        "    #train_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n",
        "\n",
        "    # For each batch of training data...\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "\n",
        "        # Progress update every 40 batches.\n",
        "        if step % 40 == 0 and not step == 0:\n",
        "            # Calculate elapsed time in minutes.\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "\n",
        "            # Report progress.\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "        # Unpack this training batch from our dataloader.\n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using the\n",
        "        # `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids\n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "        #b_labels = b_labels.float()\n",
        "        # Always clear any previously calculated gradients before performing a\n",
        "        # backward pass. PyTorch doesn't do this automatically because\n",
        "        # accumulating the gradients is \"convenient while training RNNs\".\n",
        "        # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
        "        model.zero_grad()\n",
        "\n",
        "        # Perform a forward pass (evaluate the model on this training batch).\n",
        "        # The documentation for this `model` function is here:\n",
        "        # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "        # It returns different numbers of parameters depending on what arguments\n",
        "        # arge given and what flags are set. For our useage here, it returns\n",
        "        # the loss (because we provided labels) and the \"logits\"--the model\n",
        "        # outputs prior to activation.\n",
        "\n",
        "        #b_labels = b_labels.float()\n",
        "\n",
        "        loss, logits = model(b_input_ids,\n",
        "                             token_type_ids=None,\n",
        "                             attention_mask=b_input_mask,\n",
        "                             labels=b_labels\n",
        "                             ).to_tuple()\n",
        "\n",
        "        # Accumulate the training loss over all of the batches so that we can\n",
        "        # calculate the average loss at the end. `loss` is a Tensor containing a\n",
        "        # single value; the `.item()` function just returns the Python value\n",
        "        # from the tensor.\n",
        "        total_train_loss += loss.item()\n",
        "\n",
        "        # Perform a backward pass to calculate the gradients.\n",
        "        loss.backward()\n",
        "\n",
        "        # Clip the norm of the gradients to 1.0.\n",
        "        # This is to help prevent the \"exploding gradients\" problem.\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "        # Update parameters and take a step using the computed gradient.\n",
        "        # The optimizer dictates the \"update rule\"--how the parameters are\n",
        "        # modified based on their gradients, the learning rate, etc.\n",
        "        optimizer.step()\n",
        "\n",
        "        # Update the learning rate.\n",
        "        scheduler.step()\n",
        "\n",
        "    # Calculate the average loss over all of the batches.\n",
        "    avg_train_loss = total_train_loss / len(train_dataloader)\n",
        "\n",
        "    # Measure how long this epoch took.\n",
        "    training_time = format_time(time.time() - t0)\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Training epcoh took: {:}\".format(training_time))\n",
        "\n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "    # After the completion of each training epoch, measure our performance on\n",
        "    # our validation set.\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Put the model in evaluation mode--the dropout layers behave differently\n",
        "    # during evaluation.\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables\n",
        "    total_eval_accuracy = 0\n",
        "    total_eval_loss = 0\n",
        "    nb_eval_steps = 0\n",
        "\n",
        "    # Evaluate data for one epoch\n",
        "    for batch in validation_dataloader:\n",
        "\n",
        "        # Unpack this training batch from our dataloader.\n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using\n",
        "        # the `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids\n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "\n",
        "        # Tell pytorch not to bother with constructing the compute graph during\n",
        "        # the forward pass, since this is only needed for backprop (training).\n",
        "        with torch.no_grad():\n",
        "\n",
        "            # Forward pass, calculate logit predictions.\n",
        "            # token_type_ids is the same as the \"segment ids\", which\n",
        "            # differentiates sentence 1 and 2 in 2-sentence tasks.\n",
        "            # The documentation for this `model` function is here:\n",
        "            # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "            # Get the \"logits\" output by the model. The \"logits\" are the output\n",
        "            # values prior to applying an activation function like the softmax.\n",
        "            (loss, logits) = model(b_input_ids,\n",
        "                                   token_type_ids=None,\n",
        "                                   attention_mask=b_input_mask,\n",
        "                                   labels=b_labels).to_tuple()\n",
        "\n",
        "        # Accumulate the validation loss.\n",
        "        total_eval_loss += loss.item()\n",
        "\n",
        "        # Move logits and labels to CPU\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "\n",
        "        # Calculate the accuracy for this batch of test sentences, and\n",
        "        # accumulate it over all batches.\n",
        "        total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
        "\n",
        "\n",
        "    # Report the final accuracy for this validation run.\n",
        "    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
        "    print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
        "\n",
        "    # Calculate the average loss over all of the batches.\n",
        "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
        "\n",
        "    # Measure how long the validation run took.\n",
        "    validation_time = format_time(time.time() - t0)\n",
        "\n",
        "    print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
        "    print(\"  Validation took: {:}\".format(validation_time))\n",
        "\n",
        "    # Record all statistics from this epoch.\n",
        "    training_stats.append(\n",
        "        {\n",
        "            'epoch': epoch_i + 1,\n",
        "            'Training Loss': avg_train_loss,\n",
        "            'Valid. Loss': avg_val_loss,\n",
        "            'Valid. Accur.': avg_val_accuracy,\n",
        "            'Training Time': training_time,\n",
        "            'Validation Time': validation_time\n",
        "        }\n",
        "    )\n",
        "\n",
        "print(\"\")\n",
        "print(\"Training complete!\")\n",
        "\n",
        "print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "b_input_ids.type()\n",
        "#b_input_mask.type()\n",
        "#b_labels.type()"
      ],
      "metadata": {
        "id": "OE-I8TTgKvz2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(batch[0])\n",
        "print(batch[1])\n",
        "print(batch[2])"
      ],
      "metadata": {
        "id": "sPVZsNH3YLOD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "b_labels.type()"
      ],
      "metadata": {
        "id": "1T8LBL4oLPvy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "b_input_mask.type()"
      ],
      "metadata": {
        "id": "YTZYb5V5LMcl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WcJSfa1NJf_f"
      },
      "outputs": [],
      "source": [
        "b_input_ids.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3lYd8-FEfRMh"
      },
      "outputs": [],
      "source": [
        "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=37)\n",
        "last_layer = model.classifier\n",
        "output_dim = last_layer.out_features\n",
        "print(output_dim)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}