semantic_similarity_bert_SNLI.ipynb

{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "ZZmS890a7SsS"
      },
      "source": [
        "## Introduction\n",
        "\n",
        "Semantic Similarity is the task of determining how similar\n",
        "two sentences are.\n",
        "\n",
        "> \n",
        "\n",
        "\n",
        "I used SNLI (Stanford Natural Language Inference) Corpus\n",
        "to predict sentence semantic similarity with Transformers.\n",
        "\n",
        "### References\n",
        "\n",
        "* [BERT](https://arxiv.org/pdf/1810.04805.pdf)\n",
        "* [SNLI](https://nlp.stanford.edu/projects/snli/)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "47aWgdXo7SsT"
      },
      "source": [
        "## Setup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q6X5jmZH7SsU",
        "outputId": "b69255bc-abeb-4513-d92a-9a7e1248f0e9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.26.1-py3-none-any.whl (6.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m56.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.22.4)\n",
            "Collecting huggingface-hub<1.0,>=0.11.0\n",
            "  Downloading huggingface_hub-0.13.1-py3-none-any.whl (199 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.2/199.2 KB\u001b[0m \u001b[31m14.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.25.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m43.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.9.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (4.0.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (1.26.14)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.13.1 tokenizers-0.13.2 transformers-4.26.1\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "!pip install transformers\n",
        "import transformers"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yh-Wa5fb7SsV"
      },
      "source": [
        "## Configuration"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "L2kGr0C67SsW"
      },
      "outputs": [],
      "source": [
        "max_length = 128  # Maximum length of input sentence to the model.\n",
        "batch_size = 32\n",
        "epochs = 5\n",
        "\n",
        "# Labels in our dataset.\n",
        "labels = [\"contradiction\", \"entailment\", \"neutral\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rMSxO40J7SsW"
      },
      "source": [
        "## Load the Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2f2ZLtCf7SsX",
        "outputId": "f3602df5-ecff-457e-ad73-4fa74b658858"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100 11.1M  100 11.1M    0     0  10.6M      0  0:00:01  0:00:01 --:--:-- 10.6M\n",
            "SNLI_Corpus/\n",
            "SNLI_Corpus/snli_1.0_dev.csv\n",
            "SNLI_Corpus/snli_1.0_train.csv\n",
            "SNLI_Corpus/snli_1.0_test.csv\n"
          ]
        }
      ],
      "source": [
        "!curl -LO https://raw.githubusercontent.com/MohamadMerchant/SNLI/master/data.tar.gz\n",
        "!tar -xvzf data.tar.gz"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "kM94o2yUxZvC"
      },
      "source": [
        "**OR Mount Google drive and load the data from it**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gmrj04-ZxY2c",
        "outputId": "e317edb3-cb8a-49be-fa6a-1d87f354bfe1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L0ZPnxYixwYB"
      },
      "source": [
        "**Load the dataset and extract it from Google drive**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XfT3l-bRw9lE",
        "outputId": "58485c25-6a5a-40f5-91fd-c4059845d15b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "SNLI_Corpus/\n",
            "SNLI_Corpus/snli_1.0_dev.csv\n",
            "SNLI_Corpus/snli_1.0_train.csv\n",
            "SNLI_Corpus/snli_1.0_test.csv\n"
          ]
        }
      ],
      "source": [
        "!cp '/content/gdrive/My Drive/Datasets/SNLI_Corpus.tar.gz' '/content/'\n",
        "!tar -xvzf SNLI_Corpus.tar.gz"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-isjyqlwxger"
      },
      "source": [
        "**Save the dataset to Google drive**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VEqpbfmQvbxE"
      },
      "outputs": [],
      "source": [
        "!cp 'data.tar.gz' '/content/gdrive/My Drive/Datasets/SNLI_Corpus.tar.gz'"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "**Reading the data**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g_8g3rGq7SsX",
        "outputId": "905afc78-0869-418d-baac-5c7da47108ec"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Total train samples : 100000\n",
            "Total validation samples: 10000\n",
            "Total test samples: 10000\n"
          ]
        }
      ],
      "source": [
        "# There are more than 550k samples in total; we will use 100k for this example.\n",
        "train_df = pd.read_csv(\"SNLI_Corpus/snli_1.0_train.csv\", nrows=100000)\n",
        "valid_df = pd.read_csv(\"SNLI_Corpus/snli_1.0_dev.csv\")\n",
        "test_df = pd.read_csv(\"SNLI_Corpus/snli_1.0_test.csv\")\n",
        "\n",
        "# Shape of the data\n",
        "print(f\"Total train samples : {train_df.shape[0]}\")\n",
        "print(f\"Total validation samples: {valid_df.shape[0]}\")\n",
        "print(f\"Total test samples: {valid_df.shape[0]}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Av4EDxaC7SsY"
      },
      "source": [
        "Dataset Overview:\n",
        "\n",
        "- sentence1: The premise caption that was supplied to the author of the pair.\n",
        "- sentence2: The hypothesis caption that was written by the author of the pair.\n",
        "- similarity: This is the label chosen by the majority of annotators.\n",
        "Where no majority exists, the label \"-\" is used (we will skip such samples here).\n",
        "\n",
        "Here are the \"similarity\" label values in our dataset:\n",
        "\n",
        "- Contradiction: The sentences share no similarity.\n",
        "- Entailment: The sentences have similar meaning.\n",
        "- Neutral: The sentences are neutral."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mh-JtO3R7SsZ"
      },
      "source": [
        "Let's look at one sample from the dataset:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KPPfHTd07SsZ",
        "outputId": "6ac02934-56e7-49e2-fa16-451a72ed80a3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Sentence1: A person on a horse jumps over a broken down airplane.\n",
            "Sentence2: A person is at a diner, ordering an omelette.\n",
            "Similarity: contradiction\n"
          ]
        }
      ],
      "source": [
        "print(f\"Sentence1: {train_df.loc[1, 'sentence1']}\")\n",
        "print(f\"Sentence2: {train_df.loc[1, 'sentence2']}\")\n",
        "print(f\"Similarity: {train_df.loc[1, 'similarity']}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BBEsPXTv7Ssa"
      },
      "source": [
        "## Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LBaWisZQ7Ssa",
        "outputId": "4a98887d-6314-450a-9442-0c8f9ce45f33"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of missing values\n",
            "similarity    0\n",
            "sentence1     0\n",
            "sentence2     3\n",
            "dtype: int64\n"
          ]
        }
      ],
      "source": [
        "# We have some NaN entries in our train data, we will simply drop them.\n",
        "print(\"Number of missing values\")\n",
        "print(train_df.isnull().sum())\n",
        "train_df.dropna(axis=0, inplace=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bnl-k8Sg7Ssa"
      },
      "source": [
        "Distribution of our training targets."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q9eR87SG7Ssa",
        "outputId": "d9e37456-51c3-4a23-de11-f57ef0921ac8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Target Distribution\n",
            "entailment       33384\n",
            "contradiction    33310\n",
            "neutral          33193\n",
            "-                  110\n",
            "Name: similarity, dtype: int64\n"
          ]
        }
      ],
      "source": [
        "print(\"Train Target Distribution\")\n",
        "print(train_df.similarity.value_counts())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_6g-oYjI7Ssb"
      },
      "source": [
        "Distribution of our validation targets."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aOtCbbal7Ssb",
        "outputId": "f54d61d6-b4e3-4ff5-a7fe-0d96ce941f18"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Validation Target Distribution\n",
            "entailment       3329\n",
            "contradiction    3278\n",
            "neutral          3235\n",
            "-                 158\n",
            "Name: similarity, dtype: int64\n"
          ]
        }
      ],
      "source": [
        "print(\"Validation Target Distribution\")\n",
        "print(valid_df.similarity.value_counts())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9a1Q7oYU7Ssb"
      },
      "source": [
        "The value \"-\" appears as part of our training and validation targets.\n",
        "We will skip these samples."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X96Xw4PY7Ssb"
      },
      "outputs": [],
      "source": [
        "train_df = (\n",
        "    train_df[train_df.similarity != \"-\"]\n",
        "    .sample(frac=1.0, random_state=42)\n",
        "    .reset_index(drop=True)\n",
        ")\n",
        "valid_df = (\n",
        "    valid_df[valid_df.similarity != \"-\"]\n",
        "    .sample(frac=1.0, random_state=42)\n",
        "    .reset_index(drop=True)\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DwZe5mC27Ssb"
      },
      "source": [
        "One-hot encode training, validation, and test labels."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vVITmBXo7Ssc"
      },
      "outputs": [],
      "source": [
        "train_df[\"label\"] = train_df[\"similarity\"].apply(\n",
        "    lambda x: 0 if x == \"contradiction\" else 1 if x == \"entailment\" else 2\n",
        ")\n",
        "y_train = tf.keras.utils.to_categorical(train_df.label, num_classes=3)\n",
        "\n",
        "valid_df[\"label\"] = valid_df[\"similarity\"].apply(\n",
        "    lambda x: 0 if x == \"contradiction\" else 1 if x == \"entailment\" else 2\n",
        ")\n",
        "y_val = tf.keras.utils.to_categorical(valid_df.label, num_classes=3)\n",
        "\n",
        "test_df[\"label\"] = test_df[\"similarity\"].apply(\n",
        "    lambda x: 0 if x == \"contradiction\" else 1 if x == \"entailment\" else 2\n",
        ")\n",
        "y_test = tf.keras.utils.to_categorical(test_df.label, num_classes=3)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5gPYZ03C7Ssc"
      },
      "source": [
        "## Keras Custom Data Generator"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T1vINeMI7Ssc"
      },
      "outputs": [],
      "source": [
        "\n",
        "class BertSemanticDataGenerator(tf.keras.utils.Sequence):\n",
        "    \"\"\"Generates batches of data.\n",
        "\n",
        "    Args:\n",
        "        sentence_pairs: Array of premise and hypothesis input sentences.\n",
        "        labels: Array of labels.\n",
        "        batch_size: Integer batch size.\n",
        "        shuffle: boolean, whether to shuffle the data.\n",
        "        include_targets: boolean, whether to incude the labels.\n",
        "\n",
        "    Returns:\n",
        "        Tuples `([input_ids, attention_mask, `token_type_ids], labels)`\n",
        "        (or just `[input_ids, attention_mask, `token_type_ids]`\n",
        "         if `include_targets=False`)\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        sentence_pairs,\n",
        "        labels,\n",
        "        batch_size=batch_size,\n",
        "        shuffle=True,\n",
        "        include_targets=True,\n",
        "    ):\n",
        "        self.sentence_pairs = sentence_pairs\n",
        "        self.labels = labels\n",
        "        self.shuffle = shuffle\n",
        "        self.batch_size = batch_size\n",
        "        self.include_targets = include_targets\n",
        "        # Load our BERT Tokenizer to encode the text.\n",
        "        # We will use base-base-uncased pretrained model.\n",
        "        self.tokenizer = transformers.BertTokenizer.from_pretrained(\n",
        "            \"bert-base-uncased\", do_lower_case=True\n",
        "        )\n",
        "        self.indexes = np.arange(len(self.sentence_pairs))\n",
        "        self.on_epoch_end()\n",
        "\n",
        "    def __len__(self):\n",
        "        # Denotes the number of batches per epoch.\n",
        "        return len(self.sentence_pairs) // self.batch_size\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        # Retrieves the batch of index.\n",
        "        indexes = self.indexes[idx * self.batch_size : (idx + 1) * self.batch_size]\n",
        "        sentence_pairs = self.sentence_pairs[indexes]\n",
        "\n",
        "        # With BERT tokenizer's batch_encode_plus batch of both the sentences are\n",
        "        # encoded together and separated by [SEP] token.\n",
        "        encoded = self.tokenizer.batch_encode_plus(\n",
        "            sentence_pairs.tolist(),\n",
        "            add_special_tokens=True,\n",
        "            max_length=max_length,\n",
        "            return_attention_mask=True,\n",
        "            return_token_type_ids=True,\n",
        "            pad_to_max_length=True,\n",
        "            return_tensors=\"tf\",\n",
        "        )\n",
        "\n",
        "        # Convert batch of encoded features to numpy array.\n",
        "        input_ids = np.array(encoded[\"input_ids\"], dtype=\"int32\")\n",
        "        attention_masks = np.array(encoded[\"attention_mask\"], dtype=\"int32\")\n",
        "        token_type_ids = np.array(encoded[\"token_type_ids\"], dtype=\"int32\")\n",
        "\n",
        "        # Set to true if data generator is used for training/validation.\n",
        "        if self.include_targets:\n",
        "            labels = np.array(self.labels[indexes], dtype=\"int32\")\n",
        "            return [input_ids, attention_masks, token_type_ids], labels\n",
        "        else:\n",
        "            return [input_ids, attention_masks, token_type_ids]\n",
        "\n",
        "    def on_epoch_end(self):\n",
        "        # Shuffle indexes after each epoch if shuffle is set to True.\n",
        "        if self.shuffle:\n",
        "            np.random.RandomState(42).shuffle(self.indexes)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5bnbXCUa7Ssc"
      },
      "source": [
        "## Build the model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 952,
          "referenced_widgets": [
            "d3fff89b328d4c6e94232f9ab806a3b7",
            "4c7a232fde45459cad319c4ab4c6f341",
            "b5b06be08cd945b19f36240197be2a53",
            "50f26b0b383a4503871862beb811b758",
            "9993f5e65f07431287b102028a1f1e7c",
            "cc9edd23e7ca40abbb15ff18b4cb789d",
            "7af01ea0f8504feb9113c9befc3520d5",
            "762795bd5de2457c85cb96db0fafdb82",
            "c7334a20740142c89e49bdf95f701274",
            "0cdf880f1b024b10b9b3328884447e56",
            "935f602ccf8a40f993ebd5700649d429",
            "14461ab71058459096f4b7baafc1a680",
            "9927124e47ef4cc3b04096c9b0a48434",
            "0a325ca4f368425fb5808178354336b0",
            "2aca288b72254dfc830e932cb7471b3f",
            "1f183225c7f048e1ad42542ef2772831",
            "50f1439f6d324993ba816e45214dc555",
            "0fa158a1539a4520bac64ee670398398",
            "043f9aacaa4945989cbfea46369a883a",
            "9e78dffafe3e4715a280dbb645e6413c",
            "855aef50288145db85b216d14dc7ef55",
            "9950d0328e464bfd9a2c28d5589cb1aa"
          ]
        },
        "id": "aTno3g_17Ssd",
        "outputId": "b63ec8b1-4584-449d-86fb-c9bf512e3fe2"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d3fff89b328d4c6e94232f9ab806a3b7",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "14461ab71058459096f4b7baafc1a680",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading tf_model.h5:   0%|          | 0.00/536M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some layers from the model checkpoint at bert-base-uncased were not used when initializing TFBertModel: ['nsp___cls', 'mlm___cls']\n",
            "- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the layers of TFBertModel were initialized from the model checkpoint at bert-base-uncased.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Strategy: <tensorflow.python.distribute.mirrored_strategy.MirroredStrategy object at 0x7f9bb74b3310>\n",
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_ids (InputLayer)         [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " attention_masks (InputLayer)   [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " token_type_ids (InputLayer)    [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " bert (TFBertMainLayer)         TFBaseModelOutputWi  109482240   ['input_ids[0][0]',              \n",
            "                                thPoolingAndCrossAt               'attention_masks[0][0]',        \n",
            "                                tentions(last_hidde               'token_type_ids[0][0]']         \n",
            "                                n_state=(None, 128,                                               \n",
            "                                 768),                                                            \n",
            "                                 pooler_output=(Non                                               \n",
            "                                e, 768),                                                          \n",
            "                                 past_key_values=No                                               \n",
            "                                ne, hidden_states=N                                               \n",
            "                                one, attentions=Non                                               \n",
            "                                e, cross_attentions                                               \n",
            "                                =None)                                                            \n",
            "                                                                                                  \n",
            " bidirectional (Bidirectional)  (None, 128, 128)     426496      ['bert[0][0]']                   \n",
            "                                                                                                  \n",
            " global_average_pooling1d (Glob  (None, 128)         0           ['bidirectional[0][0]']          \n",
            " alAveragePooling1D)                                                                              \n",
            "                                                                                                  \n",
            " global_max_pooling1d (GlobalMa  (None, 128)         0           ['bidirectional[0][0]']          \n",
            " xPooling1D)                                                                                      \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 256)          0           ['global_average_pooling1d[0][0]'\n",
            "                                                                 , 'global_max_pooling1d[0][0]']  \n",
            "                                                                                                  \n",
            " dropout_37 (Dropout)           (None, 256)          0           ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            " dense (Dense)                  (None, 3)            771         ['dropout_37[0][0]']             \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 109,909,507\n",
            "Trainable params: 427,267\n",
            "Non-trainable params: 109,482,240\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# Create the model under a distribution strategy scope.\n",
        "strategy = tf.distribute.MirroredStrategy()\n",
        "\n",
        "with strategy.scope():\n",
        "    # Encoded token ids from BERT tokenizer.\n",
        "    input_ids = tf.keras.layers.Input(\n",
        "        shape=(max_length,), dtype=tf.int32, name=\"input_ids\"\n",
        "    )\n",
        "    # Attention masks indicates to the model which tokens should be attended to.\n",
        "    attention_masks = tf.keras.layers.Input(\n",
        "        shape=(max_length,), dtype=tf.int32, name=\"attention_masks\"\n",
        "    )\n",
        "    # Token type ids are binary masks identifying different sequences in the model.\n",
        "    token_type_ids = tf.keras.layers.Input(\n",
        "        shape=(max_length,), dtype=tf.int32, name=\"token_type_ids\"\n",
        "    )\n",
        "    # Loading pretrained BERT model.\n",
        "    bert_model = transformers.TFBertModel.from_pretrained(\"bert-base-uncased\")\n",
        "    # Freeze the BERT model to reuse the pretrained features without modifying them.\n",
        "    bert_model.trainable = False\n",
        "\n",
        "    bert_output = bert_model.bert(\n",
        "        input_ids, attention_mask=attention_masks, token_type_ids=token_type_ids\n",
        "    )\n",
        "    sequence_output = bert_output.last_hidden_state\n",
        "    pooled_output = bert_output.pooler_output\n",
        "    # Add trainable layers on top of frozen layers to adapt the pretrained features on the new data.\n",
        "    bi_lstm = tf.keras.layers.Bidirectional(\n",
        "        tf.keras.layers.LSTM(64, return_sequences=True)\n",
        "    )(sequence_output)\n",
        "    # Applying hybrid pooling approach to bi_lstm sequence output.\n",
        "    avg_pool = tf.keras.layers.GlobalAveragePooling1D()(bi_lstm)\n",
        "    max_pool = tf.keras.layers.GlobalMaxPooling1D()(bi_lstm)\n",
        "    concat = tf.keras.layers.concatenate([avg_pool, max_pool])\n",
        "    dropout = tf.keras.layers.Dropout(0.3)(concat)\n",
        "    output = tf.keras.layers.Dense(3, activation=\"softmax\")(dropout)\n",
        "    model = tf.keras.models.Model(\n",
        "        inputs=[input_ids, attention_masks, token_type_ids], outputs=output\n",
        "    )\n",
        "\n",
        "    model.compile(\n",
        "        optimizer=tf.keras.optimizers.Adam(),\n",
        "        loss=\"categorical_crossentropy\",\n",
        "        metrics=[\"acc\"],\n",
        "    )\n",
        "\n",
        "\n",
        "print(f\"Strategy: {strategy}\")\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gpTSiuxF7Ssd"
      },
      "source": [
        "Create train and validation data generators"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "77f776b5df7e40c79567390424a2ece6",
            "b4c4fb9ac89f4f21885e3b982dc47db3",
            "ccbf22f58f7b4decab7e0dfdc8c4e015",
            "b505cfbc0ce0493baf778c35fa00f0c0",
            "c6b0095aee8f461eb4eedcf94c894892",
            "416758f4e9cb4fa5adb998abf97b684d",
            "87b470e6dde749ccb92dac44f0861346",
            "77b5176534d34979996d3325cec25568",
            "1911a57198024c1ab7a7ab6e8dff8a27",
            "9d565a92a7ad4a71a0e499d7baff5cd4",
            "2d3c27d3efa34faabf731439e6cace68",
            "a47f95b4ef7442a2aeec4e1c59daba48",
            "936a0c1b88b04289b4c1b06660bc9092",
            "126cdb01363d4cbc93a665661aaf19cc",
            "97e09f058cee46038b819933d7ff63a1",
            "92d8a9ae98354d619ca04adebbb8b21c",
            "3a24cb742f43487b87267ccfa387328d",
            "eb9273d7841d447492f4e2c3af487e2f",
            "74e9603271f9408d95a654251a88115f",
            "632ca6cab6ed4d7192ae6f0e2b5d7d5b",
            "fa00bda5ed6445368673117e0be326d5",
            "c94ba20d3cb741c090b0055b2b269735"
          ]
        },
        "id": "2GxFCZwP7Ssd",
        "outputId": "c0586a88-74a8-46d7-f60f-4ba79163b960"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "77f776b5df7e40c79567390424a2ece6",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a47f95b4ef7442a2aeec4e1c59daba48",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "train_data = BertSemanticDataGenerator(\n",
        "    train_df[[\"sentence1\", \"sentence2\"]].values.astype(\"str\"),\n",
        "    y_train,\n",
        "    batch_size=batch_size,\n",
        "    shuffle=True,\n",
        ")\n",
        "valid_data = BertSemanticDataGenerator(\n",
        "    valid_df[[\"sentence1\", \"sentence2\"]].values.astype(\"str\"),\n",
        "    y_val,\n",
        "    batch_size=batch_size,\n",
        "    shuffle=False,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "moNFAIWn7Ssd"
      },
      "source": [
        "## Train the Model\n",
        "\n",
        "Training is done only for the top layers to perform \"feature extraction\",\n",
        "which will allow the model to use the representations of the pretrained model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BvH-DpC-7Ssd",
        "outputId": "dabb24c1-38d6-433e-8837-ee685c5a7e8c"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.8/dist-packages/transformers/tokenization_utils_base.py:2339: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/2\n",
            "  26/3121 [..............................] - ETA: 16:20 - loss: 1.1141 - acc: 0.4123"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 118/3121 [>.............................] - ETA: 16:08 - loss: 1.0314 - acc: 0.4767"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 133/3121 [>.............................] - ETA: 16:04 - loss: 1.0283 - acc: 0.4814"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 287/3121 [=>............................] - ETA: 15:14 - loss: 0.9735 - acc: 0.5234"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 305/3121 [=>............................] - ETA: 15:07 - loss: 0.9672 - acc: 0.5276"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 325/3121 [==>...........................] - ETA: 15:01 - loss: 0.9603 - acc: 0.5330"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 353/3121 [==>...........................] - ETA: 14:52 - loss: 0.9530 - acc: 0.5382"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 379/3121 [==>...........................] - ETA: 14:43 - loss: 0.9430 - acc: 0.5463"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 394/3121 [==>...........................] - ETA: 14:38 - loss: 0.9385 - acc: 0.5493"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 399/3121 [==>...........................] - ETA: 14:37 - loss: 0.9365 - acc: 0.5507"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 498/3121 [===>..........................] - ETA: 14:08 - loss: 0.9056 - acc: 0.5741"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 499/3121 [===>..........................] - ETA: 14:08 - loss: 0.9051 - acc: 0.5743"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 580/3121 [====>.........................] - ETA: 13:41 - loss: 0.8831 - acc: 0.5905"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 587/3121 [====>.........................] - ETA: 13:39 - loss: 0.8818 - acc: 0.5914"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 620/3121 [====>.........................] - ETA: 13:28 - loss: 0.8753 - acc: 0.5955"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 665/3121 [=====>........................] - ETA: 13:13 - loss: 0.8654 - acc: 0.6023"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 691/3121 [=====>........................] - ETA: 13:05 - loss: 0.8604 - acc: 0.6050"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 717/3121 [=====>........................] - ETA: 12:56 - loss: 0.8548 - acc: 0.6087"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 768/3121 [======>.......................] - ETA: 12:39 - loss: 0.8451 - acc: 0.6157"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 788/3121 [======>.......................] - ETA: 12:33 - loss: 0.8418 - acc: 0.6180"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 799/3121 [======>.......................] - ETA: 12:29 - loss: 0.8397 - acc: 0.6195"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 858/3121 [=======>......................] - ETA: 12:10 - loss: 0.8306 - acc: 0.6253"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 869/3121 [=======>......................] - ETA: 12:07 - loss: 0.8291 - acc: 0.6262"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 905/3121 [=======>......................] - ETA: 11:55 - loss: 0.8228 - acc: 0.6307"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 935/3121 [=======>......................] - ETA: 11:45 - loss: 0.8181 - acc: 0.6338"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 973/3121 [========>.....................] - ETA: 11:33 - loss: 0.8133 - acc: 0.6369"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1025/3121 [========>.....................] - ETA: 11:16 - loss: 0.8076 - acc: 0.6402"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1204/3121 [==========>...................] - ETA: 10:18 - loss: 0.7898 - acc: 0.6515"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1267/3121 [===========>..................] - ETA: 9:58 - loss: 0.7842 - acc: 0.6550"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1324/3121 [===========>..................] - ETA: 9:40 - loss: 0.7797 - acc: 0.6580"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1377/3121 [============>.................] - ETA: 9:22 - loss: 0.7760 - acc: 0.6600"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1414/3121 [============>.................] - ETA: 9:10 - loss: 0.7729 - acc: 0.6616"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1432/3121 [============>.................] - ETA: 9:05 - loss: 0.7709 - acc: 0.6629"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1454/3121 [============>.................] - ETA: 8:58 - loss: 0.7689 - acc: 0.6640"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1463/3121 [=============>................] - ETA: 8:55 - loss: 0.7686 - acc: 0.6642"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1466/3121 [=============>................] - ETA: 8:54 - loss: 0.7686 - acc: 0.6642"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1546/3121 [=============>................] - ETA: 8:29 - loss: 0.7624 - acc: 0.6683"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1582/3121 [==============>...............] - ETA: 8:17 - loss: 0.7604 - acc: 0.6694"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1585/3121 [==============>...............] - ETA: 8:16 - loss: 0.7601 - acc: 0.6696"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1587/3121 [==============>...............] - ETA: 8:15 - loss: 0.7600 - acc: 0.6697"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1605/3121 [==============>...............] - ETA: 8:09 - loss: 0.7587 - acc: 0.6706"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1616/3121 [==============>...............] - ETA: 8:06 - loss: 0.7577 - acc: 0.6711"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1649/3121 [==============>...............] - ETA: 7:55 - loss: 0.7562 - acc: 0.6718"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1666/3121 [===============>..............] - ETA: 7:50 - loss: 0.7549 - acc: 0.6726"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1689/3121 [===============>..............] - ETA: 7:42 - loss: 0.7537 - acc: 0.6730"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1702/3121 [===============>..............] - ETA: 7:38 - loss: 0.7534 - acc: 0.6729"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1726/3121 [===============>..............] - ETA: 7:30 - loss: 0.7517 - acc: 0.6739"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1752/3121 [===============>..............] - ETA: 7:22 - loss: 0.7501 - acc: 0.6746"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1777/3121 [================>.............] - ETA: 7:14 - loss: 0.7487 - acc: 0.6755"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1787/3121 [================>.............] - ETA: 7:11 - loss: 0.7476 - acc: 0.6759"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1825/3121 [================>.............] - ETA: 6:58 - loss: 0.7448 - acc: 0.6772"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1849/3121 [================>.............] - ETA: 6:50 - loss: 0.7435 - acc: 0.6780"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1858/3121 [================>.............] - ETA: 6:48 - loss: 0.7430 - acc: 0.6783"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1909/3121 [=================>............] - ETA: 6:31 - loss: 0.7399 - acc: 0.6802"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1923/3121 [=================>............] - ETA: 6:27 - loss: 0.7390 - acc: 0.6807"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1949/3121 [=================>............] - ETA: 6:18 - loss: 0.7383 - acc: 0.6813"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1999/3121 [==================>...........] - ETA: 6:02 - loss: 0.7365 - acc: 0.6824"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2026/3121 [==================>...........] - ETA: 5:53 - loss: 0.7345 - acc: 0.6833"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2124/3121 [===================>..........] - ETA: 5:22 - loss: 0.7296 - acc: 0.6857"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2188/3121 [====================>.........] - ETA: 5:01 - loss: 0.7274 - acc: 0.6870"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2328/3121 [=====================>........] - ETA: 4:16 - loss: 0.7219 - acc: 0.6900"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2341/3121 [=====================>........] - ETA: 4:12 - loss: 0.7213 - acc: 0.6904"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2366/3121 [=====================>........] - ETA: 4:03 - loss: 0.7201 - acc: 0.6911"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2381/3121 [=====================>........] - ETA: 3:59 - loss: 0.7195 - acc: 0.6914"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2397/3121 [======================>.......] - ETA: 3:53 - loss: 0.7197 - acc: 0.6914"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2458/3121 [======================>.......] - ETA: 3:34 - loss: 0.7173 - acc: 0.6928"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2464/3121 [======================>.......] - ETA: 3:32 - loss: 0.7171 - acc: 0.6929"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2520/3121 [=======================>......] - ETA: 3:14 - loss: 0.7149 - acc: 0.6939"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2593/3121 [=======================>......] - ETA: 2:50 - loss: 0.7122 - acc: 0.6954"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2664/3121 [========================>.....] - ETA: 2:27 - loss: 0.7102 - acc: 0.6966"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2671/3121 [========================>.....] - ETA: 2:25 - loss: 0.7101 - acc: 0.6966"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r2672/3121 [========================>.....] - ETA: 2:25 - loss: 0.7101 - acc: 0.6966"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2681/3121 [========================>.....] - ETA: 2:22 - loss: 0.7100 - acc: 0.6967"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2725/3121 [=========================>....] - ETA: 2:07 - loss: 0.7092 - acc: 0.6973"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2840/3121 [==========================>...] - ETA: 1:30 - loss: 0.7059 - acc: 0.6990"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2863/3121 [==========================>...] - ETA: 1:23 - loss: 0.7047 - acc: 0.6997"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2868/3121 [==========================>...] - ETA: 1:21 - loss: 0.7046 - acc: 0.6997"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "3024/3121 [============================>.] - ETA: 31s - loss: 0.7002 - acc: 0.7021"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "3121/3121 [==============================] - ETA: 0s - loss: 0.6979 - acc: 0.7037"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r3121/3121 [==============================] - 1124s 353ms/step - loss: 0.6979 - acc: 0.7037 - val_loss: 0.5311 - val_acc: 0.7933\n",
            "Epoch 2/2\n",
            "  28/3121 [..............................] - ETA: 16:36 - loss: 0.6102 - acc: 0.7578"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  56/3121 [..............................] - ETA: 16:27 - loss: 0.6139 - acc: 0.7511"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  64/3121 [..............................] - ETA: 16:23 - loss: 0.6231 - acc: 0.7441"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  91/3121 [..............................] - ETA: 16:14 - loss: 0.6174 - acc: 0.7455"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 125/3121 [>.............................] - ETA: 16:03 - loss: 0.6155 - acc: 0.7487"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 172/3121 [>.............................] - ETA: 15:48 - loss: 0.6223 - acc: 0.7442"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 263/3121 [=>............................] - ETA: 15:19 - loss: 0.6119 - acc: 0.7489"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 314/3121 [==>...........................] - ETA: 15:02 - loss: 0.6150 - acc: 0.7470"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 326/3121 [==>...........................] - ETA: 14:58 - loss: 0.6143 - acc: 0.7472"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 346/3121 [==>...........................] - ETA: 14:52 - loss: 0.6122 - acc: 0.7481"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 380/3121 [==>...........................] - ETA: 14:40 - loss: 0.6091 - acc: 0.7496"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 493/3121 [===>..........................] - ETA: 14:04 - loss: 0.6140 - acc: 0.7480"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 524/3121 [====>.........................] - ETA: 13:55 - loss: 0.6147 - acc: 0.7473"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 526/3121 [====>.........................] - ETA: 13:54 - loss: 0.6148 - acc: 0.7471"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 606/3121 [====>.........................] - ETA: 13:28 - loss: 0.6107 - acc: 0.7495"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 616/3121 [====>.........................] - ETA: 13:25 - loss: 0.6107 - acc: 0.7492"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 685/3121 [=====>........................] - ETA: 13:03 - loss: 0.6093 - acc: 0.7504"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 701/3121 [=====>........................] - ETA: 12:58 - loss: 0.6093 - acc: 0.7504"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 705/3121 [=====>........................] - ETA: 12:56 - loss: 0.6090 - acc: 0.7504"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 711/3121 [=====>........................] - ETA: 12:54 - loss: 0.6083 - acc: 0.7508"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 717/3121 [=====>........................] - ETA: 12:53 - loss: 0.6078 - acc: 0.7510"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 764/3121 [======>.......................] - ETA: 12:38 - loss: 0.6087 - acc: 0.7507"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 796/3121 [======>.......................] - ETA: 12:27 - loss: 0.6073 - acc: 0.7513"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 804/3121 [======>.......................] - ETA: 12:25 - loss: 0.6070 - acc: 0.7514"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 894/3121 [=======>......................] - ETA: 11:56 - loss: 0.6064 - acc: 0.7515"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 945/3121 [========>.....................] - ETA: 11:39 - loss: 0.6074 - acc: 0.7509"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 966/3121 [========>.....................] - ETA: 11:33 - loss: 0.6068 - acc: 0.7513"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1032/3121 [========>.....................] - ETA: 11:11 - loss: 0.6051 - acc: 0.7520"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1083/3121 [=========>....................] - ETA: 10:55 - loss: 0.6049 - acc: 0.7522"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1198/3121 [==========>...................] - ETA: 10:18 - loss: 0.6033 - acc: 0.7534"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1201/3121 [==========>...................] - ETA: 10:17 - loss: 0.6036 - acc: 0.7533"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1248/3121 [==========>...................] - ETA: 10:02 - loss: 0.6023 - acc: 0.7538"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1313/3121 [===========>..................] - ETA: 9:41 - loss: 0.6033 - acc: 0.7534"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1342/3121 [===========>..................] - ETA: 9:32 - loss: 0.6035 - acc: 0.7532"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1461/3121 [=============>................] - ETA: 8:53 - loss: 0.6027 - acc: 0.7539"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1466/3121 [=============>................] - ETA: 8:52 - loss: 0.6028 - acc: 0.7538"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1543/3121 [=============>................] - ETA: 8:27 - loss: 0.6021 - acc: 0.7543"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1585/3121 [==============>...............] - ETA: 8:13 - loss: 0.6014 - acc: 0.7545"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1612/3121 [==============>...............] - ETA: 8:05 - loss: 0.6007 - acc: 0.7550"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1617/3121 [==============>...............] - ETA: 8:03 - loss: 0.6007 - acc: 0.7550"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1826/3121 [================>.............] - ETA: 6:56 - loss: 0.5989 - acc: 0.7556"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1843/3121 [================>.............] - ETA: 6:50 - loss: 0.5988 - acc: 0.7556"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1879/3121 [=================>............] - ETA: 6:39 - loss: 0.5983 - acc: 0.7556"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1931/3121 [=================>............] - ETA: 6:22 - loss: 0.5978 - acc: 0.7559"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1962/3121 [=================>............] - ETA: 6:12 - loss: 0.5977 - acc: 0.7559"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1998/3121 [==================>...........] - ETA: 6:01 - loss: 0.5972 - acc: 0.7559"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2055/3121 [==================>...........] - ETA: 5:42 - loss: 0.5977 - acc: 0.7558"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2063/3121 [==================>...........] - ETA: 5:40 - loss: 0.5975 - acc: 0.7559"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2087/3121 [===================>..........] - ETA: 5:32 - loss: 0.5979 - acc: 0.7558"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2156/3121 [===================>..........] - ETA: 5:10 - loss: 0.5976 - acc: 0.7563"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2170/3121 [===================>..........] - ETA: 5:05 - loss: 0.5978 - acc: 0.7563"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2184/3121 [===================>..........] - ETA: 5:01 - loss: 0.5979 - acc: 0.7562"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2201/3121 [====================>.........] - ETA: 4:55 - loss: 0.5985 - acc: 0.7560"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2219/3121 [====================>.........] - ETA: 4:50 - loss: 0.5984 - acc: 0.7560"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2242/3121 [====================>.........] - ETA: 4:42 - loss: 0.5984 - acc: 0.7561"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2253/3121 [====================>.........] - ETA: 4:39 - loss: 0.5982 - acc: 0.7560"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2256/3121 [====================>.........] - ETA: 4:38 - loss: 0.5983 - acc: 0.7560"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2284/3121 [====================>.........] - ETA: 4:29 - loss: 0.5981 - acc: 0.7561"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2356/3121 [=====================>........] - ETA: 4:05 - loss: 0.5972 - acc: 0.7565"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2372/3121 [=====================>........] - ETA: 4:00 - loss: 0.5973 - acc: 0.7566"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2458/3121 [======================>.......] - ETA: 3:33 - loss: 0.5968 - acc: 0.7566"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2486/3121 [======================>.......] - ETA: 3:24 - loss: 0.5967 - acc: 0.7564"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2522/3121 [=======================>......] - ETA: 3:12 - loss: 0.5961 - acc: 0.7566"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2524/3121 [=======================>......] - ETA: 3:11 - loss: 0.5961 - acc: 0.7566"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2550/3121 [=======================>......] - ETA: 3:03 - loss: 0.5968 - acc: 0.7564"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2571/3121 [=======================>......] - ETA: 2:56 - loss: 0.5965 - acc: 0.7565"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2587/3121 [=======================>......] - ETA: 2:51 - loss: 0.5965 - acc: 0.7566"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2608/3121 [========================>.....] - ETA: 2:44 - loss: 0.5960 - acc: 0.7568"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2621/3121 [========================>.....] - ETA: 2:40 - loss: 0.5964 - acc: 0.7566"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2661/3121 [========================>.....] - ETA: 2:27 - loss: 0.5957 - acc: 0.7567"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2674/3121 [========================>.....] - ETA: 2:23 - loss: 0.5956 - acc: 0.7567"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2770/3121 [=========================>....] - ETA: 1:52 - loss: 0.5947 - acc: 0.7572"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2828/3121 [==========================>...] - ETA: 1:34 - loss: 0.5942 - acc: 0.7577"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2849/3121 [==========================>...] - ETA: 1:27 - loss: 0.5942 - acc: 0.7577"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "3021/3121 [============================>.] - ETA: 32s - loss: 0.5935 - acc: 0.7580"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "3061/3121 [============================>.] - ETA: 19s - loss: 0.5935 - acc: 0.7581"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "3069/3121 [============================>.] - ETA: 16s - loss: 0.5936 - acc: 0.7580"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "3121/3121 [==============================] - ETA: 0s - loss: 0.5939 - acc: 0.7579"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r3121/3121 [==============================] - 1093s 350ms/step - loss: 0.5939 - acc: 0.7579 - val_loss: 0.4947 - val_acc: 0.8051\n"
          ]
        }
      ],
      "source": [
        "history = model.fit(\n",
        "    train_data,\n",
        "    validation_data=valid_data,\n",
        "    epochs=epochs,\n",
        "    use_multiprocessing=True,\n",
        "    workers=-1,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xwmw5uGw7Ssd"
      },
      "source": [
        "## Fine-tuning\n",
        "\n",
        "This step must only be performed after the feature extraction model has\n",
        "been trained to convergence on the new data.\n",
        "\n",
        "This is an optional last step where `bert_model` is unfreezed and retrained\n",
        "with a very low learning rate. This can deliver meaningful improvement by\n",
        "incrementally adapting the pretrained features to the new data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YtM1CSQE7Sse",
        "outputId": "26a3c2e7-1dfe-4092-ed7e-125466a6f87d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_ids (InputLayer)         [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " attention_masks (InputLayer)   [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " token_type_ids (InputLayer)    [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " bert (TFBertMainLayer)         TFBaseModelOutputWi  109482240   ['input_ids[0][0]',              \n",
            "                                thPoolingAndCrossAt               'attention_masks[0][0]',        \n",
            "                                tentions(last_hidde               'token_type_ids[0][0]']         \n",
            "                                n_state=(None, 128,                                               \n",
            "                                 768),                                                            \n",
            "                                 pooler_output=(Non                                               \n",
            "                                e, 768),                                                          \n",
            "                                 past_key_values=No                                               \n",
            "                                ne, hidden_states=N                                               \n",
            "                                one, attentions=Non                                               \n",
            "                                e, cross_attentions                                               \n",
            "                                =None)                                                            \n",
            "                                                                                                  \n",
            " bidirectional (Bidirectional)  (None, 128, 128)     426496      ['bert[0][0]']                   \n",
            "                                                                                                  \n",
            " global_average_pooling1d (Glob  (None, 128)         0           ['bidirectional[0][0]']          \n",
            " alAveragePooling1D)                                                                              \n",
            "                                                                                                  \n",
            " global_max_pooling1d (GlobalMa  (None, 128)         0           ['bidirectional[0][0]']          \n",
            " xPooling1D)                                                                                      \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 256)          0           ['global_average_pooling1d[0][0]'\n",
            "                                                                 , 'global_max_pooling1d[0][0]']  \n",
            "                                                                                                  \n",
            " dropout_37 (Dropout)           (None, 256)          0           ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            " dense (Dense)                  (None, 3)            771         ['dropout_37[0][0]']             \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 109,909,507\n",
            "Trainable params: 109,909,507\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# Unfreeze the bert_model.\n",
        "bert_model.trainable = True\n",
        "# Recompile the model to make the change effective.\n",
        "model.compile(\n",
        "    optimizer=tf.keras.optimizers.Adam(1e-5),\n",
        "    loss=\"categorical_crossentropy\",\n",
        "    metrics=[\"accuracy\"],\n",
        ")\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LRMTD7LB7Sse"
      },
      "source": [
        "# Train the entire model end-to-end."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xzV2ibZz7Sse",
        "outputId": "c2b7156b-35e0-4173-a285-fbc46c6598c0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/2\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss` argument?\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss` argument?\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss` argument?\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss` argument?\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "   9/3121 [..............................] - ETA: 43:36 - loss: 0.5596 - accuracy: 0.7639"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  70/3121 [..............................] - ETA: 43:15 - loss: 0.5431 - accuracy: 0.7817"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 103/3121 [..............................] - ETA: 42:28 - loss: 0.5375 - accuracy: 0.7837"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 153/3121 [>.............................] - ETA: 41:41 - loss: 0.5402 - accuracy: 0.7810"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 157/3121 [>.............................] - ETA: 41:38 - loss: 0.5411 - accuracy: 0.7811"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 167/3121 [>.............................] - ETA: 41:29 - loss: 0.5380 - accuracy: 0.7824"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 223/3121 [=>............................] - ETA: 40:37 - loss: 0.5318 - accuracy: 0.7873"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 242/3121 [=>............................] - ETA: 40:20 - loss: 0.5294 - accuracy: 0.7887"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 264/3121 [=>............................] - ETA: 40:01 - loss: 0.5314 - accuracy: 0.7891"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 371/3121 [==>...........................] - ETA: 38:30 - loss: 0.5268 - accuracy: 0.7901"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 386/3121 [==>...........................] - ETA: 38:17 - loss: 0.5277 - accuracy: 0.7892"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 387/3121 [==>...........................] - ETA: 38:16 - loss: 0.5279 - accuracy: 0.7890"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 400/3121 [==>...........................] - ETA: 38:05 - loss: 0.5285 - accuracy: 0.7883"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 441/3121 [===>..........................] - ETA: 37:31 - loss: 0.5219 - accuracy: 0.7915"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 471/3121 [===>..........................] - ETA: 37:05 - loss: 0.5187 - accuracy: 0.7933"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 493/3121 [===>..........................] - ETA: 36:46 - loss: 0.5163 - accuracy: 0.7948"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 495/3121 [===>..........................] - ETA: 36:45 - loss: 0.5157 - accuracy: 0.7952"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 596/3121 [====>.........................] - ETA: 35:19 - loss: 0.5138 - accuracy: 0.7966"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 658/3121 [=====>........................] - ETA: 34:27 - loss: 0.5143 - accuracy: 0.7971"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 771/3121 [======>.......................] - ETA: 32:51 - loss: 0.5115 - accuracy: 0.7977"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 778/3121 [======>.......................] - ETA: 32:45 - loss: 0.5113 - accuracy: 0.7977"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 806/3121 [======>.......................] - ETA: 32:22 - loss: 0.5109 - accuracy: 0.7978"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 895/3121 [=======>......................] - ETA: 31:07 - loss: 0.5096 - accuracy: 0.7991"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 919/3121 [=======>......................] - ETA: 30:46 - loss: 0.5084 - accuracy: 0.7997"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 952/3121 [========>.....................] - ETA: 30:19 - loss: 0.5089 - accuracy: 0.7996"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 967/3121 [========>.....................] - ETA: 30:06 - loss: 0.5082 - accuracy: 0.7998"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1042/3121 [=========>....................] - ETA: 29:03 - loss: 0.5061 - accuracy: 0.8013"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1058/3121 [=========>....................] - ETA: 28:50 - loss: 0.5063 - accuracy: 0.8013"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1097/3121 [=========>....................] - ETA: 28:17 - loss: 0.5053 - accuracy: 0.8016"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r1098/3121 [=========>....................] - ETA: 28:16 - loss: 0.5051 - accuracy: 0.8016"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1103/3121 [=========>....................] - ETA: 28:12 - loss: 0.5047 - accuracy: 0.8018"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1141/3121 [=========>....................] - ETA: 27:40 - loss: 0.5038 - accuracy: 0.8020"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1160/3121 [==========>...................] - ETA: 27:24 - loss: 0.5037 - accuracy: 0.8020"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1170/3121 [==========>...................] - ETA: 27:16 - loss: 0.5033 - accuracy: 0.8022"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1217/3121 [==========>...................] - ETA: 26:37 - loss: 0.5012 - accuracy: 0.8031"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r1218/3121 [==========>...................] - ETA: 26:36 - loss: 0.5012 - accuracy: 0.8031"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1384/3121 [============>.................] - ETA: 24:16 - loss: 0.4958 - accuracy: 0.8053"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1458/3121 [=============>................] - ETA: 23:14 - loss: 0.4945 - accuracy: 0.8060"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1474/3121 [=============>................] - ETA: 23:01 - loss: 0.4945 - accuracy: 0.8061"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1516/3121 [=============>................] - ETA: 22:25 - loss: 0.4938 - accuracy: 0.8062"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1521/3121 [=============>................] - ETA: 22:21 - loss: 0.4937 - accuracy: 0.8064"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1562/3121 [==============>...............] - ETA: 21:47 - loss: 0.4926 - accuracy: 0.8067"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1627/3121 [==============>...............] - ETA: 20:52 - loss: 0.4916 - accuracy: 0.8074"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1638/3121 [==============>...............] - ETA: 20:43 - loss: 0.4915 - accuracy: 0.8073"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1755/3121 [===============>..............] - ETA: 19:05 - loss: 0.4901 - accuracy: 0.8080"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1799/3121 [================>.............] - ETA: 18:28 - loss: 0.4895 - accuracy: 0.8083"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1870/3121 [================>.............] - ETA: 17:28 - loss: 0.4879 - accuracy: 0.8091"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1961/3121 [=================>............] - ETA: 16:12 - loss: 0.4858 - accuracy: 0.8101"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2005/3121 [==================>...........] - ETA: 15:35 - loss: 0.4854 - accuracy: 0.8105"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2017/3121 [==================>...........] - ETA: 15:25 - loss: 0.4854 - accuracy: 0.8105"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r2018/3121 [==================>...........] - ETA: 15:24 - loss: 0.4853 - accuracy: 0.8106"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2074/3121 [==================>...........] - ETA: 14:37 - loss: 0.4844 - accuracy: 0.8110"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2130/3121 [===================>..........] - ETA: 13:50 - loss: 0.4840 - accuracy: 0.8111"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2204/3121 [====================>.........] - ETA: 12:48 - loss: 0.4832 - accuracy: 0.8116"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2207/3121 [====================>.........] - ETA: 12:46 - loss: 0.4833 - accuracy: 0.8116"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2234/3121 [====================>.........] - ETA: 12:23 - loss: 0.4827 - accuracy: 0.8120"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2268/3121 [====================>.........] - ETA: 11:55 - loss: 0.4820 - accuracy: 0.8122"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2273/3121 [====================>.........] - ETA: 11:50 - loss: 0.4819 - accuracy: 0.8123"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2337/3121 [=====================>........] - ETA: 10:57 - loss: 0.4804 - accuracy: 0.8131"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2360/3121 [=====================>........] - ETA: 10:37 - loss: 0.4801 - accuracy: 0.8134"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2379/3121 [=====================>........] - ETA: 10:22 - loss: 0.4800 - accuracy: 0.8134"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2393/3121 [======================>.......] - ETA: 10:10 - loss: 0.4795 - accuracy: 0.8136"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2396/3121 [======================>.......] - ETA: 10:07 - loss: 0.4794 - accuracy: 0.8137"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2421/3121 [======================>.......] - ETA: 9:46 - loss: 0.4788 - accuracy: 0.8140"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2462/3121 [======================>.......] - ETA: 9:12 - loss: 0.4785 - accuracy: 0.8143"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2575/3121 [=======================>......] - ETA: 7:37 - loss: 0.4771 - accuracy: 0.8150"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2580/3121 [=======================>......] - ETA: 7:33 - loss: 0.4769 - accuracy: 0.8152"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2627/3121 [========================>.....] - ETA: 6:54 - loss: 0.4761 - accuracy: 0.8154"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2703/3121 [========================>.....] - ETA: 5:50 - loss: 0.4752 - accuracy: 0.8158"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2799/3121 [=========================>....] - ETA: 4:29 - loss: 0.4742 - accuracy: 0.8162"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2877/3121 [==========================>...] - ETA: 3:24 - loss: 0.4728 - accuracy: 0.8168"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2908/3121 [==========================>...] - ETA: 2:58 - loss: 0.4719 - accuracy: 0.8171"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2912/3121 [==========================>...] - ETA: 2:55 - loss: 0.4718 - accuracy: 0.8172"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2945/3121 [===========================>..] - ETA: 2:27 - loss: 0.4714 - accuracy: 0.8174"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "3019/3121 [============================>.] - ETA: 1:25 - loss: 0.4705 - accuracy: 0.8179"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "3029/3121 [============================>.] - ETA: 1:17 - loss: 0.4705 - accuracy: 0.8179"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "3050/3121 [============================>.] - ETA: 59s - loss: 0.4701 - accuracy: 0.8181 "
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "3093/3121 [============================>.] - ETA: 23s - loss: 0.4696 - accuracy: 0.8183"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "3109/3121 [============================>.] - ETA: 10s - loss: 0.4693 - accuracy: 0.8185"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "3121/3121 [==============================] - ETA: 0s - loss: 0.4689 - accuracy: 0.8186"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r3121/3121 [==============================] - 2812s 885ms/step - loss: 0.4689 - accuracy: 0.8186 - val_loss: 0.3621 - val_accuracy: 0.8651\n",
            "Epoch 2/2\n",
            "  15/3121 [..............................] - ETA: 43:04 - loss: 0.3823 - accuracy: 0.8521"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  18/3121 [..............................] - ETA: 43:03 - loss: 0.3866 - accuracy: 0.8542"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  70/3121 [..............................] - ETA: 43:06 - loss: 0.3496 - accuracy: 0.8701"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  85/3121 [..............................] - ETA: 42:46 - loss: 0.3577 - accuracy: 0.8699"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  99/3121 [..............................] - ETA: 42:28 - loss: 0.3542 - accuracy: 0.8715"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 107/3121 [>.............................] - ETA: 42:18 - loss: 0.3592 - accuracy: 0.8703"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 109/3121 [>.............................] - ETA: 42:16 - loss: 0.3582 - accuracy: 0.8710"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 116/3121 [>.............................] - ETA: 42:08 - loss: 0.3573 - accuracy: 0.8704"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 174/3121 [>.............................] - ETA: 41:18 - loss: 0.3653 - accuracy: 0.8664"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 186/3121 [>.............................] - ETA: 41:08 - loss: 0.3654 - accuracy: 0.8661"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 336/3121 [==>...........................] - ETA: 38:59 - loss: 0.3632 - accuracy: 0.8669"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 365/3121 [==>...........................] - ETA: 38:34 - loss: 0.3619 - accuracy: 0.8663"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 366/3121 [==>...........................] - ETA: 38:33 - loss: 0.3621 - accuracy: 0.8661"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 373/3121 [==>...........................] - ETA: 38:28 - loss: 0.3631 - accuracy: 0.8659"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 410/3121 [==>...........................] - ETA: 37:57 - loss: 0.3646 - accuracy: 0.8654"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 431/3121 [===>..........................] - ETA: 37:39 - loss: 0.3631 - accuracy: 0.8656"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 443/3121 [===>..........................] - ETA: 37:29 - loss: 0.3631 - accuracy: 0.8662"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 458/3121 [===>..........................] - ETA: 37:16 - loss: 0.3653 - accuracy: 0.8654"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 475/3121 [===>..........................] - ETA: 37:02 - loss: 0.3644 - accuracy: 0.8656"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 546/3121 [====>.........................] - ETA: 36:02 - loss: 0.3633 - accuracy: 0.8657"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 610/3121 [====>.........................] - ETA: 35:08 - loss: 0.3633 - accuracy: 0.8660"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 623/3121 [====>.........................] - ETA: 34:57 - loss: 0.3638 - accuracy: 0.8661"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 715/3121 [=====>........................] - ETA: 33:39 - loss: 0.3636 - accuracy: 0.8658"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 747/3121 [======>.......................] - ETA: 33:12 - loss: 0.3627 - accuracy: 0.8657"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 765/3121 [======>.......................] - ETA: 32:57 - loss: 0.3618 - accuracy: 0.8661"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 797/3121 [======>.......................] - ETA: 32:30 - loss: 0.3613 - accuracy: 0.8661"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 807/3121 [======>.......................] - ETA: 32:21 - loss: 0.3615 - accuracy: 0.8661"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 824/3121 [======>.......................] - ETA: 32:07 - loss: 0.3603 - accuracy: 0.8667"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 878/3121 [=======>......................] - ETA: 31:22 - loss: 0.3587 - accuracy: 0.8673"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 892/3121 [=======>......................] - ETA: 31:10 - loss: 0.3580 - accuracy: 0.8674"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 975/3121 [========>.....................] - ETA: 30:00 - loss: 0.3576 - accuracy: 0.8677"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1021/3121 [========>.....................] - ETA: 29:22 - loss: 0.3571 - accuracy: 0.8677"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1062/3121 [=========>....................] - ETA: 28:47 - loss: 0.3576 - accuracy: 0.8671"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1145/3121 [==========>...................] - ETA: 27:38 - loss: 0.3586 - accuracy: 0.8671"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1198/3121 [==========>...................] - ETA: 26:53 - loss: 0.3584 - accuracy: 0.8671"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1285/3121 [===========>..................] - ETA: 25:40 - loss: 0.3574 - accuracy: 0.8670"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1300/3121 [===========>..................] - ETA: 25:27 - loss: 0.3571 - accuracy: 0.8671"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1347/3121 [===========>..................] - ETA: 24:48 - loss: 0.3578 - accuracy: 0.8668"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1355/3121 [============>.................] - ETA: 24:41 - loss: 0.3578 - accuracy: 0.8668"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1413/3121 [============>.................] - ETA: 23:52 - loss: 0.3569 - accuracy: 0.8673"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1454/3121 [============>.................] - ETA: 23:18 - loss: 0.3575 - accuracy: 0.8673"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1456/3121 [============>.................] - ETA: 23:16 - loss: 0.3573 - accuracy: 0.8674"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1462/3121 [=============>................] - ETA: 23:11 - loss: 0.3573 - accuracy: 0.8673"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1509/3121 [=============>................] - ETA: 22:32 - loss: 0.3564 - accuracy: 0.8674"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1527/3121 [=============>................] - ETA: 22:17 - loss: 0.3566 - accuracy: 0.8673"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1578/3121 [==============>...............] - ETA: 21:34 - loss: 0.3565 - accuracy: 0.8672"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1602/3121 [==============>...............] - ETA: 21:14 - loss: 0.3562 - accuracy: 0.8674"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1608/3121 [==============>...............] - ETA: 21:09 - loss: 0.3563 - accuracy: 0.8674"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1624/3121 [==============>...............] - ETA: 20:55 - loss: 0.3567 - accuracy: 0.8672"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1745/3121 [===============>..............] - ETA: 19:14 - loss: 0.3557 - accuracy: 0.8675"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1769/3121 [================>.............] - ETA: 18:54 - loss: 0.3562 - accuracy: 0.8672"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1851/3121 [================>.............] - ETA: 17:45 - loss: 0.3568 - accuracy: 0.8668"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1862/3121 [================>.............] - ETA: 17:35 - loss: 0.3568 - accuracy: 0.8669"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1878/3121 [=================>............] - ETA: 17:22 - loss: 0.3569 - accuracy: 0.8669"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1882/3121 [=================>............] - ETA: 17:19 - loss: 0.3568 - accuracy: 0.8670"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r1883/3121 [=================>............] - ETA: 17:18 - loss: 0.3568 - accuracy: 0.8670"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1967/3121 [=================>............] - ETA: 16:07 - loss: 0.3561 - accuracy: 0.8675"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2043/3121 [==================>...........] - ETA: 15:04 - loss: 0.3563 - accuracy: 0.8675"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2081/3121 [===================>..........] - ETA: 14:32 - loss: 0.3562 - accuracy: 0.8677"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2088/3121 [===================>..........] - ETA: 14:26 - loss: 0.3557 - accuracy: 0.8678"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2102/3121 [===================>..........] - ETA: 14:14 - loss: 0.3556 - accuracy: 0.8678"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2192/3121 [====================>.........] - ETA: 12:59 - loss: 0.3551 - accuracy: 0.8679"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2198/3121 [====================>.........] - ETA: 12:54 - loss: 0.3552 - accuracy: 0.8679"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2258/3121 [====================>.........] - ETA: 12:03 - loss: 0.3551 - accuracy: 0.8679"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2264/3121 [====================>.........] - ETA: 11:58 - loss: 0.3549 - accuracy: 0.8680"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2290/3121 [=====================>........] - ETA: 11:36 - loss: 0.3552 - accuracy: 0.8680"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2306/3121 [=====================>........] - ETA: 11:23 - loss: 0.3550 - accuracy: 0.8682"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2497/3121 [=======================>......] - ETA: 8:43 - loss: 0.3528 - accuracy: 0.8693"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2505/3121 [=======================>......] - ETA: 8:36 - loss: 0.3527 - accuracy: 0.8694"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2727/3121 [=========================>....] - ETA: 5:30 - loss: 0.3515 - accuracy: 0.8699"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2764/3121 [=========================>....] - ETA: 4:59 - loss: 0.3516 - accuracy: 0.8698"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2777/3121 [=========================>....] - ETA: 4:48 - loss: 0.3514 - accuracy: 0.8698"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2833/3121 [==========================>...] - ETA: 4:01 - loss: 0.3509 - accuracy: 0.8699"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2851/3121 [==========================>...] - ETA: 3:46 - loss: 0.3506 - accuracy: 0.8700"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2857/3121 [==========================>...] - ETA: 3:41 - loss: 0.3508 - accuracy: 0.8699"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2911/3121 [==========================>...] - ETA: 2:56 - loss: 0.3509 - accuracy: 0.8698"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2967/3121 [===========================>..] - ETA: 2:09 - loss: 0.3516 - accuracy: 0.8696"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "3038/3121 [============================>.] - ETA: 1:09 - loss: 0.3514 - accuracy: 0.8697"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "3093/3121 [============================>.] - ETA: 23s - loss: 0.3514 - accuracy: 0.8697"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "3121/3121 [==============================] - ETA: 0s - loss: 0.3511 - accuracy: 0.8698"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r3121/3121 [==============================] - 2707s 867ms/step - loss: 0.3511 - accuracy: 0.8698 - val_loss: 0.3468 - val_accuracy: 0.8728\n"
          ]
        }
      ],
      "source": [
        "history = model.fit(\n",
        "    train_data,\n",
        "    validation_data=valid_data,\n",
        "    epochs=epochs,\n",
        "    use_multiprocessing=True,\n",
        "    workers=-1,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o4yi0mz57Sse"
      },
      "source": [
        "## Evaluate model on the test set"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "utjCO-Rx7Sse",
        "outputId": "12e04ffa-ee93-4dab-f740-2c83a7f15db5"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 70/312 [=====>........................] - ETA: 1:11 - loss: 0.3817 - accuracy: 0.8589"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n",
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 83/312 [======>.......................] - ETA: 1:07 - loss: 0.3764 - accuracy: 0.8607"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "312/312 [==============================] - 93s 297ms/step - loss: 0.3614 - accuracy: 0.8672\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[0.36143583059310913, 0.8671875]"
            ]
          },
          "execution_count": 21,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "test_data = BertSemanticDataGenerator(\n",
        "    test_df[[\"sentence1\", \"sentence2\"]].values.astype(\"str\"),\n",
        "    y_test,\n",
        "    batch_size=batch_size,\n",
        "    shuffle=False,\n",
        ")\n",
        "model.evaluate(test_data, verbose=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gF4YaCaK7Sse"
      },
      "source": [
        "## Inference on custom sentences"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OGdnEYMp7Sse"
      },
      "outputs": [],
      "source": [
        "\n",
        "def check_similarity(sentence1, sentence2):\n",
        "    sentence_pairs = np.array([[str(sentence1), str(sentence2)]])\n",
        "    test_data = BertSemanticDataGenerator(\n",
        "        sentence_pairs, labels=None, batch_size=1, shuffle=False, include_targets=False,\n",
        "    )\n",
        "\n",
        "    proba = model.predict(test_data[0])[0]\n",
        "    idx = np.argmax(proba)\n",
        "    proba = f\"{proba[idx]: .2f}%\"\n",
        "    pred = labels[idx]\n",
        "    return pred, proba\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i1Gkn0me7Ssf"
      },
      "source": [
        "Check results on some example sentence pairs."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NAqFlt0P7Ssf",
        "outputId": "0d62d5d1-b53b-4422-ff8a-428f66232178"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1/1 [==============================] - 0s 66ms/step\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "('contradiction', ' 0.77%')"
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "sentence1 = \"Two women are observing something together.\"\n",
        "sentence2 = \"Two women are standing with their eyes closed.\"\n",
        "check_similarity(sentence1, sentence2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HnrUiBqv7Ssf"
      },
      "source": [
        "Check results on some example sentence pairs."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 188,
          "referenced_widgets": [
            "b8eb3d071b0143e38668fc1dc852835f",
            "188beb1cd8104f9e95afd31d69873a9f",
            "5f92e3e437754c218d8941a5c2f7cd85",
            "0340ff8ac534460cabf515ba033fb7d7",
            "367304d8ee374ebc89f9035a6322c68c",
            "c98d49a1c82745a189a76aa3cc65f2dd",
            "764859f4fc184b67806f1334f078e385",
            "80505e6e8838463896f37bf98096a79f",
            "13c6385879004a4486a00ed61a9e0a3c",
            "5368ba1a0cd647e0a2eb048ae999f7f8",
            "8510ab23994c47cc9af1ac577644b867",
            "b01c8a6653534f43a770df098d82ce4b",
            "a0c65102c1504d2d91e9f650f8c2650a",
            "0905b7162fb14f38a4f26f3ae8d3d08d",
            "3f298db61eb7434abf4f69c27025502b",
            "8d53ec3ba73441bca609298c7b9d7051",
            "fd6de03d60b245549fb65d3b76187fa4",
            "0101b5a6669249e2a330d3158d6a5044",
            "5baf93834a9c4dc7ba0575a8606c0e6d",
            "21fec35c51244488b2376208e287e1c4",
            "797a9271dd974a67a05d3dcbcbe75ae5",
            "2217c37c047840448635ec6bd3851a22"
          ]
        },
        "id": "_wn7LAoK7Ssf",
        "outputId": "5a870cf4-aee0-461d-aef1-8fd57cba7829"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b8eb3d071b0143e38668fc1dc852835f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b01c8a6653534f43a770df098d82ce4b",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.8/dist-packages/transformers/tokenization_utils_base.py:2339: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1/1 [==============================] - 9s 9s/step\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "('neutral', ' 0.94%')"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "sentence1 = \"A smiling costumed woman is holding an umbrella\"\n",
        "sentence2 = \"A happy woman in a fairy costume holds an umbrella\"\n",
        "check_similarity(sentence1, sentence2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gHvPGqfg7Ssf"
      },
      "source": [
        "Check results on some example sentence pairs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8Bs2lfwQ7Ssf",
        "outputId": "2e69b74a-aeff-464d-c140-d41f34363a4a"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1/1 [==============================] - 0s 85ms/step\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "('contradiction', ' 0.89%')"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "sentence1 = \"Cyberpunk is a cool game\"\n",
        "sentence2 = \"i dont like cyberpunk\"\n",
        "check_similarity(sentence1, sentence2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hyMYos2fsb2-"
      },
      "source": [
        "**Save the model**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5krGD8VRbJV1"
      },
      "outputs": [],
      "source": [
        "model.save(\"my_model.h5\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jUjQXF57zxkd"
      },
      "source": [
        "**Save the model to Google drive**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-lKkehBVlmZs"
      },
      "outputs": [],
      "source": [
        "model_name = \"my_model.h5\"\n",
        "path = F\"/content/gdrive/My Drive/ML models/{model_name}\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "A97LpUmBlwcE"
      },
      "outputs": [],
      "source": [
        "model.save(path)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yXyt8MsQshAf"
      },
      "source": [
        "**Load the model**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dShdAcF1q6lm",
        "outputId": "2caef791-8dd4-409f-c3e4-329f42fd9a1e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_ids (InputLayer)         [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " attention_masks (InputLayer)   [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " token_type_ids (InputLayer)    [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " bert (TFBertMainLayer)         TFBaseModelOutputWi  109482240   ['input_ids[0][0]',              \n",
            "                                thPoolingAndCrossAt               'attention_masks[0][0]',        \n",
            "                                tentions(last_hidde               'token_type_ids[0][0]']         \n",
            "                                n_state=(None, 128,                                               \n",
            "                                 768),                                                            \n",
            "                                 pooler_output=(Non                                               \n",
            "                                e, 768),                                                          \n",
            "                                 past_key_values=No                                               \n",
            "                                ne, hidden_states=N                                               \n",
            "                                one, attentions=Non                                               \n",
            "                                e, cross_attentions                                               \n",
            "                                =None)                                                            \n",
            "                                                                                                  \n",
            " bidirectional (Bidirectional)  (None, 128, 128)     426496      ['bert[0][0]']                   \n",
            "                                                                                                  \n",
            " global_average_pooling1d (Glob  (None, 128)         0           ['bidirectional[0][0]']          \n",
            " alAveragePooling1D)                                                                              \n",
            "                                                                                                  \n",
            " global_max_pooling1d (GlobalMa  (None, 128)         0           ['bidirectional[0][0]']          \n",
            " xPooling1D)                                                                                      \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 256)          0           ['global_average_pooling1d[0][0]'\n",
            "                                                                 , 'global_max_pooling1d[0][0]']  \n",
            "                                                                                                  \n",
            " dropout_37 (Dropout)           (None, 256)          0           ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            " dense (Dense)                  (None, 3)            771         ['dropout_37[0][0]']             \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 109,909,507\n",
            "Trainable params: 109,909,507\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "model = tf.keras.models.load_model(path)\n",
        "model.summary()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.0"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0101b5a6669249e2a330d3158d6a5044": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0340ff8ac534460cabf515ba033fb7d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5368ba1a0cd647e0a2eb048ae999f7f8",
            "placeholder": "​",
            "style": "IPY_MODEL_8510ab23994c47cc9af1ac577644b867",
            "value": " 232k/232k [00:00&lt;00:00, 1.81MB/s]"
          }
        },
        "043f9aacaa4945989cbfea46369a883a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0905b7162fb14f38a4f26f3ae8d3d08d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5baf93834a9c4dc7ba0575a8606c0e6d",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_21fec35c51244488b2376208e287e1c4",
            "value": 28
          }
        },
        "0a325ca4f368425fb5808178354336b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_043f9aacaa4945989cbfea46369a883a",
            "max": 536063208,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9e78dffafe3e4715a280dbb645e6413c",
            "value": 536063208
          }
        },
        "0cdf880f1b024b10b9b3328884447e56": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0fa158a1539a4520bac64ee670398398": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "126cdb01363d4cbc93a665661aaf19cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_74e9603271f9408d95a654251a88115f",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_632ca6cab6ed4d7192ae6f0e2b5d7d5b",
            "value": 28
          }
        },
        "13c6385879004a4486a00ed61a9e0a3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "14461ab71058459096f4b7baafc1a680": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9927124e47ef4cc3b04096c9b0a48434",
              "IPY_MODEL_0a325ca4f368425fb5808178354336b0",
              "IPY_MODEL_2aca288b72254dfc830e932cb7471b3f"
            ],
            "layout": "IPY_MODEL_1f183225c7f048e1ad42542ef2772831"
          }
        },
        "188beb1cd8104f9e95afd31d69873a9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c98d49a1c82745a189a76aa3cc65f2dd",
            "placeholder": "​",
            "style": "IPY_MODEL_764859f4fc184b67806f1334f078e385",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "1911a57198024c1ab7a7ab6e8dff8a27": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1f183225c7f048e1ad42542ef2772831": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "21fec35c51244488b2376208e287e1c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2217c37c047840448635ec6bd3851a22": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2aca288b72254dfc830e932cb7471b3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_855aef50288145db85b216d14dc7ef55",
            "placeholder": "​",
            "style": "IPY_MODEL_9950d0328e464bfd9a2c28d5589cb1aa",
            "value": " 536M/536M [00:03&lt;00:00, 190MB/s]"
          }
        },
        "2d3c27d3efa34faabf731439e6cace68": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "367304d8ee374ebc89f9035a6322c68c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3a24cb742f43487b87267ccfa387328d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3f298db61eb7434abf4f69c27025502b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_797a9271dd974a67a05d3dcbcbe75ae5",
            "placeholder": "​",
            "style": "IPY_MODEL_2217c37c047840448635ec6bd3851a22",
            "value": " 28.0/28.0 [00:00&lt;00:00, 325B/s]"
          }
        },
        "416758f4e9cb4fa5adb998abf97b684d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4c7a232fde45459cad319c4ab4c6f341": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cc9edd23e7ca40abbb15ff18b4cb789d",
            "placeholder": "​",
            "style": "IPY_MODEL_7af01ea0f8504feb9113c9befc3520d5",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "50f1439f6d324993ba816e45214dc555": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "50f26b0b383a4503871862beb811b758": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0cdf880f1b024b10b9b3328884447e56",
            "placeholder": "​",
            "style": "IPY_MODEL_935f602ccf8a40f993ebd5700649d429",
            "value": " 570/570 [00:00&lt;00:00, 4.78kB/s]"
          }
        },
        "5368ba1a0cd647e0a2eb048ae999f7f8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5baf93834a9c4dc7ba0575a8606c0e6d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5f92e3e437754c218d8941a5c2f7cd85": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_80505e6e8838463896f37bf98096a79f",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_13c6385879004a4486a00ed61a9e0a3c",
            "value": 231508
          }
        },
        "632ca6cab6ed4d7192ae6f0e2b5d7d5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "74e9603271f9408d95a654251a88115f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "762795bd5de2457c85cb96db0fafdb82": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "764859f4fc184b67806f1334f078e385": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "77b5176534d34979996d3325cec25568": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "77f776b5df7e40c79567390424a2ece6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b4c4fb9ac89f4f21885e3b982dc47db3",
              "IPY_MODEL_ccbf22f58f7b4decab7e0dfdc8c4e015",
              "IPY_MODEL_b505cfbc0ce0493baf778c35fa00f0c0"
            ],
            "layout": "IPY_MODEL_c6b0095aee8f461eb4eedcf94c894892"
          }
        },
        "797a9271dd974a67a05d3dcbcbe75ae5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7af01ea0f8504feb9113c9befc3520d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "80505e6e8838463896f37bf98096a79f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8510ab23994c47cc9af1ac577644b867": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "855aef50288145db85b216d14dc7ef55": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "87b470e6dde749ccb92dac44f0861346": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8d53ec3ba73441bca609298c7b9d7051": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "92d8a9ae98354d619ca04adebbb8b21c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "935f602ccf8a40f993ebd5700649d429": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "936a0c1b88b04289b4c1b06660bc9092": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3a24cb742f43487b87267ccfa387328d",
            "placeholder": "​",
            "style": "IPY_MODEL_eb9273d7841d447492f4e2c3af487e2f",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "97e09f058cee46038b819933d7ff63a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fa00bda5ed6445368673117e0be326d5",
            "placeholder": "​",
            "style": "IPY_MODEL_c94ba20d3cb741c090b0055b2b269735",
            "value": " 28.0/28.0 [00:00&lt;00:00, 955B/s]"
          }
        },
        "9927124e47ef4cc3b04096c9b0a48434": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_50f1439f6d324993ba816e45214dc555",
            "placeholder": "​",
            "style": "IPY_MODEL_0fa158a1539a4520bac64ee670398398",
            "value": "Downloading tf_model.h5: 100%"
          }
        },
        "9950d0328e464bfd9a2c28d5589cb1aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9993f5e65f07431287b102028a1f1e7c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9d565a92a7ad4a71a0e499d7baff5cd4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9e78dffafe3e4715a280dbb645e6413c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a0c65102c1504d2d91e9f650f8c2650a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fd6de03d60b245549fb65d3b76187fa4",
            "placeholder": "​",
            "style": "IPY_MODEL_0101b5a6669249e2a330d3158d6a5044",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "a47f95b4ef7442a2aeec4e1c59daba48": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_936a0c1b88b04289b4c1b06660bc9092",
              "IPY_MODEL_126cdb01363d4cbc93a665661aaf19cc",
              "IPY_MODEL_97e09f058cee46038b819933d7ff63a1"
            ],
            "layout": "IPY_MODEL_92d8a9ae98354d619ca04adebbb8b21c"
          }
        },
        "b01c8a6653534f43a770df098d82ce4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a0c65102c1504d2d91e9f650f8c2650a",
              "IPY_MODEL_0905b7162fb14f38a4f26f3ae8d3d08d",
              "IPY_MODEL_3f298db61eb7434abf4f69c27025502b"
            ],
            "layout": "IPY_MODEL_8d53ec3ba73441bca609298c7b9d7051"
          }
        },
        "b4c4fb9ac89f4f21885e3b982dc47db3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_416758f4e9cb4fa5adb998abf97b684d",
            "placeholder": "​",
            "style": "IPY_MODEL_87b470e6dde749ccb92dac44f0861346",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "b505cfbc0ce0493baf778c35fa00f0c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9d565a92a7ad4a71a0e499d7baff5cd4",
            "placeholder": "​",
            "style": "IPY_MODEL_2d3c27d3efa34faabf731439e6cace68",
            "value": " 232k/232k [00:00&lt;00:00, 1.93MB/s]"
          }
        },
        "b5b06be08cd945b19f36240197be2a53": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_762795bd5de2457c85cb96db0fafdb82",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c7334a20740142c89e49bdf95f701274",
            "value": 570
          }
        },
        "b8eb3d071b0143e38668fc1dc852835f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_188beb1cd8104f9e95afd31d69873a9f",
              "IPY_MODEL_5f92e3e437754c218d8941a5c2f7cd85",
              "IPY_MODEL_0340ff8ac534460cabf515ba033fb7d7"
            ],
            "layout": "IPY_MODEL_367304d8ee374ebc89f9035a6322c68c"
          }
        },
        "c6b0095aee8f461eb4eedcf94c894892": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c7334a20740142c89e49bdf95f701274": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c94ba20d3cb741c090b0055b2b269735": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c98d49a1c82745a189a76aa3cc65f2dd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cc9edd23e7ca40abbb15ff18b4cb789d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ccbf22f58f7b4decab7e0dfdc8c4e015": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_77b5176534d34979996d3325cec25568",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1911a57198024c1ab7a7ab6e8dff8a27",
            "value": 231508
          }
        },
        "d3fff89b328d4c6e94232f9ab806a3b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4c7a232fde45459cad319c4ab4c6f341",
              "IPY_MODEL_b5b06be08cd945b19f36240197be2a53",
              "IPY_MODEL_50f26b0b383a4503871862beb811b758"
            ],
            "layout": "IPY_MODEL_9993f5e65f07431287b102028a1f1e7c"
          }
        },
        "eb9273d7841d447492f4e2c3af487e2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fa00bda5ed6445368673117e0be326d5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fd6de03d60b245549fb65d3b76187fa4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}