Check_Pretrained_Model_on_SQUAD2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Check_Pretrained_Model_on SQUAD2.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyOOWDp4beUl2SfWCLdQdGp/",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bhadreshpsavani/UnderstandingNLP/blob/master/Check_Pretrained_Model_on_SQUAD2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I3F-u_bdHV6K",
        "outputId": "9e8c1a57-cc46-454f-cf54-9f2da77de407"
      },
      "source": [
        "!git clone https://github.com/huggingface/transformers\r\n",
        "%cd transformers\r\n",
        "!pip install .\r\n",
        "!pip install -r ./examples/question-answering/requirements.txt\r\n",
        "%cd .."
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'transformers'...\n",
            "remote: Enumerating objects: 2, done.\u001b[K\n",
            "remote: Counting objects: 100% (2/2), done.\u001b[K\n",
            "remote: Compressing objects: 100% (2/2), done.\u001b[K\n",
            "remote: Total 66869 (delta 0), reused 1 (delta 0), pack-reused 66867\u001b[K\n",
            "Receiving objects: 100% (66869/66869), 49.79 MiB | 24.30 MiB/s, done.\n",
            "Resolving deltas: 100% (47507/47507), done.\n",
            "/content/transformers\n",
            "Processing /content/transformers\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==4.4.0.dev0) (4.41.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==4.4.0.dev0) (3.0.12)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.4.0.dev0) (1.19.5)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 4.4MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/23/2ddc317b2121117bf34dd00f5b0de194158f2a44ee2bf5e47c7166878a97/tokenizers-0.10.1-cp37-cp37m-manylinux2010_x86_64.whl (3.2MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 8.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.4.0.dev0) (2019.12.20)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers==4.4.0.dev0) (3.7.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==4.4.0.dev0) (2.23.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers==4.4.0.dev0) (20.9)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.4.0.dev0) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.4.0.dev0) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.4.0.dev0) (1.0.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers==4.4.0.dev0) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers==4.4.0.dev0) (3.4.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.4.0.dev0) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.4.0.dev0) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.4.0.dev0) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.4.0.dev0) (2020.12.5)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers==4.4.0.dev0) (2.4.7)\n",
            "Building wheels for collected packages: transformers\n",
            "  Building wheel for transformers (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformers: filename=transformers-4.4.0.dev0-cp37-none-any.whl size=1969146 sha256=a61fe9da1e654ac8af5d04dd3c9421b7ca628c5b5e6e6f7219655ce027cf84f7\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-a7a3xbfk/wheels/23/19/dd/2561a4e47240cf6b307729d58e56f8077dd0c698f5992216cf\n",
            "Successfully built transformers\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp37-none-any.whl size=893262 sha256=7c12b6cb36ff57e15afb20dbc1f4c6fff09580a18093e75735c785f1078edd35\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.10.1 transformers-4.4.0.dev0\n",
            "Collecting datasets>=1.2.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/3e/73/742d17d8a9a1c639132affccc9250f0743e484cbf263ede6ddcbe34ef212/datasets-1.4.1-py3-none-any.whl (186kB)\n",
            "\u001b[K     |████████████████████████████████| 194kB 5.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (2.23.0)\n",
            "Requirement already satisfied: tqdm<4.50.0,>=4.27 in /usr/local/lib/python3.7/dist-packages (from datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (4.41.1)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (0.3.3)\n",
            "Collecting xxhash\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e7/27/1c0b37c53a7852f1c190ba5039404d27b3ae96a55f48203a74259f8213c9/xxhash-2.0.0-cp37-cp37m-manylinux2010_x86_64.whl (243kB)\n",
            "\u001b[K     |████████████████████████████████| 245kB 9.9MB/s \n",
            "\u001b[?25hCollecting huggingface-hub==0.0.2\n",
            "  Downloading https://files.pythonhosted.org/packages/b5/93/7cb0755c62c36cdadc70c79a95681df685b52cbaf76c724facb6ecac3272/huggingface_hub-0.0.2-py3-none-any.whl\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (1.19.5)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (1.1.5)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (0.70.11.1)\n",
            "Requirement already satisfied: pyarrow>=0.17.1 in /usr/local/lib/python3.7/dist-packages (from datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (3.0.0)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (3.7.0)\n",
            "Collecting fsspec\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/91/0d/a6bfee0ddf47b254286b9bd574e6f50978c69897647ae15b14230711806e/fsspec-0.8.7-py3-none-any.whl (103kB)\n",
            "\u001b[K     |████████████████████████████████| 112kB 14.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (1.24.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub==0.0.2->datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (3.0.12)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (2.8.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (3.4.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets>=1.2.1->-r ./examples/question-answering/requirements.txt (line 1)) (1.15.0)\n",
            "Installing collected packages: xxhash, huggingface-hub, fsspec, datasets\n",
            "Successfully installed datasets-1.4.1 fsspec-0.8.7 huggingface-hub-0.0.2 xxhash-2.0.0\n",
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FX2E4B74JFcI",
        "outputId": "07b24058-4a59-45a6-bc2e-3a7e0d4c5350"
      },
      "source": [
        "!pip install -q sentencepiece"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 1.2MB 5.2MB/s \n",
            "\u001b[?25h"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o6XNWBPpIVHc",
        "outputId": "a15045f0-969f-44ce-8e20-c1a2406e331f"
      },
      "source": [
        "!python ./transformers/examples/question-answering/run_qa.py \\\r\n",
        "--model_name_or_path ktrapeznikov/albert-xlarge-v2-squad-v2 \\\r\n",
        "--dataset_name squad_v2 \\\r\n",
        "--do_eval \\\r\n",
        "--per_device_train_batch_size 12 \\\r\n",
        "--learning_rate 3e-5 \\\r\n",
        "--num_train_epochs 2 \\\r\n",
        "--max_seq_length 384 \\\r\n",
        "--doc_stride 128 \\\r\n",
        "--output_dir /tmp/debug_squad/"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-03-15 08:29:19.275713: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "03/15/2021 08:29:21 - WARNING - __main__ -   Process rank: -1, device: cuda:0, n_gpu: 1distributed training: False, 16-bits training: False\n",
            "03/15/2021 08:29:21 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(output_dir=/tmp/debug_squad/, overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, evaluation_strategy=IntervalStrategy.NO, prediction_loss_only=False, per_device_train_batch_size=12, per_device_eval_batch_size=8, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=3e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=2.0, max_steps=-1, lr_scheduler_type=SchedulerType.LINEAR, warmup_ratio=0.0, warmup_steps=0, logging_dir=runs/Mar15_08-29-21_cbb7255b36a9, logging_strategy=IntervalStrategy.STEPS, logging_first_step=False, logging_steps=500, save_strategy=IntervalStrategy.STEPS, save_steps=500, save_total_limit=None, no_cuda=False, seed=42, fp16=False, fp16_opt_level=O1, fp16_backend=auto, fp16_full_eval=False, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=500, dataloader_num_workers=0, past_index=-1, run_name=/tmp/debug_squad/, disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, report_to=['tensorboard'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, skip_memory_metrics=False, _n_gpu=1)\n",
            "Downloading: 5.07kB [00:00, 4.85MB/s]       \n",
            "Downloading: 2.23kB [00:00, 1.85MB/s]     \n",
            "Downloading and preparing dataset squad_v2/squad_v2 (download: 44.34 MiB, generated: 122.57 MiB, post-processed: Unknown size, total: 166.91 MiB) to /root/.cache/huggingface/datasets/squad_v2/squad_v2/2.0.0/9cac55034b086140f0649ecb5c604d09d7da2f2f5b73a90caa2e2bcc1f5cac09...\n",
            "Downloading: 42.1MB [00:00, 66.8MB/s]\n",
            "Downloading: 4.37MB [00:00, 31.8MB/s]\n",
            "Dataset squad_v2 downloaded and prepared to /root/.cache/huggingface/datasets/squad_v2/squad_v2/2.0.0/9cac55034b086140f0649ecb5c604d09d7da2f2f5b73a90caa2e2bcc1f5cac09. Subsequent calls will reuse this data.\n",
            "[INFO|file_utils.py:1382] 2021-03-15 08:29:34,483 >> https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpemj25fvo\n",
            "Downloading: 100% 717/717 [00:00<00:00, 633kB/s]\n",
            "[INFO|file_utils.py:1386] 2021-03-15 08:29:34,744 >> storing https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/config.json in cache at /root/.cache/huggingface/transformers/63d792c1dbb44e8689d1e7a2c0ae7fc7eaa5d787021a85ebfba99d7e7fd4a0c1.e2cacd8449f3336b177473e9974534c5d5c11e7a6c3602c7f8b160b52b703710\n",
            "[INFO|file_utils.py:1389] 2021-03-15 08:29:34,744 >> creating metadata file for /root/.cache/huggingface/transformers/63d792c1dbb44e8689d1e7a2c0ae7fc7eaa5d787021a85ebfba99d7e7fd4a0c1.e2cacd8449f3336b177473e9974534c5d5c11e7a6c3602c7f8b160b52b703710\n",
            "[INFO|configuration_utils.py:463] 2021-03-15 08:29:34,745 >> loading configuration file https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/63d792c1dbb44e8689d1e7a2c0ae7fc7eaa5d787021a85ebfba99d7e7fd4a0c1.e2cacd8449f3336b177473e9974534c5d5c11e7a6c3602c7f8b160b52b703710\n",
            "[INFO|configuration_utils.py:499] 2021-03-15 08:29:34,746 >> Model config AlbertConfig {\n",
            "  \"architectures\": [\n",
            "    \"AlbertForQuestionAnswering\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 2,\n",
            "  \"classifier_dropout_prob\": 0.1,\n",
            "  \"down_scale_factor\": 1,\n",
            "  \"embedding_size\": 128,\n",
            "  \"eos_token_id\": 3,\n",
            "  \"gap_size\": 0,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 2048,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"inner_group_num\": 1,\n",
            "  \"intermediate_size\": 8192,\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"albert\",\n",
            "  \"net_structure_type\": 0,\n",
            "  \"num_attention_heads\": 16,\n",
            "  \"num_hidden_groups\": 1,\n",
            "  \"num_hidden_layers\": 24,\n",
            "  \"num_memory_blocks\": 0,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.4.0.dev0\",\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"vocab_size\": 30000\n",
            "}\n",
            "\n",
            "[INFO|configuration_utils.py:463] 2021-03-15 08:29:35,006 >> loading configuration file https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/63d792c1dbb44e8689d1e7a2c0ae7fc7eaa5d787021a85ebfba99d7e7fd4a0c1.e2cacd8449f3336b177473e9974534c5d5c11e7a6c3602c7f8b160b52b703710\n",
            "[INFO|configuration_utils.py:499] 2021-03-15 08:29:35,006 >> Model config AlbertConfig {\n",
            "  \"architectures\": [\n",
            "    \"AlbertForQuestionAnswering\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 2,\n",
            "  \"classifier_dropout_prob\": 0.1,\n",
            "  \"down_scale_factor\": 1,\n",
            "  \"embedding_size\": 128,\n",
            "  \"eos_token_id\": 3,\n",
            "  \"gap_size\": 0,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 2048,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"inner_group_num\": 1,\n",
            "  \"intermediate_size\": 8192,\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"albert\",\n",
            "  \"net_structure_type\": 0,\n",
            "  \"num_attention_heads\": 16,\n",
            "  \"num_hidden_groups\": 1,\n",
            "  \"num_hidden_layers\": 24,\n",
            "  \"num_memory_blocks\": 0,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.4.0.dev0\",\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"vocab_size\": 30000\n",
            "}\n",
            "\n",
            "[INFO|file_utils.py:1382] 2021-03-15 08:29:35,273 >> https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/spiece.model not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmprkp87qt4\n",
            "Downloading: 100% 760k/760k [00:00<00:00, 1.84MB/s]\n",
            "[INFO|file_utils.py:1386] 2021-03-15 08:29:35,955 >> storing https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/spiece.model in cache at /root/.cache/huggingface/transformers/6276f297211781cf72c7b6c6c5b40ce82b3bd0f41078b46284be1243a02d8e0c.d6110e25022b713452eb83d5bfa8ae64530995a93d8e694fe52e05aa85dd3a7d\n",
            "[INFO|file_utils.py:1389] 2021-03-15 08:29:35,955 >> creating metadata file for /root/.cache/huggingface/transformers/6276f297211781cf72c7b6c6c5b40ce82b3bd0f41078b46284be1243a02d8e0c.d6110e25022b713452eb83d5bfa8ae64530995a93d8e694fe52e05aa85dd3a7d\n",
            "[INFO|file_utils.py:1382] 2021-03-15 08:29:36,474 >> https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/added_tokens.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp5fa17rc7\n",
            "Downloading: 100% 2.00/2.00 [00:00<00:00, 1.58kB/s]\n",
            "[INFO|file_utils.py:1386] 2021-03-15 08:29:36,734 >> storing https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/added_tokens.json in cache at /root/.cache/huggingface/transformers/ba6c93b2c4622b97c1822a73e4ca016a60e7de0950186c61a7343b414f435f0b.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b\n",
            "[INFO|file_utils.py:1389] 2021-03-15 08:29:36,735 >> creating metadata file for /root/.cache/huggingface/transformers/ba6c93b2c4622b97c1822a73e4ca016a60e7de0950186c61a7343b414f435f0b.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b\n",
            "[INFO|file_utils.py:1382] 2021-03-15 08:29:36,997 >> https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/special_tokens_map.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpdm49zw7p\n",
            "Downloading: 100% 156/156 [00:00<00:00, 137kB/s]\n",
            "[INFO|file_utils.py:1386] 2021-03-15 08:29:37,255 >> storing https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/special_tokens_map.json in cache at /root/.cache/huggingface/transformers/0865fb818019b2e454970b3c0ebd6ad9c5ef1eac68c6188d7703bd1f13f02ae7.623993453f3f6b9f6ad831899812f482e5cde100e664124feb3a6446d69a26bf\n",
            "[INFO|file_utils.py:1389] 2021-03-15 08:29:37,256 >> creating metadata file for /root/.cache/huggingface/transformers/0865fb818019b2e454970b3c0ebd6ad9c5ef1eac68c6188d7703bd1f13f02ae7.623993453f3f6b9f6ad831899812f482e5cde100e664124feb3a6446d69a26bf\n",
            "[INFO|file_utils.py:1382] 2021-03-15 08:29:37,515 >> https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/tokenizer_config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpyfk938hq\n",
            "Downloading: 100% 58.0/58.0 [00:00<00:00, 56.9kB/s]\n",
            "[INFO|file_utils.py:1386] 2021-03-15 08:29:37,778 >> storing https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/tokenizer_config.json in cache at /root/.cache/huggingface/transformers/feef3909ed005e95d21b0c3824093015275138c70d80e78c2d75d03f8fbc0ebd.11d9edb6b1301b5af13d33c1585ff45ff84dd55cc6915c2872f856d1ee2dc409\n",
            "[INFO|file_utils.py:1389] 2021-03-15 08:29:37,778 >> creating metadata file for /root/.cache/huggingface/transformers/feef3909ed005e95d21b0c3824093015275138c70d80e78c2d75d03f8fbc0ebd.11d9edb6b1301b5af13d33c1585ff45ff84dd55cc6915c2872f856d1ee2dc409\n",
            "[INFO|tokenization_utils_base.py:1702] 2021-03-15 08:29:37,779 >> loading file https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/spiece.model from cache at /root/.cache/huggingface/transformers/6276f297211781cf72c7b6c6c5b40ce82b3bd0f41078b46284be1243a02d8e0c.d6110e25022b713452eb83d5bfa8ae64530995a93d8e694fe52e05aa85dd3a7d\n",
            "[INFO|tokenization_utils_base.py:1702] 2021-03-15 08:29:37,779 >> loading file https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/tokenizer.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1702] 2021-03-15 08:29:37,779 >> loading file https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/added_tokens.json from cache at /root/.cache/huggingface/transformers/ba6c93b2c4622b97c1822a73e4ca016a60e7de0950186c61a7343b414f435f0b.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b\n",
            "[INFO|tokenization_utils_base.py:1702] 2021-03-15 08:29:37,779 >> loading file https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/special_tokens_map.json from cache at /root/.cache/huggingface/transformers/0865fb818019b2e454970b3c0ebd6ad9c5ef1eac68c6188d7703bd1f13f02ae7.623993453f3f6b9f6ad831899812f482e5cde100e664124feb3a6446d69a26bf\n",
            "[INFO|tokenization_utils_base.py:1702] 2021-03-15 08:29:37,779 >> loading file https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/tokenizer_config.json from cache at /root/.cache/huggingface/transformers/feef3909ed005e95d21b0c3824093015275138c70d80e78c2d75d03f8fbc0ebd.11d9edb6b1301b5af13d33c1585ff45ff84dd55cc6915c2872f856d1ee2dc409\n",
            "[INFO|file_utils.py:1382] 2021-03-15 08:29:38,187 >> https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/pytorch_model.bin not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmps2rwt5_x\n",
            "Downloading: 100% 235M/235M [00:06<00:00, 34.2MB/s]\n",
            "[INFO|file_utils.py:1386] 2021-03-15 08:29:45,630 >> storing https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/pytorch_model.bin in cache at /root/.cache/huggingface/transformers/8d16ceb92c5e32ffc26f0d5a6e0681bc1808e5b3f9491d07a97a05e8ac320365.c61293af3110d8892035c62624a45f32947d2dbea7e43dd8c2683ce06b9b6648\n",
            "[INFO|file_utils.py:1389] 2021-03-15 08:29:45,630 >> creating metadata file for /root/.cache/huggingface/transformers/8d16ceb92c5e32ffc26f0d5a6e0681bc1808e5b3f9491d07a97a05e8ac320365.c61293af3110d8892035c62624a45f32947d2dbea7e43dd8c2683ce06b9b6648\n",
            "[INFO|modeling_utils.py:1051] 2021-03-15 08:29:45,631 >> loading weights file https://huggingface.co/ktrapeznikov/albert-xlarge-v2-squad-v2/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/8d16ceb92c5e32ffc26f0d5a6e0681bc1808e5b3f9491d07a97a05e8ac320365.c61293af3110d8892035c62624a45f32947d2dbea7e43dd8c2683ce06b9b6648\n",
            "[INFO|modeling_utils.py:1167] 2021-03-15 08:29:47,669 >> All model checkpoint weights were used when initializing AlbertForQuestionAnswering.\n",
            "\n",
            "[INFO|modeling_utils.py:1176] 2021-03-15 08:29:47,670 >> All the weights of AlbertForQuestionAnswering were initialized from the model checkpoint at ktrapeznikov/albert-xlarge-v2-squad-v2.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use AlbertForQuestionAnswering for predictions without further training.\n",
            "100% 12/12 [00:26<00:00,  2.18s/ba]\n",
            "Downloading: 4.51kB [00:00, 3.93MB/s]       \n",
            "Downloading: 3.35kB [00:00, 3.31MB/s]       \n",
            "[INFO|trainer.py:482] 2021-03-15 08:30:24,046 >> The following columns in the evaluation set  don't have a corresponding argument in `AlbertForQuestionAnswering.forward` and have been ignored: example_id, offset_mapping.\n",
            "03/15/2021 08:30:24 - INFO - __main__ -   *** Evaluate ***\n",
            "[INFO|trainer.py:482] 2021-03-15 08:30:24,156 >> The following columns in the evaluation set  don't have a corresponding argument in `AlbertForQuestionAnswering.forward` and have been ignored: example_id, offset_mapping.\n",
            "[INFO|trainer.py:1772] 2021-03-15 08:30:24,157 >> ***** Running Evaluation *****\n",
            "[INFO|trainer.py:1773] 2021-03-15 08:30:24,157 >>   Num examples = 12171\n",
            "[INFO|trainer.py:1774] 2021-03-15 08:30:24,157 >>   Batch size = 8\n",
            "100% 1522/1522 [1:41:26<00:00,  4.00s/it]03/15/2021 10:12:04 - INFO - utils_qa -   Post-processing 11873 example predictions split into 12171 features.\n",
            "\n",
            "  0% 0/11873 [00:00<?, ?it/s]\u001b[A\n",
            "  0% 36/11873 [00:00<00:32, 358.90it/s]\u001b[A\n",
            "  1% 68/11873 [00:00<00:34, 346.04it/s]\u001b[A\n",
            "  1% 104/11873 [00:00<00:33, 348.20it/s]\u001b[A\n",
            "  1% 141/11873 [00:00<00:33, 352.33it/s]\u001b[A\n",
            "  2% 183/11873 [00:00<00:31, 368.37it/s]\u001b[A\n",
            "  2% 224/11873 [00:00<00:30, 378.62it/s]\u001b[A\n",
            "  2% 262/11873 [00:00<00:30, 378.03it/s]\u001b[A\n",
            "  3% 302/11873 [00:00<00:30, 383.72it/s]\u001b[A\n",
            "  3% 341/11873 [00:00<00:30, 382.69it/s]\u001b[A\n",
            "  3% 382/11873 [00:01<00:29, 389.48it/s]\u001b[A\n",
            "  4% 421/11873 [00:01<00:29, 389.38it/s]\u001b[A\n",
            "  4% 460/11873 [00:01<00:29, 381.01it/s]\u001b[A\n",
            "  4% 498/11873 [00:01<00:30, 373.43it/s]\u001b[A\n",
            "  5% 536/11873 [00:01<00:30, 370.01it/s]\u001b[A\n",
            "  5% 573/11873 [00:01<00:30, 368.34it/s]\u001b[A\n",
            "  5% 615/11873 [00:01<00:29, 380.67it/s]\u001b[A\n",
            "  6% 654/11873 [00:01<00:29, 382.57it/s]\u001b[A\n",
            "  6% 694/11873 [00:01<00:29, 384.65it/s]\u001b[A\n",
            "  6% 733/11873 [00:01<00:29, 383.26it/s]\u001b[A\n",
            "  7% 772/11873 [00:02<00:29, 382.05it/s]\u001b[A\n",
            "  7% 811/11873 [00:02<00:28, 383.83it/s]\u001b[A\n",
            "  7% 851/11873 [00:02<00:28, 386.92it/s]\u001b[A\n",
            "  8% 895/11873 [00:02<00:27, 400.13it/s]\u001b[A\n",
            "  8% 936/11873 [00:02<00:27, 395.81it/s]\u001b[A\n",
            "  8% 978/11873 [00:02<00:27, 401.93it/s]\u001b[A\n",
            "  9% 1019/11873 [00:02<00:28, 387.16it/s]\u001b[A\n",
            "  9% 1058/11873 [00:02<00:27, 386.39it/s]\u001b[A\n",
            "  9% 1097/11873 [00:02<00:28, 384.11it/s]\u001b[A\n",
            " 10% 1136/11873 [00:02<00:28, 376.87it/s]\u001b[A\n",
            " 10% 1174/11873 [00:03<00:28, 373.43it/s]\u001b[A\n",
            " 10% 1215/11873 [00:03<00:27, 381.40it/s]\u001b[A\n",
            " 11% 1254/11873 [00:03<00:28, 366.62it/s]\u001b[A\n",
            " 11% 1291/11873 [00:03<00:29, 364.31it/s]\u001b[A\n",
            " 11% 1328/11873 [00:03<00:29, 359.17it/s]\u001b[A\n",
            " 12% 1366/11873 [00:03<00:28, 363.61it/s]\u001b[A\n",
            " 12% 1406/11873 [00:03<00:28, 372.48it/s]\u001b[A\n",
            " 12% 1444/11873 [00:03<00:28, 371.81it/s]\u001b[A\n",
            " 12% 1482/11873 [00:03<00:28, 365.64it/s]\u001b[A\n",
            " 13% 1522/11873 [00:04<00:27, 375.13it/s]\u001b[A\n",
            " 13% 1560/11873 [00:04<00:28, 360.73it/s]\u001b[A\n",
            " 13% 1597/11873 [00:04<00:28, 359.20it/s]\u001b[A\n",
            " 14% 1634/11873 [00:04<00:28, 355.39it/s]\u001b[A\n",
            " 14% 1670/11873 [00:04<00:29, 349.89it/s]\u001b[A\n",
            " 14% 1706/11873 [00:04<00:29, 350.23it/s]\u001b[A\n",
            " 15% 1745/11873 [00:04<00:28, 360.30it/s]\u001b[A\n",
            " 15% 1782/11873 [00:04<00:28, 352.82it/s]\u001b[A\n",
            " 15% 1820/11873 [00:04<00:28, 357.67it/s]\u001b[A\n",
            " 16% 1860/11873 [00:04<00:27, 367.94it/s]\u001b[A\n",
            " 16% 1897/11873 [00:05<00:27, 359.38it/s]\u001b[A\n",
            " 16% 1936/11873 [00:05<00:27, 367.70it/s]\u001b[A\n",
            " 17% 1974/11873 [00:05<00:26, 369.32it/s]\u001b[A\n",
            " 17% 2012/11873 [00:05<00:26, 370.63it/s]\u001b[A\n",
            " 17% 2050/11873 [00:05<00:26, 364.68it/s]\u001b[A\n",
            " 18% 2088/11873 [00:05<00:26, 368.39it/s]\u001b[A\n",
            " 18% 2125/11873 [00:05<00:27, 360.30it/s]\u001b[A\n",
            " 18% 2163/11873 [00:05<00:26, 365.09it/s]\u001b[A\n",
            " 19% 2201/11873 [00:05<00:26, 369.07it/s]\u001b[A\n",
            " 19% 2238/11873 [00:06<00:26, 362.05it/s]\u001b[A\n",
            " 19% 2275/11873 [00:06<00:26, 360.27it/s]\u001b[A\n",
            " 19% 2312/11873 [00:06<00:26, 358.25it/s]\u001b[A\n",
            " 20% 2348/11873 [00:06<00:26, 357.13it/s]\u001b[A\n",
            " 20% 2386/11873 [00:06<00:26, 362.47it/s]\u001b[A\n",
            " 20% 2423/11873 [00:06<00:26, 353.39it/s]\u001b[A\n",
            " 21% 2459/11873 [00:06<00:27, 348.17it/s]\u001b[A\n",
            " 21% 2495/11873 [00:06<00:26, 349.22it/s]\u001b[A\n",
            " 21% 2533/11873 [00:06<00:26, 355.62it/s]\u001b[A\n",
            " 22% 2570/11873 [00:06<00:25, 358.23it/s]\u001b[A\n",
            " 22% 2606/11873 [00:07<00:26, 355.89it/s]\u001b[A\n",
            " 22% 2644/11873 [00:07<00:25, 360.50it/s]\u001b[A\n",
            " 23% 2681/11873 [00:07<00:25, 354.97it/s]\u001b[A\n",
            " 23% 2717/11873 [00:07<00:26, 347.84it/s]\u001b[A\n",
            " 23% 2752/11873 [00:07<00:27, 328.17it/s]\u001b[A\n",
            " 23% 2786/11873 [00:07<00:27, 328.85it/s]\u001b[A\n",
            " 24% 2824/11873 [00:07<00:26, 342.17it/s]\u001b[A\n",
            " 24% 2864/11873 [00:07<00:25, 355.59it/s]\u001b[A\n",
            " 24% 2900/11873 [00:07<00:25, 350.69it/s]\u001b[A\n",
            " 25% 2936/11873 [00:08<00:25, 353.37it/s]\u001b[A\n",
            " 25% 2976/11873 [00:08<00:24, 364.51it/s]\u001b[A\n",
            " 25% 3013/11873 [00:08<00:26, 340.24it/s]\u001b[A\n",
            " 26% 3048/11873 [00:08<00:27, 322.77it/s]\u001b[A\n",
            " 26% 3081/11873 [00:08<00:28, 311.97it/s]\u001b[A\n",
            " 26% 3113/11873 [00:08<00:32, 270.26it/s]\u001b[A\n",
            " 26% 3142/11873 [00:08<00:37, 231.02it/s]\u001b[A\n",
            " 27% 3168/11873 [00:08<00:38, 228.50it/s]\u001b[A\n",
            " 27% 3205/11873 [00:09<00:33, 256.59it/s]\u001b[A\n",
            " 27% 3240/11873 [00:09<00:31, 278.40it/s]\u001b[A\n",
            " 28% 3270/11873 [00:09<00:31, 269.51it/s]\u001b[A\n",
            " 28% 3299/11873 [00:09<00:44, 193.49it/s]\u001b[A\n",
            " 28% 3323/11873 [00:09<00:51, 165.97it/s]\u001b[A\n",
            " 28% 3344/11873 [00:09<00:48, 176.31it/s]\u001b[A\n",
            " 28% 3365/11873 [00:09<00:52, 161.32it/s]\u001b[A\n",
            " 29% 3400/11873 [00:10<00:44, 192.15it/s]\u001b[A\n",
            " 29% 3435/11873 [00:10<00:38, 221.78it/s]\u001b[A\n",
            " 29% 3470/11873 [00:10<00:33, 248.28it/s]\u001b[A\n",
            " 30% 3504/11873 [00:10<00:31, 269.27it/s]\u001b[A\n",
            " 30% 3542/11873 [00:10<00:28, 294.08it/s]\u001b[A\n",
            " 30% 3579/11873 [00:10<00:26, 312.93it/s]\u001b[A\n",
            " 30% 3616/11873 [00:10<00:25, 327.07it/s]\u001b[A\n",
            " 31% 3653/11873 [00:10<00:24, 338.31it/s]\u001b[A\n",
            " 31% 3689/11873 [00:10<00:25, 321.97it/s]\u001b[A\n",
            " 31% 3723/11873 [00:10<00:25, 325.51it/s]\u001b[A\n",
            " 32% 3763/11873 [00:11<00:23, 342.73it/s]\u001b[A\n",
            " 32% 3799/11873 [00:11<00:23, 337.42it/s]\u001b[A\n",
            " 32% 3834/11873 [00:11<00:26, 303.99it/s]\u001b[A\n",
            " 33% 3871/11873 [00:11<00:25, 319.32it/s]\u001b[A\n",
            " 33% 3909/11873 [00:11<00:23, 334.36it/s]\u001b[A\n",
            " 33% 3944/11873 [00:11<00:26, 301.90it/s]\u001b[A\n",
            " 33% 3976/11873 [00:11<00:25, 304.71it/s]\u001b[A\n",
            " 34% 4013/11873 [00:11<00:24, 321.72it/s]\u001b[A\n",
            " 34% 4053/11873 [00:11<00:22, 341.26it/s]\u001b[A\n",
            " 34% 4092/11873 [00:12<00:22, 352.75it/s]\u001b[A\n",
            " 35% 4129/11873 [00:12<00:21, 352.67it/s]\u001b[A\n",
            " 35% 4165/11873 [00:12<00:24, 312.86it/s]\u001b[A\n",
            " 35% 4198/11873 [00:12<00:26, 291.99it/s]\u001b[A\n",
            " 36% 4236/11873 [00:12<00:24, 312.57it/s]\u001b[A\n",
            " 36% 4269/11873 [00:12<00:24, 310.78it/s]\u001b[A\n",
            " 36% 4307/11873 [00:12<00:23, 327.17it/s]\u001b[A\n",
            " 37% 4346/11873 [00:12<00:21, 342.82it/s]\u001b[A\n",
            " 37% 4382/11873 [00:12<00:21, 345.48it/s]\u001b[A\n",
            " 37% 4418/11873 [00:13<00:25, 295.32it/s]\u001b[A\n",
            " 37% 4450/11873 [00:13<00:26, 279.49it/s]\u001b[A\n",
            " 38% 4489/11873 [00:13<00:24, 303.97it/s]\u001b[A\n",
            " 38% 4523/11873 [00:13<00:23, 311.48it/s]\u001b[A\n",
            " 38% 4560/11873 [00:13<00:22, 326.56it/s]\u001b[A\n",
            " 39% 4596/11873 [00:13<00:21, 333.43it/s]\u001b[A\n",
            " 39% 4631/11873 [00:13<00:21, 336.20it/s]\u001b[A\n",
            " 39% 4668/11873 [00:13<00:20, 343.13it/s]\u001b[A\n",
            " 40% 4704/11873 [00:13<00:20, 345.54it/s]\u001b[A\n",
            " 40% 4739/11873 [00:14<00:21, 339.66it/s]\u001b[A\n",
            " 40% 4777/11873 [00:14<00:20, 348.07it/s]\u001b[A\n",
            " 41% 4813/11873 [00:14<00:21, 332.07it/s]\u001b[A\n",
            " 41% 4847/11873 [00:14<00:21, 329.42it/s]\u001b[A\n",
            " 41% 4882/11873 [00:14<00:20, 333.15it/s]\u001b[A\n",
            " 41% 4917/11873 [00:14<00:20, 334.14it/s]\u001b[A\n",
            " 42% 4954/11873 [00:14<00:20, 341.57it/s]\u001b[A\n",
            " 42% 4989/11873 [00:14<00:20, 343.34it/s]\u001b[A\n",
            " 42% 5024/11873 [00:14<00:20, 340.77it/s]\u001b[A\n",
            " 43% 5059/11873 [00:15<00:20, 338.18it/s]\u001b[A\n",
            " 43% 5097/11873 [00:15<00:19, 347.77it/s]\u001b[A\n",
            " 43% 5132/11873 [00:15<00:19, 348.00it/s]\u001b[A\n",
            " 44% 5167/11873 [00:15<00:19, 341.48it/s]\u001b[A\n",
            " 44% 5205/11873 [00:15<00:19, 349.76it/s]\u001b[A\n",
            " 44% 5242/11873 [00:15<00:18, 353.80it/s]\u001b[A\n",
            " 44% 5278/11873 [00:15<00:20, 314.75it/s]\u001b[A\n",
            " 45% 5316/11873 [00:15<00:19, 329.83it/s]\u001b[A\n",
            " 45% 5355/11873 [00:15<00:18, 344.41it/s]\u001b[A\n",
            " 45% 5391/11873 [00:15<00:18, 347.16it/s]\u001b[A\n",
            " 46% 5429/11873 [00:16<00:18, 355.52it/s]\u001b[A\n",
            " 46% 5465/11873 [00:16<00:18, 354.30it/s]\u001b[A\n",
            " 46% 5501/11873 [00:16<00:17, 354.73it/s]\u001b[A\n",
            " 47% 5537/11873 [00:16<00:17, 353.63it/s]\u001b[A\n",
            " 47% 5573/11873 [00:16<00:18, 349.61it/s]\u001b[A\n",
            " 47% 5609/11873 [00:16<00:18, 340.57it/s]\u001b[A\n",
            " 48% 5644/11873 [00:16<00:18, 332.86it/s]\u001b[A\n",
            " 48% 5679/11873 [00:16<00:18, 333.80it/s]\u001b[A\n",
            " 48% 5713/11873 [00:16<00:18, 331.52it/s]\u001b[A\n",
            " 48% 5750/11873 [00:17<00:18, 339.89it/s]\u001b[A\n",
            " 49% 5785/11873 [00:17<00:17, 340.15it/s]\u001b[A\n",
            " 49% 5825/11873 [00:17<00:17, 354.11it/s]\u001b[A\n",
            " 49% 5862/11873 [00:17<00:16, 356.22it/s]\u001b[A\n",
            " 50% 5898/11873 [00:17<00:16, 354.35it/s]\u001b[A\n",
            " 50% 5936/11873 [00:17<00:16, 361.25it/s]\u001b[A\n",
            " 50% 5975/11873 [00:17<00:16, 368.47it/s]\u001b[A\n",
            " 51% 6012/11873 [00:17<00:16, 360.54it/s]\u001b[A\n",
            " 51% 6049/11873 [00:17<00:16, 361.58it/s]\u001b[A\n",
            " 51% 6086/11873 [00:17<00:15, 361.87it/s]\u001b[A\n",
            " 52% 6123/11873 [00:18<00:16, 356.86it/s]\u001b[A\n",
            " 52% 6159/11873 [00:18<00:16, 356.99it/s]\u001b[A\n",
            " 52% 6198/11873 [00:18<00:15, 364.83it/s]\u001b[A\n",
            " 53% 6235/11873 [00:18<00:15, 363.93it/s]\u001b[A\n",
            " 53% 6272/11873 [00:18<00:15, 358.71it/s]\u001b[A\n",
            " 53% 6308/11873 [00:18<00:15, 353.80it/s]\u001b[A\n",
            " 53% 6344/11873 [00:18<00:15, 349.13it/s]\u001b[A\n",
            " 54% 6380/11873 [00:18<00:15, 351.45it/s]\u001b[A\n",
            " 54% 6417/11873 [00:18<00:15, 354.84it/s]\u001b[A\n",
            " 54% 6453/11873 [00:18<00:15, 347.58it/s]\u001b[A\n",
            " 55% 6490/11873 [00:19<00:15, 352.03it/s]\u001b[A\n",
            " 55% 6526/11873 [00:19<00:15, 338.34it/s]\u001b[A\n",
            " 55% 6560/11873 [00:19<00:15, 338.22it/s]\u001b[A\n",
            " 56% 6594/11873 [00:19<00:15, 336.66it/s]\u001b[A\n",
            " 56% 6633/11873 [00:19<00:14, 350.56it/s]\u001b[A\n",
            " 56% 6672/11873 [00:19<00:14, 358.99it/s]\u001b[A\n",
            " 57% 6709/11873 [00:19<00:16, 319.12it/s]\u001b[A\n",
            " 57% 6742/11873 [00:19<00:16, 306.37it/s]\u001b[A\n",
            " 57% 6778/11873 [00:19<00:15, 319.94it/s]\u001b[A\n",
            " 57% 6812/11873 [00:20<00:15, 324.70it/s]\u001b[A\n",
            " 58% 6851/11873 [00:20<00:14, 340.00it/s]\u001b[A\n",
            " 58% 6886/11873 [00:20<00:14, 341.16it/s]\u001b[A\n",
            " 58% 6921/11873 [00:20<00:14, 336.70it/s]\u001b[A\n",
            " 59% 6958/11873 [00:20<00:14, 344.78it/s]\u001b[A\n",
            " 59% 6995/11873 [00:20<00:13, 351.21it/s]\u001b[A\n",
            " 59% 7031/11873 [00:20<00:13, 353.13it/s]\u001b[A\n",
            " 60% 7067/11873 [00:20<00:13, 347.24it/s]\u001b[A\n",
            " 60% 7103/11873 [00:20<00:13, 349.91it/s]\u001b[A\n",
            " 60% 7140/11873 [00:21<00:13, 355.64it/s]\u001b[A\n",
            " 60% 7177/11873 [00:21<00:13, 357.69it/s]\u001b[A\n",
            " 61% 7213/11873 [00:21<00:13, 343.04it/s]\u001b[A\n",
            " 61% 7248/11873 [00:21<00:13, 344.68it/s]\u001b[A\n",
            " 61% 7283/11873 [00:21<00:13, 335.20it/s]\u001b[A\n",
            " 62% 7319/11873 [00:21<00:13, 341.61it/s]\u001b[A\n",
            " 62% 7356/11873 [00:21<00:12, 348.20it/s]\u001b[A\n",
            " 62% 7391/11873 [00:21<00:13, 332.66it/s]\u001b[A\n",
            " 63% 7425/11873 [00:21<00:13, 317.77it/s]\u001b[A\n",
            " 63% 7463/11873 [00:21<00:13, 334.12it/s]\u001b[A\n",
            " 63% 7502/11873 [00:22<00:12, 346.99it/s]\u001b[A\n",
            " 64% 7541/11873 [00:22<00:12, 357.75it/s]\u001b[A\n",
            " 64% 7578/11873 [00:22<00:11, 361.07it/s]\u001b[A\n",
            " 64% 7615/11873 [00:22<00:12, 351.44it/s]\u001b[A\n",
            " 64% 7651/11873 [00:22<00:12, 348.29it/s]\u001b[A\n",
            " 65% 7687/11873 [00:22<00:12, 348.34it/s]\u001b[A\n",
            " 65% 7722/11873 [00:22<00:12, 333.74it/s]\u001b[A\n",
            " 65% 7756/11873 [00:22<00:12, 330.64it/s]\u001b[A\n",
            " 66% 7791/11873 [00:22<00:12, 333.70it/s]\u001b[A\n",
            " 66% 7826/11873 [00:23<00:12, 336.48it/s]\u001b[A\n",
            " 66% 7860/11873 [00:23<00:11, 335.10it/s]\u001b[A\n",
            " 66% 7894/11873 [00:23<00:13, 300.43it/s]\u001b[A\n",
            " 67% 7932/11873 [00:23<00:12, 319.74it/s]\u001b[A\n",
            " 67% 7967/11873 [00:23<00:11, 327.09it/s]\u001b[A\n",
            " 67% 8003/11873 [00:23<00:11, 332.81it/s]\u001b[A\n",
            " 68% 8041/11873 [00:23<00:11, 344.44it/s]\u001b[A\n",
            " 68% 8076/11873 [00:23<00:11, 340.49it/s]\u001b[A\n",
            " 68% 8112/11873 [00:23<00:10, 344.71it/s]\u001b[A\n",
            " 69% 8150/11873 [00:23<00:10, 353.77it/s]\u001b[A\n",
            " 69% 8186/11873 [00:24<00:10, 352.90it/s]\u001b[A\n",
            " 69% 8222/11873 [00:24<00:10, 338.82it/s]\u001b[A\n",
            " 70% 8260/11873 [00:24<00:10, 348.85it/s]\u001b[A\n",
            " 70% 8296/11873 [00:24<00:10, 346.75it/s]\u001b[A\n",
            " 70% 8332/11873 [00:24<00:10, 350.34it/s]\u001b[A\n",
            " 70% 8368/11873 [00:24<00:09, 350.85it/s]\u001b[A\n",
            " 71% 8404/11873 [00:24<00:10, 344.50it/s]\u001b[A\n",
            " 71% 8439/11873 [00:24<00:10, 327.70it/s]\u001b[A\n",
            " 71% 8473/11873 [00:24<00:10, 324.61it/s]\u001b[A\n",
            " 72% 8508/11873 [00:25<00:10, 331.50it/s]\u001b[A\n",
            " 72% 8544/11873 [00:25<00:09, 339.02it/s]\u001b[A\n",
            " 72% 8579/11873 [00:25<00:09, 339.32it/s]\u001b[A\n",
            " 73% 8614/11873 [00:25<00:10, 322.55it/s]\u001b[A\n",
            " 73% 8650/11873 [00:25<00:09, 330.90it/s]\u001b[A\n",
            " 73% 8687/11873 [00:25<00:09, 338.78it/s]\u001b[A\n",
            " 73% 8723/11873 [00:25<00:09, 344.78it/s]\u001b[A\n",
            " 74% 8762/11873 [00:25<00:08, 354.69it/s]\u001b[A\n",
            " 74% 8798/11873 [00:25<00:08, 344.01it/s]\u001b[A\n",
            " 74% 8834/11873 [00:25<00:08, 348.48it/s]\u001b[A\n",
            " 75% 8871/11873 [00:26<00:08, 354.61it/s]\u001b[A\n",
            " 75% 8907/11873 [00:26<00:08, 348.76it/s]\u001b[A\n",
            " 75% 8942/11873 [00:26<00:08, 346.56it/s]\u001b[A\n",
            " 76% 8980/11873 [00:26<00:08, 354.22it/s]\u001b[A\n",
            " 76% 9016/11873 [00:26<00:08, 352.58it/s]\u001b[A\n",
            " 76% 9052/11873 [00:26<00:08, 351.31it/s]\u001b[A\n",
            " 77% 9091/11873 [00:26<00:07, 359.73it/s]\u001b[A\n",
            " 77% 9128/11873 [00:26<00:07, 349.87it/s]\u001b[A\n",
            " 77% 9167/11873 [00:26<00:07, 359.89it/s]\u001b[A\n",
            " 78% 9206/11873 [00:27<00:07, 366.07it/s]\u001b[A\n",
            " 78% 9243/11873 [00:27<00:07, 359.21it/s]\u001b[A\n",
            " 78% 9280/11873 [00:27<00:07, 358.37it/s]\u001b[A\n",
            " 78% 9316/11873 [00:27<00:07, 349.80it/s]\u001b[A\n",
            " 79% 9352/11873 [00:27<00:07, 343.11it/s]\u001b[A\n",
            " 79% 9388/11873 [00:27<00:07, 345.60it/s]\u001b[A\n",
            " 79% 9424/11873 [00:27<00:07, 348.13it/s]\u001b[A\n",
            " 80% 9460/11873 [00:27<00:06, 350.04it/s]\u001b[A\n",
            " 80% 9496/11873 [00:27<00:06, 348.60it/s]\u001b[A\n",
            " 80% 9533/11873 [00:27<00:06, 354.19it/s]\u001b[A\n",
            " 81% 9569/11873 [00:28<00:06, 350.58it/s]\u001b[A\n",
            " 81% 9605/11873 [00:28<00:06, 351.50it/s]\u001b[A\n",
            " 81% 9642/11873 [00:28<00:06, 355.17it/s]\u001b[A\n",
            " 82% 9678/11873 [00:28<00:06, 353.96it/s]\u001b[A\n",
            " 82% 9717/11873 [00:28<00:05, 363.57it/s]\u001b[A\n",
            " 82% 9754/11873 [00:28<00:05, 361.93it/s]\u001b[A\n",
            " 82% 9791/11873 [00:28<00:05, 357.46it/s]\u001b[A\n",
            " 83% 9827/11873 [00:28<00:05, 355.99it/s]\u001b[A\n",
            " 83% 9864/11873 [00:28<00:05, 358.52it/s]\u001b[A\n",
            " 83% 9900/11873 [00:28<00:05, 351.09it/s]\u001b[A\n",
            " 84% 9937/11873 [00:29<00:05, 355.65it/s]\u001b[A\n",
            " 84% 9976/11873 [00:29<00:05, 363.77it/s]\u001b[A\n",
            " 84% 10013/11873 [00:29<00:05, 353.41it/s]\u001b[A\n",
            " 85% 10052/11873 [00:29<00:05, 360.83it/s]\u001b[A\n",
            " 85% 10089/11873 [00:29<00:05, 353.14it/s]\u001b[A\n",
            " 85% 10125/11873 [00:29<00:04, 354.32it/s]\u001b[A\n",
            " 86% 10163/11873 [00:29<00:04, 361.63it/s]\u001b[A\n",
            " 86% 10200/11873 [00:29<00:04, 359.80it/s]\u001b[A\n",
            " 86% 10237/11873 [00:29<00:04, 357.33it/s]\u001b[A\n",
            " 87% 10273/11873 [00:30<00:04, 357.37it/s]\u001b[A\n",
            " 87% 10309/11873 [00:30<00:04, 341.83it/s]\u001b[A\n",
            " 87% 10345/11873 [00:30<00:04, 344.50it/s]\u001b[A\n",
            " 87% 10380/11873 [00:30<00:04, 342.71it/s]\u001b[A\n",
            " 88% 10417/11873 [00:30<00:04, 348.44it/s]\u001b[A\n",
            " 88% 10452/11873 [00:30<00:04, 319.43it/s]\u001b[A\n",
            " 88% 10491/11873 [00:30<00:04, 337.74it/s]\u001b[A\n",
            " 89% 10528/11873 [00:30<00:03, 345.29it/s]\u001b[A\n",
            " 89% 10564/11873 [00:30<00:03, 333.86it/s]\u001b[A\n",
            " 89% 10599/11873 [00:31<00:03, 336.67it/s]\u001b[A\n",
            " 90% 10635/11873 [00:31<00:03, 340.14it/s]\u001b[A\n",
            " 90% 10670/11873 [00:31<00:03, 341.97it/s]\u001b[A\n",
            " 90% 10707/11873 [00:31<00:03, 349.80it/s]\u001b[A\n",
            " 90% 10744/11873 [00:31<00:03, 354.26it/s]\u001b[A\n",
            " 91% 10780/11873 [00:31<00:03, 341.33it/s]\u001b[A\n",
            " 91% 10815/11873 [00:31<00:03, 339.61it/s]\u001b[A\n",
            " 91% 10850/11873 [00:31<00:03, 322.88it/s]\u001b[A\n",
            " 92% 10885/11873 [00:31<00:02, 329.73it/s]\u001b[A\n",
            " 92% 10919/11873 [00:31<00:02, 326.31it/s]\u001b[A\n",
            " 92% 10952/11873 [00:32<00:02, 326.18it/s]\u001b[A\n",
            " 93% 10991/11873 [00:32<00:02, 341.89it/s]\u001b[A\n",
            " 93% 11029/11873 [00:32<00:02, 351.21it/s]\u001b[A\n",
            " 93% 11065/11873 [00:32<00:02, 344.13it/s]\u001b[A\n",
            " 94% 11104/11873 [00:32<00:02, 354.39it/s]\u001b[A\n",
            " 94% 11141/11873 [00:32<00:02, 357.31it/s]\u001b[A\n",
            " 94% 11178/11873 [00:32<00:01, 360.33it/s]\u001b[A\n",
            " 94% 11216/11873 [00:32<00:01, 365.98it/s]\u001b[A\n",
            " 95% 11253/11873 [00:32<00:01, 352.16it/s]\u001b[A\n",
            " 95% 11289/11873 [00:33<00:01, 349.22it/s]\u001b[A\n",
            " 95% 11325/11873 [00:33<00:01, 344.19it/s]\u001b[A\n",
            " 96% 11361/11873 [00:33<00:01, 347.52it/s]\u001b[A\n",
            " 96% 11399/11873 [00:33<00:01, 354.97it/s]\u001b[A\n",
            " 96% 11435/11873 [00:33<00:01, 350.99it/s]\u001b[A\n",
            " 97% 11471/11873 [00:33<00:01, 340.22it/s]\u001b[A\n",
            " 97% 11506/11873 [00:33<00:01, 334.30it/s]\u001b[A\n",
            " 97% 11542/11873 [00:33<00:00, 341.15it/s]\u001b[A\n",
            " 98% 11577/11873 [00:33<00:00, 342.29it/s]\u001b[A\n",
            " 98% 11612/11873 [00:33<00:00, 341.85it/s]\u001b[A\n",
            " 98% 11649/11873 [00:34<00:00, 348.16it/s]\u001b[A\n",
            " 98% 11684/11873 [00:34<00:00, 339.68it/s]\u001b[A\n",
            " 99% 11719/11873 [00:34<00:00, 337.62it/s]\u001b[A\n",
            " 99% 11754/11873 [00:34<00:00, 338.98it/s]\u001b[A\n",
            " 99% 11788/11873 [00:34<00:00, 337.85it/s]\u001b[A\n",
            "100% 11826/11873 [00:34<00:00, 348.35it/s]\u001b[A\n",
            "100% 11873/11873 [00:34<00:00, 342.27it/s]\n",
            "03/15/2021 10:12:39 - INFO - utils_qa -   Saving predictions to /tmp/debug_squad/predictions.json.\n",
            "03/15/2021 10:12:39 - INFO - utils_qa -   Saving nbest_preds to /tmp/debug_squad/nbest_predictions.json.\n",
            "Traceback (most recent call last):\n",
            "  File \"./transformers/examples/question-answering/run_qa.py\", line 546, in <module>\n",
            "    main()\n",
            "  File \"./transformers/examples/question-answering/run_qa.py\", line 531, in main\n",
            "    metrics = trainer.evaluate()\n",
            "  File \"/content/transformers/examples/question-answering/trainer_qa.py\", line 63, in evaluate\n",
            "    metrics = self.compute_metrics(eval_preds)\n",
            "  File \"./transformers/examples/question-answering/run_qa.py\", line 492, in compute_metrics\n",
            "    return metric.compute(predictions=p.predictions, references=p.label_ids)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/datasets/metric.py\", line 403, in compute\n",
            "    output = self._compute(predictions=predictions, references=references, **kwargs)\n",
            "  File \"/root/.cache/huggingface/modules/datasets_modules/metrics/squad/c0855591f1a2c2af8b7949e3146b9c86a6b7f536b4154019b03472639d310181/squad.py\", line 109, in _compute\n",
            "    score = evaluate(dataset=dataset, predictions=pred_dict)\n",
            "  File \"/root/.cache/huggingface/modules/datasets_modules/metrics/squad/c0855591f1a2c2af8b7949e3146b9c86a6b7f536b4154019b03472639d310181/evaluate.py\", line 68, in evaluate\n",
            "    exact_match += metric_max_over_ground_truths(exact_match_score, prediction, ground_truths)\n",
            "  File \"/root/.cache/huggingface/modules/datasets_modules/metrics/squad/c0855591f1a2c2af8b7949e3146b9c86a6b7f536b4154019b03472639d310181/evaluate.py\", line 53, in metric_max_over_ground_truths\n",
            "    return max(scores_for_ground_truths)\n",
            "ValueError: max() arg is an empty sequence\n",
            "100% 1522/1522 [1:42:20<00:00,  4.03s/it]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4dl-8ryNI3T2"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}