TestingAllHuggingfaceScripts_.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TestingAllHuggingfaceScripts_.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyPWio/uikSdJiO+mB20Z/wd",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bhadreshpsavani/UnderstandingNLP/blob/master/TestingAllHuggingfaceScripts_.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lYmlQiIUtS4O"
      },
      "source": [
        "## Install Transformers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k1aipZH0tEvW",
        "outputId": "82d36124-97f8-4212-d71c-0cf5acb84ada"
      },
      "source": [
        "!git clone https://github.com/huggingface/transformers.git\n",
        "# !git clone -b val-to-eval https://github.com/bhadreshpsavani/transformers.git\n",
        "%cd transformers\n",
        "!pip install ."
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'transformers'...\n",
            "remote: Enumerating objects: 70806, done.\u001b[K\n",
            "remote: Counting objects: 100% (895/895), done.\u001b[K\n",
            "remote: Compressing objects: 100% (497/497), done.\u001b[K\n",
            "remote: Total 70806 (delta 464), reused 707 (delta 333), pack-reused 69911\n",
            "Receiving objects: 100% (70806/70806), 54.07 MiB | 24.56 MiB/s, done.\n",
            "Resolving deltas: 100% (50034/50034), done.\n",
            "/content/transformers\n",
            "Processing /content/transformers\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==4.6.0.dev0) (2.23.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.6.0.dev0) (1.19.5)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 5.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==4.6.0.dev0) (4.41.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.6.0.dev0) (2019.12.20)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers==4.6.0.dev0) (20.9)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers==4.6.0.dev0) (3.10.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==4.6.0.dev0) (3.0.12)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 41.1MB/s \n",
            "\u001b[?25hCollecting huggingface-hub>=0.0.8\n",
            "  Downloading https://files.pythonhosted.org/packages/a1/88/7b1e45720ecf59c6c6737ff332f41c955963090a18e72acbcbeac6b25e86/huggingface_hub-0.0.8-py3-none-any.whl\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.6.0.dev0) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.6.0.dev0) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.6.0.dev0) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.6.0.dev0) (3.0.4)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers==4.6.0.dev0) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers==4.6.0.dev0) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers==4.6.0.dev0) (3.7.4.3)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.6.0.dev0) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.6.0.dev0) (1.0.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.6.0.dev0) (1.15.0)\n",
            "Building wheels for collected packages: transformers\n",
            "  Building wheel for transformers (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformers: filename=transformers-4.6.0.dev0-cp37-none-any.whl size=2112886 sha256=ca93905a5527a918acd853333309d4b7aa48aae04755f93b54c58e7bd7d90b6b\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-betod6fu/wheels/23/19/dd/2561a4e47240cf6b307729d58e56f8077dd0c698f5992216cf\n",
            "Successfully built transformers\n",
            "Installing collected packages: tokenizers, sacremoses, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.0.8 sacremoses-0.0.45 tokenizers-0.10.2 transformers-4.6.0.dev0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tukrPTYieLUP",
        "outputId": "42bb2468-81fc-4214-b3a6-1e6546693a0f"
      },
      "source": [
        "!pip install -q -r ./examples/pytorch/language-modeling/requirements.txt\n",
        "!pip install -q -r ./examples/pytorch/multiple-choice/requirements.txt\n",
        "!pip install -q -r ./examples/pytorch/question-answering/requirements.txt\n",
        "!pip install -q -r ./examples/pytorch/summarization/requirements.txt\n",
        "!pip install -q -r ./examples/pytorch/text-classification/requirements.txt\n",
        "!pip install -q -r ./examples/pytorch/translation/requirements.txt\n",
        "!pip install -q -r ./examples/pytorch/token-classification/requirements.txt\n",
        "%cd .."
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 204kB 5.3MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.2MB 9.3MB/s \n",
            "\u001b[K     |████████████████████████████████| 112kB 26.4MB/s \n",
            "\u001b[K     |████████████████████████████████| 245kB 19.0MB/s \n",
            "\u001b[K     |████████████████████████████████| 71kB 4.0MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.9MB 10.0MB/s \n",
            "\u001b[K     |████████████████████████████████| 2.2MB 28.4MB/s \n",
            "\u001b[K     |████████████████████████████████| 133kB 48.6MB/s \n",
            "\u001b[K     |████████████████████████████████| 51kB 3.1MB/s \n",
            "\u001b[K     |████████████████████████████████| 61kB 3.5MB/s \n",
            "\u001b[K     |████████████████████████████████| 51kB 2.9MB/s \n",
            "\u001b[?25h  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qNAHYedQtXru"
      },
      "source": [
        "## Language-modeling"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0Dyh-gx5e2KX",
        "outputId": "a9f388d4-0513-402c-d14f-5e4bbee3491c"
      },
      "source": [
        "!python transformers/examples/pytorch/language-modeling/run_clm.py \\\n",
        "--model_name_or_path gpt2 \\\n",
        "--dataset_name wikitext \\\n",
        "--max_train_samples 5 \\\n",
        "--max_val_samples 5 \\\n",
        "--dataset_config_name wikitext-2-raw-v1 \\\n",
        "--do_train \\\n",
        "--do_eval \\\n",
        "--output_dir /tmp/test-clm"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-22 03:27:23.848154: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "04/22/2021 03:27:26 - WARNING - __main__ -   Process rank: -1, device: cpu, n_gpu: 0distributed training: False, 16-bits training: False\n",
            "04/22/2021 03:27:26 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(output_dir=/tmp/test-clm, overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=IntervalStrategy.NO, prediction_loss_only=False, per_device_train_batch_size=8, per_device_eval_batch_size=8, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=5e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=3.0, max_steps=-1, lr_scheduler_type=SchedulerType.LINEAR, warmup_ratio=0.0, warmup_steps=0, logging_dir=runs/Apr22_03-27-26_a44dfd0e8363, logging_strategy=IntervalStrategy.STEPS, logging_first_step=False, logging_steps=500, save_strategy=IntervalStrategy.STEPS, save_steps=500, save_total_limit=None, no_cuda=False, seed=42, fp16=False, fp16_opt_level=O1, fp16_backend=auto, fp16_full_eval=False, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=500, dataloader_num_workers=0, past_index=-1, run_name=/tmp/test-clm, disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, length_column_name=length, report_to=['tensorboard'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, skip_memory_metrics=False, use_legacy_prediction_loop=False, _n_gpu=0, mp_parameters=)\n",
            "Downloading: 8.33kB [00:00, 5.18MB/s]       \n",
            "Downloading: 5.83kB [00:00, 4.06MB/s]       \n",
            "Downloading and preparing dataset wikitext/wikitext-2-raw-v1 (download: 4.50 MiB, generated: 12.91 MiB, post-processed: Unknown size, total: 17.41 MiB) to /root/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/aa5e094000ec7afeb74c3be92c88313cd6f132d564c7effd961c10fd47c76f20...\n",
            "Downloading: 100% 4.72M/4.72M [00:00<00:00, 15.6MB/s]\n",
            "Dataset wikitext downloaded and prepared to /root/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/aa5e094000ec7afeb74c3be92c88313cd6f132d564c7effd961c10fd47c76f20. Subsequent calls will reuse this data.\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:27:28,786 >> https://huggingface.co/gpt2/resolve/main/config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp70k7z03h\n",
            "Downloading: 100% 665/665 [00:00<00:00, 462kB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:27:28,890 >> storing https://huggingface.co/gpt2/resolve/main/config.json in cache at /root/.cache/huggingface/transformers/fc674cd6907b4c9e933cb42d67662436b89fa9540a1f40d7c919d0109289ad01.7d2e0efa5ca20cef4fb199382111e9d3ad96fd77b849e1d4bed13a66e1336f51\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:27:28,890 >> creating metadata file for /root/.cache/huggingface/transformers/fc674cd6907b4c9e933cb42d67662436b89fa9540a1f40d7c919d0109289ad01.7d2e0efa5ca20cef4fb199382111e9d3ad96fd77b849e1d4bed13a66e1336f51\n",
            "[INFO|configuration_utils.py:491] 2021-04-22 03:27:28,890 >> loading configuration file https://huggingface.co/gpt2/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/fc674cd6907b4c9e933cb42d67662436b89fa9540a1f40d7c919d0109289ad01.7d2e0efa5ca20cef4fb199382111e9d3ad96fd77b849e1d4bed13a66e1336f51\n",
            "[INFO|configuration_utils.py:527] 2021-04-22 03:27:28,891 >> Model config GPT2Config {\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPT2LMHeadModel\"\n",
            "  ],\n",
            "  \"attn_pdrop\": 0.1,\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embd_pdrop\": 0.1,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"model_type\": \"gpt2\",\n",
            "  \"n_ctx\": 1024,\n",
            "  \"n_embd\": 768,\n",
            "  \"n_head\": 12,\n",
            "  \"n_inner\": null,\n",
            "  \"n_layer\": 12,\n",
            "  \"n_positions\": 1024,\n",
            "  \"resid_pdrop\": 0.1,\n",
            "  \"scale_attn_weights\": true,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 50\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257\n",
            "}\n",
            "\n",
            "[INFO|configuration_utils.py:491] 2021-04-22 03:27:28,993 >> loading configuration file https://huggingface.co/gpt2/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/fc674cd6907b4c9e933cb42d67662436b89fa9540a1f40d7c919d0109289ad01.7d2e0efa5ca20cef4fb199382111e9d3ad96fd77b849e1d4bed13a66e1336f51\n",
            "[INFO|configuration_utils.py:527] 2021-04-22 03:27:28,994 >> Model config GPT2Config {\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPT2LMHeadModel\"\n",
            "  ],\n",
            "  \"attn_pdrop\": 0.1,\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embd_pdrop\": 0.1,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"model_type\": \"gpt2\",\n",
            "  \"n_ctx\": 1024,\n",
            "  \"n_embd\": 768,\n",
            "  \"n_head\": 12,\n",
            "  \"n_inner\": null,\n",
            "  \"n_layer\": 12,\n",
            "  \"n_positions\": 1024,\n",
            "  \"resid_pdrop\": 0.1,\n",
            "  \"scale_attn_weights\": true,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 50\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257\n",
            "}\n",
            "\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:27:29,096 >> https://huggingface.co/gpt2/resolve/main/vocab.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpqv2a3d6r\n",
            "Downloading: 100% 1.04M/1.04M [00:00<00:00, 5.84MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:27:29,387 >> storing https://huggingface.co/gpt2/resolve/main/vocab.json in cache at /root/.cache/huggingface/transformers/684fe667923972fb57f6b4dcb61a3c92763ad89882f3da5da9866baf14f2d60f.c7ed1f96aac49e745788faa77ba0a26a392643a50bb388b9c04ff469e555241f\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:27:29,387 >> creating metadata file for /root/.cache/huggingface/transformers/684fe667923972fb57f6b4dcb61a3c92763ad89882f3da5da9866baf14f2d60f.c7ed1f96aac49e745788faa77ba0a26a392643a50bb388b9c04ff469e555241f\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:27:29,490 >> https://huggingface.co/gpt2/resolve/main/merges.txt not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpywzmb5t2\n",
            "Downloading: 100% 456k/456k [00:00<00:00, 3.17MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:27:29,742 >> storing https://huggingface.co/gpt2/resolve/main/merges.txt in cache at /root/.cache/huggingface/transformers/c0c761a63004025aeadd530c4c27b860ec4ecbe8a00531233de21d865a402598.5d12962c5ee615a4c803841266e9c3be9a691a924f72d395d3a6c6c81157788b\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:27:29,742 >> creating metadata file for /root/.cache/huggingface/transformers/c0c761a63004025aeadd530c4c27b860ec4ecbe8a00531233de21d865a402598.5d12962c5ee615a4c803841266e9c3be9a691a924f72d395d3a6c6c81157788b\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:27:29,846 >> https://huggingface.co/gpt2/resolve/main/tokenizer.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmplfai2dx6\n",
            "Downloading: 100% 1.36M/1.36M [00:00<00:00, 7.53MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:27:30,140 >> storing https://huggingface.co/gpt2/resolve/main/tokenizer.json in cache at /root/.cache/huggingface/transformers/16a2f78023c8dc511294f0c97b5e10fde3ef9889ad6d11ffaa2a00714e73926e.cf2d0ecb83b6df91b3dbb53f1d1e4c311578bfd3aa0e04934215a49bf9898df0\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:27:30,140 >> creating metadata file for /root/.cache/huggingface/transformers/16a2f78023c8dc511294f0c97b5e10fde3ef9889ad6d11ffaa2a00714e73926e.cf2d0ecb83b6df91b3dbb53f1d1e4c311578bfd3aa0e04934215a49bf9898df0\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:27:30,452 >> loading file https://huggingface.co/gpt2/resolve/main/vocab.json from cache at /root/.cache/huggingface/transformers/684fe667923972fb57f6b4dcb61a3c92763ad89882f3da5da9866baf14f2d60f.c7ed1f96aac49e745788faa77ba0a26a392643a50bb388b9c04ff469e555241f\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:27:30,452 >> loading file https://huggingface.co/gpt2/resolve/main/merges.txt from cache at /root/.cache/huggingface/transformers/c0c761a63004025aeadd530c4c27b860ec4ecbe8a00531233de21d865a402598.5d12962c5ee615a4c803841266e9c3be9a691a924f72d395d3a6c6c81157788b\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:27:30,452 >> loading file https://huggingface.co/gpt2/resolve/main/tokenizer.json from cache at /root/.cache/huggingface/transformers/16a2f78023c8dc511294f0c97b5e10fde3ef9889ad6d11ffaa2a00714e73926e.cf2d0ecb83b6df91b3dbb53f1d1e4c311578bfd3aa0e04934215a49bf9898df0\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:27:30,452 >> loading file https://huggingface.co/gpt2/resolve/main/added_tokens.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:27:30,452 >> loading file https://huggingface.co/gpt2/resolve/main/special_tokens_map.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:27:30,452 >> loading file https://huggingface.co/gpt2/resolve/main/tokenizer_config.json from cache at None\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:27:30,645 >> https://huggingface.co/gpt2/resolve/main/pytorch_model.bin not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpxahbdliz\n",
            "Downloading: 100% 548M/548M [00:11<00:00, 47.3MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:27:42,807 >> storing https://huggingface.co/gpt2/resolve/main/pytorch_model.bin in cache at /root/.cache/huggingface/transformers/752929ace039baa8ef70fe21cdf9ab9445773d20e733cf693d667982e210837e.323c769945a351daa25546176f8208b3004b6f563438a7603e7932bae9025925\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:27:42,808 >> creating metadata file for /root/.cache/huggingface/transformers/752929ace039baa8ef70fe21cdf9ab9445773d20e733cf693d667982e210837e.323c769945a351daa25546176f8208b3004b6f563438a7603e7932bae9025925\n",
            "[INFO|modeling_utils.py:1075] 2021-04-22 03:27:42,808 >> loading weights file https://huggingface.co/gpt2/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/752929ace039baa8ef70fe21cdf9ab9445773d20e733cf693d667982e210837e.323c769945a351daa25546176f8208b3004b6f563438a7603e7932bae9025925\n",
            "[INFO|modeling_utils.py:1204] 2021-04-22 03:27:48,181 >> All model checkpoint weights were used when initializing GPT2LMHeadModel.\n",
            "\n",
            "[INFO|modeling_utils.py:1213] 2021-04-22 03:27:48,181 >> All the weights of GPT2LMHeadModel were initialized from the model checkpoint at gpt2.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use GPT2LMHeadModel for predictions without further training.\n",
            "100% 5/5 [00:00<00:00,  7.37ba/s]\n",
            "100% 37/37 [00:05<00:00,  6.34ba/s]\n",
            "100% 4/4 [00:00<00:00,  6.87ba/s]\n",
            "100% 5/5 [00:03<00:00,  1.38ba/s]\n",
            "100% 37/37 [00:40<00:00,  1.10s/ba]\n",
            "100% 4/4 [00:03<00:00,  1.18ba/s]\n",
            "[INFO|trainer.py:1107] 2021-04-22 03:28:43,449 >> ***** Running training *****\n",
            "[INFO|trainer.py:1108] 2021-04-22 03:28:43,449 >>   Num examples = 5\n",
            "[INFO|trainer.py:1109] 2021-04-22 03:28:43,449 >>   Num Epochs = 3\n",
            "[INFO|trainer.py:1110] 2021-04-22 03:28:43,449 >>   Instantaneous batch size per device = 8\n",
            "[INFO|trainer.py:1111] 2021-04-22 03:28:43,450 >>   Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "[INFO|trainer.py:1112] 2021-04-22 03:28:43,450 >>   Gradient Accumulation steps = 1\n",
            "[INFO|trainer.py:1113] 2021-04-22 03:28:43,450 >>   Total optimization steps = 3\n",
            "  0% 0/3 [00:00<?, ?it/s]^C\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gUGsmzYYfnD7",
        "outputId": "3aadabac-a929-474c-fef4-fe0bc681e067"
      },
      "source": [
        "!python transformers/examples/pytorch/language-modeling/run_mlm.py \\\n",
        "--model_name_or_path distilbert-base-cased \\\n",
        "--dataset_name wikitext \\\n",
        "--max_train_samples 5 \\\n",
        "--max_val_samples 5 \\\n",
        "--dataset_config_name wikitext-2-raw-v1 \\\n",
        "--do_train \\\n",
        "--do_eval \\\n",
        "--output_dir /tmp/test-clm"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-22 03:31:37.597298: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "04/22/2021 03:31:43 - WARNING - __main__ -   Process rank: -1, device: cpu, n_gpu: 0distributed training: False, 16-bits training: False\n",
            "04/22/2021 03:31:43 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(output_dir=/tmp/test-clm, overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=IntervalStrategy.NO, prediction_loss_only=False, per_device_train_batch_size=8, per_device_eval_batch_size=8, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=5e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=3.0, max_steps=-1, lr_scheduler_type=SchedulerType.LINEAR, warmup_ratio=0.0, warmup_steps=0, logging_dir=runs/Apr22_03-31-43_a44dfd0e8363, logging_strategy=IntervalStrategy.STEPS, logging_first_step=False, logging_steps=500, save_strategy=IntervalStrategy.STEPS, save_steps=500, save_total_limit=None, no_cuda=False, seed=42, fp16=False, fp16_opt_level=O1, fp16_backend=auto, fp16_full_eval=False, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=500, dataloader_num_workers=0, past_index=-1, run_name=/tmp/test-clm, disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, length_column_name=length, report_to=['tensorboard'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, skip_memory_metrics=False, use_legacy_prediction_loop=False, _n_gpu=0, mp_parameters=)\n",
            "04/22/2021 03:31:43 - WARNING - datasets.builder -   Reusing dataset wikitext (/root/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/aa5e094000ec7afeb74c3be92c88313cd6f132d564c7effd961c10fd47c76f20)\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:31:44,327 >> https://huggingface.co/distilbert-base-cased/resolve/main/config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp062q1hm3\n",
            "Downloading: 100% 411/411 [00:00<00:00, 270kB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:31:44,430 >> storing https://huggingface.co/distilbert-base-cased/resolve/main/config.json in cache at /root/.cache/huggingface/transformers/ebe1ea24d11aa664488b8de5b21e33989008ca78f207d4e30ec6350b693f073f.302bfd1b5e031cc1b17796e0b6e5b242ba2045d31d00f97589e12b458ebff27a\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:31:44,430 >> creating metadata file for /root/.cache/huggingface/transformers/ebe1ea24d11aa664488b8de5b21e33989008ca78f207d4e30ec6350b693f073f.302bfd1b5e031cc1b17796e0b6e5b242ba2045d31d00f97589e12b458ebff27a\n",
            "[INFO|configuration_utils.py:491] 2021-04-22 03:31:44,430 >> loading configuration file https://huggingface.co/distilbert-base-cased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/ebe1ea24d11aa664488b8de5b21e33989008ca78f207d4e30ec6350b693f073f.302bfd1b5e031cc1b17796e0b6e5b242ba2045d31d00f97589e12b458ebff27a\n",
            "[INFO|configuration_utils.py:527] 2021-04-22 03:31:44,431 >> Model config DistilBertConfig {\n",
            "  \"activation\": \"gelu\",\n",
            "  \"attention_dropout\": 0.1,\n",
            "  \"dim\": 768,\n",
            "  \"dropout\": 0.1,\n",
            "  \"hidden_dim\": 3072,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"distilbert\",\n",
            "  \"n_heads\": 12,\n",
            "  \"n_layers\": 6,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"qa_dropout\": 0.1,\n",
            "  \"seq_classif_dropout\": 0.2,\n",
            "  \"sinusoidal_pos_embds\": false,\n",
            "  \"tie_weights_\": true,\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"vocab_size\": 28996\n",
            "}\n",
            "\n",
            "[INFO|configuration_utils.py:491] 2021-04-22 03:31:44,533 >> loading configuration file https://huggingface.co/distilbert-base-cased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/ebe1ea24d11aa664488b8de5b21e33989008ca78f207d4e30ec6350b693f073f.302bfd1b5e031cc1b17796e0b6e5b242ba2045d31d00f97589e12b458ebff27a\n",
            "[INFO|configuration_utils.py:527] 2021-04-22 03:31:44,533 >> Model config DistilBertConfig {\n",
            "  \"activation\": \"gelu\",\n",
            "  \"attention_dropout\": 0.1,\n",
            "  \"dim\": 768,\n",
            "  \"dropout\": 0.1,\n",
            "  \"hidden_dim\": 3072,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"distilbert\",\n",
            "  \"n_heads\": 12,\n",
            "  \"n_layers\": 6,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"qa_dropout\": 0.1,\n",
            "  \"seq_classif_dropout\": 0.2,\n",
            "  \"sinusoidal_pos_embds\": false,\n",
            "  \"tie_weights_\": true,\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"vocab_size\": 28996\n",
            "}\n",
            "\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:31:44,637 >> https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpoam5v8em\n",
            "Downloading: 100% 213k/213k [00:00<00:00, 1.88MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:31:44,854 >> storing https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt in cache at /root/.cache/huggingface/transformers/ba377304984dc63e3ede0e23a938bbbf04d5c3835b66d5bb48343aecca188429.437aa611e89f6fc6675a049d2b5545390adbc617e7d655286421c191d2be2791\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:31:44,854 >> creating metadata file for /root/.cache/huggingface/transformers/ba377304984dc63e3ede0e23a938bbbf04d5c3835b66d5bb48343aecca188429.437aa611e89f6fc6675a049d2b5545390adbc617e7d655286421c191d2be2791\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:31:44,957 >> https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpe_tqhgqq\n",
            "Downloading: 100% 436k/436k [00:00<00:00, 3.01MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:31:45,211 >> storing https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json in cache at /root/.cache/huggingface/transformers/acb5c2138c1f8c84f074b86dafce3631667fccd6efcb1a7ea1320cf75c386a36.3dab63143af66769bbb35e3811f75f7e16b2320e12b7935e216bd6159ce6d9a6\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:31:45,211 >> creating metadata file for /root/.cache/huggingface/transformers/acb5c2138c1f8c84f074b86dafce3631667fccd6efcb1a7ea1320cf75c386a36.3dab63143af66769bbb35e3811f75f7e16b2320e12b7935e216bd6159ce6d9a6\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:31:45,529 >> https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer_config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmphe3k_axt\n",
            "Downloading: 100% 29.0/29.0 [00:00<00:00, 21.2kB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:31:45,636 >> storing https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer_config.json in cache at /root/.cache/huggingface/transformers/81e970e5e6ec68be12da0f8f3b2f2469c78d579282299a2ea65b4b7441719107.ec5c189f89475aac7d8cbd243960a0655cfadc3d0474da8ff2ed0bf1699c2a5f\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:31:45,636 >> creating metadata file for /root/.cache/huggingface/transformers/81e970e5e6ec68be12da0f8f3b2f2469c78d579282299a2ea65b4b7441719107.ec5c189f89475aac7d8cbd243960a0655cfadc3d0474da8ff2ed0bf1699c2a5f\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:31:45,636 >> loading file https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt from cache at /root/.cache/huggingface/transformers/ba377304984dc63e3ede0e23a938bbbf04d5c3835b66d5bb48343aecca188429.437aa611e89f6fc6675a049d2b5545390adbc617e7d655286421c191d2be2791\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:31:45,637 >> loading file https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json from cache at /root/.cache/huggingface/transformers/acb5c2138c1f8c84f074b86dafce3631667fccd6efcb1a7ea1320cf75c386a36.3dab63143af66769bbb35e3811f75f7e16b2320e12b7935e216bd6159ce6d9a6\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:31:45,637 >> loading file https://huggingface.co/distilbert-base-cased/resolve/main/added_tokens.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:31:45,637 >> loading file https://huggingface.co/distilbert-base-cased/resolve/main/special_tokens_map.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:31:45,637 >> loading file https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer_config.json from cache at /root/.cache/huggingface/transformers/81e970e5e6ec68be12da0f8f3b2f2469c78d579282299a2ea65b4b7441719107.ec5c189f89475aac7d8cbd243960a0655cfadc3d0474da8ff2ed0bf1699c2a5f\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:31:45,790 >> https://huggingface.co/distilbert-base-cased/resolve/main/pytorch_model.bin not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp3sm_4vni\n",
            "Downloading: 100% 263M/263M [00:13<00:00, 19.8MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:31:59,267 >> storing https://huggingface.co/distilbert-base-cased/resolve/main/pytorch_model.bin in cache at /root/.cache/huggingface/transformers/9c9f39769dba4c5fe379b4bc82973eb01297bd607954621434eb9f1bc85a23a0.06b428c87335c1bb22eae46fdab31c8286efa0aa09e898a7ac42ddf5c3f5dc19\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:31:59,267 >> creating metadata file for /root/.cache/huggingface/transformers/9c9f39769dba4c5fe379b4bc82973eb01297bd607954621434eb9f1bc85a23a0.06b428c87335c1bb22eae46fdab31c8286efa0aa09e898a7ac42ddf5c3f5dc19\n",
            "[INFO|modeling_utils.py:1075] 2021-04-22 03:31:59,268 >> loading weights file https://huggingface.co/distilbert-base-cased/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/9c9f39769dba4c5fe379b4bc82973eb01297bd607954621434eb9f1bc85a23a0.06b428c87335c1bb22eae46fdab31c8286efa0aa09e898a7ac42ddf5c3f5dc19\n",
            "[INFO|modeling_utils.py:1204] 2021-04-22 03:32:01,685 >> All model checkpoint weights were used when initializing DistilBertForMaskedLM.\n",
            "\n",
            "[INFO|modeling_utils.py:1213] 2021-04-22 03:32:01,686 >> All the weights of DistilBertForMaskedLM were initialized from the model checkpoint at distilbert-base-cased.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use DistilBertForMaskedLM for predictions without further training.\n",
            " 60% 3/5 [00:00<00:00,  5.48ba/s][WARNING|tokenization_utils_base.py:3154] 2021-04-22 03:32:02,336 >> Token indices sequence length is longer than the specified maximum sequence length for this model (546 > 512). Running this sequence through the model will result in indexing errors\n",
            "100% 5/5 [00:00<00:00,  7.10ba/s]\n",
            "100% 37/37 [00:05<00:00,  6.49ba/s]\n",
            "100% 4/4 [00:00<00:00,  7.09ba/s]\n",
            "100% 5/5 [00:02<00:00,  2.12ba/s]\n",
            "100% 37/37 [00:21<00:00,  1.74ba/s]\n",
            "100% 4/4 [00:02<00:00,  1.95ba/s]\n",
            "[INFO|trainer.py:497] 2021-04-22 03:32:34,774 >> The following columns in the training set  don't have a corresponding argument in `DistilBertForMaskedLM.forward` and have been ignored: special_tokens_mask.\n",
            "[INFO|trainer.py:1107] 2021-04-22 03:32:34,799 >> ***** Running training *****\n",
            "[INFO|trainer.py:1108] 2021-04-22 03:32:34,799 >>   Num examples = 5\n",
            "[INFO|trainer.py:1109] 2021-04-22 03:32:34,799 >>   Num Epochs = 3\n",
            "[INFO|trainer.py:1110] 2021-04-22 03:32:34,799 >>   Instantaneous batch size per device = 8\n",
            "[INFO|trainer.py:1111] 2021-04-22 03:32:34,799 >>   Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "[INFO|trainer.py:1112] 2021-04-22 03:32:34,800 >>   Gradient Accumulation steps = 1\n",
            "[INFO|trainer.py:1113] 2021-04-22 03:32:34,800 >>   Total optimization steps = 3\n",
            "100% 3/3 [01:46<00:00, 35.63s/it][INFO|trainer.py:1302] 2021-04-22 03:34:21,044 >> \n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "{'train_runtime': 106.244, 'train_samples_per_second': 0.028, 'epoch': 3.0}\n",
            "100% 3/3 [01:46<00:00, 35.41s/it]\n",
            "[INFO|trainer.py:1768] 2021-04-22 03:34:21,191 >> Saving model checkpoint to /tmp/test-clm\n",
            "[INFO|configuration_utils.py:329] 2021-04-22 03:34:21,192 >> Configuration saved in /tmp/test-clm/config.json\n",
            "[INFO|modeling_utils.py:848] 2021-04-22 03:34:21,714 >> Model weights saved in /tmp/test-clm/pytorch_model.bin\n",
            "[INFO|tokenization_utils_base.py:1918] 2021-04-22 03:34:21,714 >> tokenizer config file saved in /tmp/test-clm/tokenizer_config.json\n",
            "[INFO|tokenization_utils_base.py:1924] 2021-04-22 03:34:21,715 >> Special tokens file saved in /tmp/test-clm/special_tokens_map.json\n",
            "[INFO|trainer_pt_utils.py:898] 2021-04-22 03:34:21,763 >> ***** train metrics *****\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:34:21,764 >>   epoch                      =        3.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:34:21,764 >>   init_mem_cpu_alloc_delta   =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:34:21,764 >>   init_mem_cpu_peaked_delta  =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:34:21,764 >>   train_mem_cpu_alloc_delta  =     4039MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:34:21,764 >>   train_mem_cpu_peaked_delta =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:34:21,764 >>   train_runtime              = 0:01:46.24\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:34:21,764 >>   train_samples              =          5\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:34:21,764 >>   train_samples_per_second   =      0.028\n",
            "04/22/2021 03:34:21 - INFO - __main__ -   *** Evaluate ***\n",
            "[INFO|trainer.py:497] 2021-04-22 03:34:21,897 >> The following columns in the evaluation set  don't have a corresponding argument in `DistilBertForMaskedLM.forward` and have been ignored: special_tokens_mask.\n",
            "[INFO|trainer.py:1988] 2021-04-22 03:34:21,902 >> ***** Running Evaluation *****\n",
            "[INFO|trainer.py:1990] 2021-04-22 03:34:21,903 >>   Num examples = 5\n",
            "[INFO|trainer.py:1993] 2021-04-22 03:34:21,903 >>   Batch size = 8\n",
            "100% 1/1 [00:00<00:00, 319.40it/s]\n",
            "[INFO|trainer_pt_utils.py:898] 2021-04-22 03:34:33,168 >> ***** eval metrics *****\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:34:33,169 >>   epoch                     =        3.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:34:33,169 >>   eval_loss                 =      3.083\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:34:33,169 >>   eval_mem_cpu_alloc_delta  =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:34:33,169 >>   eval_mem_cpu_peaked_delta =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:34:33,169 >>   eval_runtime              = 0:00:11.13\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:34:33,169 >>   eval_samples              =          5\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:34:33,169 >>   eval_samples_per_second   =      0.449\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:34:33,169 >>   perplexity                =    21.8245\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RqNCV_yEfqHF",
        "outputId": "c3c58286-1082-4c69-a076-bd9117c4674d"
      },
      "source": [
        "!python transformers/examples/pytorch/language-modeling/run_plm.py \\\n",
        "--model_name_or_path xlnet-base-cased \\\n",
        "--dataset_name wikitext \\\n",
        "--max_train_samples 5 \\\n",
        "--max_val_samples 5 \\\n",
        "--dataset_config_name wikitext-2-raw-v1 \\\n",
        "--do_train \\\n",
        "--do_eval \\\n",
        "--output_dir /tmp/test-clm \\\n",
        "--overwrite_output_dir"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-22 03:34:49.464791: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "04/22/2021 03:34:51 - WARNING - __main__ -   Process rank: -1, device: cpu, n_gpu: 0distributed training: False, 16-bits training: False\n",
            "04/22/2021 03:34:51 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(output_dir=/tmp/test-clm, overwrite_output_dir=True, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=IntervalStrategy.NO, prediction_loss_only=False, per_device_train_batch_size=8, per_device_eval_batch_size=8, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=5e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=3.0, max_steps=-1, lr_scheduler_type=SchedulerType.LINEAR, warmup_ratio=0.0, warmup_steps=0, logging_dir=runs/Apr22_03-34-51_a44dfd0e8363, logging_strategy=IntervalStrategy.STEPS, logging_first_step=False, logging_steps=500, save_strategy=IntervalStrategy.STEPS, save_steps=500, save_total_limit=None, no_cuda=False, seed=42, fp16=False, fp16_opt_level=O1, fp16_backend=auto, fp16_full_eval=False, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=500, dataloader_num_workers=0, past_index=-1, run_name=/tmp/test-clm, disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, length_column_name=length, report_to=['tensorboard'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, skip_memory_metrics=False, use_legacy_prediction_loop=False, _n_gpu=0, mp_parameters=)\n",
            "04/22/2021 03:34:51 - WARNING - datasets.builder -   Reusing dataset wikitext (/root/.cache/huggingface/datasets/wikitext/wikitext-2-raw-v1/1.0.0/aa5e094000ec7afeb74c3be92c88313cd6f132d564c7effd961c10fd47c76f20)\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:34:52,146 >> https://huggingface.co/xlnet-base-cased/resolve/main/config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmphs5gn1lg\n",
            "Downloading: 100% 760/760 [00:00<00:00, 547kB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:34:52,250 >> storing https://huggingface.co/xlnet-base-cased/resolve/main/config.json in cache at /root/.cache/huggingface/transformers/06bdb0f5882dbb833618c81c3b4c996a0c79422fa2c95ffea3827f92fc2dba6b.da982e2e596ec73828dbae86525a1870e513bd63aae5a2dc773ccc840ac5c346\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:34:52,251 >> creating metadata file for /root/.cache/huggingface/transformers/06bdb0f5882dbb833618c81c3b4c996a0c79422fa2c95ffea3827f92fc2dba6b.da982e2e596ec73828dbae86525a1870e513bd63aae5a2dc773ccc840ac5c346\n",
            "[INFO|configuration_utils.py:491] 2021-04-22 03:34:52,251 >> loading configuration file https://huggingface.co/xlnet-base-cased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/06bdb0f5882dbb833618c81c3b4c996a0c79422fa2c95ffea3827f92fc2dba6b.da982e2e596ec73828dbae86525a1870e513bd63aae5a2dc773ccc840ac5c346\n",
            "[INFO|configuration_utils.py:527] 2021-04-22 03:34:52,252 >> Model config XLNetConfig {\n",
            "  \"architectures\": [\n",
            "    \"XLNetLMHeadModel\"\n",
            "  ],\n",
            "  \"attn_type\": \"bi\",\n",
            "  \"bi_data\": false,\n",
            "  \"bos_token_id\": 1,\n",
            "  \"clamp_len\": -1,\n",
            "  \"d_head\": 64,\n",
            "  \"d_inner\": 3072,\n",
            "  \"d_model\": 768,\n",
            "  \"dropout\": 0.1,\n",
            "  \"end_n_top\": 5,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"ff_activation\": \"gelu\",\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"mem_len\": null,\n",
            "  \"model_type\": \"xlnet\",\n",
            "  \"n_head\": 12,\n",
            "  \"n_layer\": 12,\n",
            "  \"pad_token_id\": 5,\n",
            "  \"reuse_len\": null,\n",
            "  \"same_length\": false,\n",
            "  \"start_n_top\": 5,\n",
            "  \"summary_activation\": \"tanh\",\n",
            "  \"summary_last_dropout\": 0.1,\n",
            "  \"summary_type\": \"last\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 250\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"untie_r\": true,\n",
            "  \"use_mems_eval\": true,\n",
            "  \"use_mems_train\": false,\n",
            "  \"vocab_size\": 32000\n",
            "}\n",
            "\n",
            "[INFO|configuration_utils.py:491] 2021-04-22 03:34:52,354 >> loading configuration file https://huggingface.co/xlnet-base-cased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/06bdb0f5882dbb833618c81c3b4c996a0c79422fa2c95ffea3827f92fc2dba6b.da982e2e596ec73828dbae86525a1870e513bd63aae5a2dc773ccc840ac5c346\n",
            "[INFO|configuration_utils.py:527] 2021-04-22 03:34:52,355 >> Model config XLNetConfig {\n",
            "  \"architectures\": [\n",
            "    \"XLNetLMHeadModel\"\n",
            "  ],\n",
            "  \"attn_type\": \"bi\",\n",
            "  \"bi_data\": false,\n",
            "  \"bos_token_id\": 1,\n",
            "  \"clamp_len\": -1,\n",
            "  \"d_head\": 64,\n",
            "  \"d_inner\": 3072,\n",
            "  \"d_model\": 768,\n",
            "  \"dropout\": 0.1,\n",
            "  \"end_n_top\": 5,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"ff_activation\": \"gelu\",\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"mem_len\": null,\n",
            "  \"model_type\": \"xlnet\",\n",
            "  \"n_head\": 12,\n",
            "  \"n_layer\": 12,\n",
            "  \"pad_token_id\": 5,\n",
            "  \"reuse_len\": null,\n",
            "  \"same_length\": false,\n",
            "  \"start_n_top\": 5,\n",
            "  \"summary_activation\": \"tanh\",\n",
            "  \"summary_last_dropout\": 0.1,\n",
            "  \"summary_type\": \"last\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 250\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"untie_r\": true,\n",
            "  \"use_mems_eval\": true,\n",
            "  \"use_mems_train\": false,\n",
            "  \"vocab_size\": 32000\n",
            "}\n",
            "\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:34:52,466 >> https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp5eiedd61\n",
            "Downloading: 100% 798k/798k [00:00<00:00, 5.24MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:34:52,728 >> storing https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model in cache at /root/.cache/huggingface/transformers/df73bc9f8d13bf2ea4dab95624895e45a550a0f0a825e41fc25440bf367ee3c8.d93497120e3a865e2970f26abdf7bf375896f97fde8b874b70909592a6c785c9\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:34:52,728 >> creating metadata file for /root/.cache/huggingface/transformers/df73bc9f8d13bf2ea4dab95624895e45a550a0f0a825e41fc25440bf367ee3c8.d93497120e3a865e2970f26abdf7bf375896f97fde8b874b70909592a6c785c9\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:34:52,834 >> https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpgeqdqbq0\n",
            "Downloading: 100% 1.38M/1.38M [00:00<00:00, 7.63MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:34:53,127 >> storing https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json in cache at /root/.cache/huggingface/transformers/46f47734f3dcaef7e236b9a3e887f27814e18836a8db7e6a49148000058a1a54.2a683f915238b4f560dab0c724066cf0a7de9a851e96b0fb3a1e7f0881552f53\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:34:53,127 >> creating metadata file for /root/.cache/huggingface/transformers/46f47734f3dcaef7e236b9a3e887f27814e18836a8db7e6a49148000058a1a54.2a683f915238b4f560dab0c724066cf0a7de9a851e96b0fb3a1e7f0881552f53\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:34:53,437 >> loading file https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model from cache at /root/.cache/huggingface/transformers/df73bc9f8d13bf2ea4dab95624895e45a550a0f0a825e41fc25440bf367ee3c8.d93497120e3a865e2970f26abdf7bf375896f97fde8b874b70909592a6c785c9\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:34:53,437 >> loading file https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json from cache at /root/.cache/huggingface/transformers/46f47734f3dcaef7e236b9a3e887f27814e18836a8db7e6a49148000058a1a54.2a683f915238b4f560dab0c724066cf0a7de9a851e96b0fb3a1e7f0881552f53\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:34:53,437 >> loading file https://huggingface.co/xlnet-base-cased/resolve/main/added_tokens.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:34:53,437 >> loading file https://huggingface.co/xlnet-base-cased/resolve/main/special_tokens_map.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:34:53,437 >> loading file https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer_config.json from cache at None\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:34:53,615 >> https://huggingface.co/xlnet-base-cased/resolve/main/pytorch_model.bin not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpwtf8j3k7\n",
            "Downloading: 100% 467M/467M [00:11<00:00, 41.8MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:35:05,277 >> storing https://huggingface.co/xlnet-base-cased/resolve/main/pytorch_model.bin in cache at /root/.cache/huggingface/transformers/9461853998373b0b2f8ef8011a13b62a2c5f540b2c535ef3ea46ed8a062b16a9.3e214f11a50e9e03eb47535b58522fc3cc11ac67c120a9450f6276de151af987\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:35:05,277 >> creating metadata file for /root/.cache/huggingface/transformers/9461853998373b0b2f8ef8011a13b62a2c5f540b2c535ef3ea46ed8a062b16a9.3e214f11a50e9e03eb47535b58522fc3cc11ac67c120a9450f6276de151af987\n",
            "[INFO|modeling_utils.py:1075] 2021-04-22 03:35:05,277 >> loading weights file https://huggingface.co/xlnet-base-cased/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/9461853998373b0b2f8ef8011a13b62a2c5f540b2c535ef3ea46ed8a062b16a9.3e214f11a50e9e03eb47535b58522fc3cc11ac67c120a9450f6276de151af987\n",
            "[INFO|modeling_utils.py:1204] 2021-04-22 03:35:09,540 >> All model checkpoint weights were used when initializing XLNetLMHeadModel.\n",
            "\n",
            "[INFO|modeling_utils.py:1213] 2021-04-22 03:35:09,540 >> All the weights of XLNetLMHeadModel were initialized from the model checkpoint at xlnet-base-cased.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use XLNetLMHeadModel for predictions without further training.\n",
            "100% 5/5 [00:01<00:00,  4.32ba/s]\n",
            "100% 37/37 [00:08<00:00,  4.26ba/s]\n",
            "100% 4/4 [00:00<00:00,  4.61ba/s]\n",
            "100% 5/5 [00:02<00:00,  1.99ba/s]\n",
            "100% 37/37 [00:22<00:00,  1.64ba/s]\n",
            "100% 4/4 [00:02<00:00,  1.79ba/s]\n",
            "[INFO|trainer.py:1107] 2021-04-22 03:35:48,009 >> ***** Running training *****\n",
            "[INFO|trainer.py:1108] 2021-04-22 03:35:48,009 >>   Num examples = 5\n",
            "[INFO|trainer.py:1109] 2021-04-22 03:35:48,009 >>   Num Epochs = 3\n",
            "[INFO|trainer.py:1110] 2021-04-22 03:35:48,009 >>   Instantaneous batch size per device = 8\n",
            "[INFO|trainer.py:1111] 2021-04-22 03:35:48,009 >>   Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "[INFO|trainer.py:1112] 2021-04-22 03:35:48,009 >>   Gradient Accumulation steps = 1\n",
            "[INFO|trainer.py:1113] 2021-04-22 03:35:48,009 >>   Total optimization steps = 3\n",
            " 33% 1/3 [02:30<05:01, 150.94s/it]^C\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xq6UpEX2eoed"
      },
      "source": [
        "## multiple-choice"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ovqijv9xhJ0r",
        "outputId": "e9fc0d47-beb1-4d05-b454-8c25d5d1c4e2"
      },
      "source": [
        "!python transformers/examples/pytorch/multiple-choice/run_swag.py \\\n",
        "--model_name_or_path distilbert-base-cased \\\n",
        "--do_train \\\n",
        "--do_eval \\\n",
        "--max_train_samples 5 \\\n",
        "--max_val_samples 5 \\\n",
        "--learning_rate 5e-5 \\\n",
        "--num_train_epochs 3 \\\n",
        "--output_dir /tmp/swag_base \\\n",
        "--per_gpu_eval_batch_size=16 \\\n",
        "--per_device_train_batch_size=16 \\\n",
        "--overwrite_output"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-22 03:39:29.628817: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "04/22/2021 03:39:35 - WARNING - __main__ -   Process rank: -1, device: cpu, n_gpu: 0distributed training: False, 16-bits training: False\n",
            "04/22/2021 03:39:35 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(output_dir=/tmp/swag_base, overwrite_output_dir=True, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=IntervalStrategy.NO, prediction_loss_only=False, per_device_train_batch_size=16, per_device_eval_batch_size=8, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=5e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=3.0, max_steps=-1, lr_scheduler_type=SchedulerType.LINEAR, warmup_ratio=0.0, warmup_steps=0, logging_dir=runs/Apr22_03-39-35_a44dfd0e8363, logging_strategy=IntervalStrategy.STEPS, logging_first_step=False, logging_steps=500, save_strategy=IntervalStrategy.STEPS, save_steps=500, save_total_limit=None, no_cuda=False, seed=42, fp16=False, fp16_opt_level=O1, fp16_backend=auto, fp16_full_eval=False, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=500, dataloader_num_workers=0, past_index=-1, run_name=/tmp/swag_base, disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, length_column_name=length, report_to=['tensorboard'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, skip_memory_metrics=False, use_legacy_prediction_loop=False, _n_gpu=0, mp_parameters=)\n",
            "Downloading: 7.97kB [00:00, 4.96MB/s]       \n",
            "Downloading: 7.10kB [00:00, 4.43MB/s]       \n",
            "Downloading and preparing dataset swag/regular (download: 41.92 MiB, generated: 44.96 MiB, post-processed: Unknown size, total: 86.88 MiB) to /root/.cache/huggingface/datasets/swag/regular/0.0.0/32c6a4e9a3c0880c9ace1df43c617cf89b7816037ebf2feabd4595ec4a02f74e...\n",
            "Downloading: 28.2MB [00:00, 61.4MB/s]\n",
            "Downloading: 7.89MB [00:00, 35.3MB/s]\n",
            "Downloading: 7.82MB [00:00, 34.1MB/s]\n",
            "Dataset swag downloaded and prepared to /root/.cache/huggingface/datasets/swag/regular/0.0.0/32c6a4e9a3c0880c9ace1df43c617cf89b7816037ebf2feabd4595ec4a02f74e. Subsequent calls will reuse this data.\n",
            "[INFO|configuration_utils.py:491] 2021-04-22 03:39:51,156 >> loading configuration file https://huggingface.co/distilbert-base-cased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/ebe1ea24d11aa664488b8de5b21e33989008ca78f207d4e30ec6350b693f073f.302bfd1b5e031cc1b17796e0b6e5b242ba2045d31d00f97589e12b458ebff27a\n",
            "[INFO|configuration_utils.py:527] 2021-04-22 03:39:51,157 >> Model config DistilBertConfig {\n",
            "  \"activation\": \"gelu\",\n",
            "  \"attention_dropout\": 0.1,\n",
            "  \"dim\": 768,\n",
            "  \"dropout\": 0.1,\n",
            "  \"hidden_dim\": 3072,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"distilbert\",\n",
            "  \"n_heads\": 12,\n",
            "  \"n_layers\": 6,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"qa_dropout\": 0.1,\n",
            "  \"seq_classif_dropout\": 0.2,\n",
            "  \"sinusoidal_pos_embds\": false,\n",
            "  \"tie_weights_\": true,\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"vocab_size\": 28996\n",
            "}\n",
            "\n",
            "[INFO|configuration_utils.py:491] 2021-04-22 03:39:51,260 >> loading configuration file https://huggingface.co/distilbert-base-cased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/ebe1ea24d11aa664488b8de5b21e33989008ca78f207d4e30ec6350b693f073f.302bfd1b5e031cc1b17796e0b6e5b242ba2045d31d00f97589e12b458ebff27a\n",
            "[INFO|configuration_utils.py:527] 2021-04-22 03:39:51,260 >> Model config DistilBertConfig {\n",
            "  \"activation\": \"gelu\",\n",
            "  \"attention_dropout\": 0.1,\n",
            "  \"dim\": 768,\n",
            "  \"dropout\": 0.1,\n",
            "  \"hidden_dim\": 3072,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"distilbert\",\n",
            "  \"n_heads\": 12,\n",
            "  \"n_layers\": 6,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"qa_dropout\": 0.1,\n",
            "  \"seq_classif_dropout\": 0.2,\n",
            "  \"sinusoidal_pos_embds\": false,\n",
            "  \"tie_weights_\": true,\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"vocab_size\": 28996\n",
            "}\n",
            "\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:39:51,780 >> loading file https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt from cache at /root/.cache/huggingface/transformers/ba377304984dc63e3ede0e23a938bbbf04d5c3835b66d5bb48343aecca188429.437aa611e89f6fc6675a049d2b5545390adbc617e7d655286421c191d2be2791\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:39:51,780 >> loading file https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json from cache at /root/.cache/huggingface/transformers/acb5c2138c1f8c84f074b86dafce3631667fccd6efcb1a7ea1320cf75c386a36.3dab63143af66769bbb35e3811f75f7e16b2320e12b7935e216bd6159ce6d9a6\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:39:51,780 >> loading file https://huggingface.co/distilbert-base-cased/resolve/main/added_tokens.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:39:51,780 >> loading file https://huggingface.co/distilbert-base-cased/resolve/main/special_tokens_map.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:39:51,780 >> loading file https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer_config.json from cache at /root/.cache/huggingface/transformers/81e970e5e6ec68be12da0f8f3b2f2469c78d579282299a2ea65b4b7441719107.ec5c189f89475aac7d8cbd243960a0655cfadc3d0474da8ff2ed0bf1699c2a5f\n",
            "[INFO|modeling_utils.py:1075] 2021-04-22 03:39:51,938 >> loading weights file https://huggingface.co/distilbert-base-cased/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/9c9f39769dba4c5fe379b4bc82973eb01297bd607954621434eb9f1bc85a23a0.06b428c87335c1bb22eae46fdab31c8286efa0aa09e898a7ac42ddf5c3f5dc19\n",
            "[WARNING|modeling_utils.py:1196] 2021-04-22 03:39:58,609 >> Some weights of the model checkpoint at distilbert-base-cased were not used when initializing DistilBertForMultipleChoice: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",
            "- This IS expected if you are initializing DistilBertForMultipleChoice from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing DistilBertForMultipleChoice from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "[WARNING|modeling_utils.py:1207] 2021-04-22 03:39:58,610 >> Some weights of DistilBertForMultipleChoice were not initialized from the model checkpoint at distilbert-base-cased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "100% 1/1 [00:00<00:00, 150.86ba/s]\n",
            "100% 1/1 [00:00<00:00, 210.92ba/s]\n",
            "[INFO|trainer.py:497] 2021-04-22 03:39:59,103 >> The following columns in the training set  don't have a corresponding argument in `DistilBertForMultipleChoice.forward` and have been ignored: sent1, ending3, video-id, ending0, startphrase, ending2, gold-source, sent2, fold-ind, ending1.\n",
            "[INFO|trainer.py:1107] 2021-04-22 03:39:59,111 >> ***** Running training *****\n",
            "[INFO|trainer.py:1108] 2021-04-22 03:39:59,111 >>   Num examples = 5\n",
            "[INFO|trainer.py:1109] 2021-04-22 03:39:59,111 >>   Num Epochs = 3\n",
            "[INFO|trainer.py:1110] 2021-04-22 03:39:59,111 >>   Instantaneous batch size per device = 16\n",
            "[INFO|trainer.py:1111] 2021-04-22 03:39:59,111 >>   Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "[INFO|trainer.py:1112] 2021-04-22 03:39:59,111 >>   Gradient Accumulation steps = 1\n",
            "[INFO|trainer.py:1113] 2021-04-22 03:39:59,111 >>   Total optimization steps = 3\n",
            "[WARNING|training_args.py:651] 2021-04-22 03:39:59,130 >> Using deprecated `--per_gpu_eval_batch_size` argument which will be removed in a future version. Using `--per_device_eval_batch_size` is preferred.\n",
            "100% 3/3 [00:22<00:00,  7.59s/it][INFO|trainer.py:1302] 2021-04-22 03:40:21,833 >> \n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "{'train_runtime': 22.7222, 'train_samples_per_second': 0.132, 'epoch': 3.0}\n",
            "100% 3/3 [00:22<00:00,  7.56s/it]\n",
            "[INFO|trainer.py:1768] 2021-04-22 03:40:21,974 >> Saving model checkpoint to /tmp/swag_base\n",
            "[INFO|configuration_utils.py:329] 2021-04-22 03:40:21,975 >> Configuration saved in /tmp/swag_base/config.json\n",
            "[INFO|modeling_utils.py:848] 2021-04-22 03:40:22,494 >> Model weights saved in /tmp/swag_base/pytorch_model.bin\n",
            "[INFO|tokenization_utils_base.py:1918] 2021-04-22 03:40:22,495 >> tokenizer config file saved in /tmp/swag_base/tokenizer_config.json\n",
            "[INFO|tokenization_utils_base.py:1924] 2021-04-22 03:40:22,495 >> Special tokens file saved in /tmp/swag_base/special_tokens_map.json\n",
            "[INFO|trainer_pt_utils.py:898] 2021-04-22 03:40:22,540 >> ***** train metrics *****\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:40:22,540 >>   epoch                      =        3.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:40:22,540 >>   init_mem_cpu_alloc_delta   =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:40:22,540 >>   init_mem_cpu_peaked_delta  =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:40:22,540 >>   train_mem_cpu_alloc_delta  =      857MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:40:22,540 >>   train_mem_cpu_peaked_delta =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:40:22,540 >>   train_runtime              = 0:00:22.72\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:40:22,540 >>   train_samples              =          5\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:40:22,540 >>   train_samples_per_second   =      0.132\n",
            "04/22/2021 03:40:22 - INFO - __main__ -   *** Evaluate ***\n",
            "[INFO|trainer.py:497] 2021-04-22 03:40:22,674 >> The following columns in the evaluation set  don't have a corresponding argument in `DistilBertForMultipleChoice.forward` and have been ignored: sent1, ending3, video-id, ending0, startphrase, ending2, gold-source, sent2, fold-ind, ending1.\n",
            "[WARNING|training_args.py:651] 2021-04-22 03:40:22,678 >> Using deprecated `--per_gpu_eval_batch_size` argument which will be removed in a future version. Using `--per_device_eval_batch_size` is preferred.\n",
            "[INFO|trainer.py:1988] 2021-04-22 03:40:22,678 >> ***** Running Evaluation *****\n",
            "[INFO|trainer.py:1990] 2021-04-22 03:40:22,678 >>   Num examples = 5\n",
            "[INFO|trainer.py:1993] 2021-04-22 03:40:22,678 >>   Batch size = 16\n",
            "100% 1/1 [00:00<00:00, 123.25it/s]\n",
            "[INFO|trainer_pt_utils.py:898] 2021-04-22 03:40:25,101 >> ***** eval metrics *****\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:40:25,101 >>   epoch                     =        3.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:40:25,101 >>   eval_accuracy             =        0.4\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:40:25,101 >>   eval_loss                 =     1.3856\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:40:25,101 >>   eval_mem_cpu_alloc_delta  =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:40:25,101 >>   eval_mem_cpu_peaked_delta =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:40:25,101 >>   eval_runtime              = 0:00:02.28\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:40:25,101 >>   eval_samples              =          5\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:40:25,101 >>   eval_samples_per_second   =      2.188\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zqMVrU4AeqKE"
      },
      "source": [
        "## question-answering"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w0O0easYhwTv",
        "outputId": "4e8c6a47-7cc5-4b37-a66e-19212c72342e"
      },
      "source": [
        "!python transformers/examples/pytorch/question-answering/run_qa.py \\\n",
        "--model_name_or_path distilbert-base-uncased \\\n",
        "--train_file transformers/tests/fixtures/tests_samples/SQUAD/sample.json \\\n",
        "--validation_file transformers/tests/fixtures/tests_samples/SQUAD/sample.json \\\n",
        "--test_file transformers/tests/fixtures/tests_samples/SQUAD/sample.json \\\n",
        "--do_train \\\n",
        "--do_eval \\\n",
        "--do_predict \\\n",
        "--max_train_samples 5 \\\n",
        "--max_val_samples 5 \\\n",
        "--max_test_samples 5 \\\n",
        "--learning_rate 3e-5 \\\n",
        "--max_seq_length 384 \\\n",
        "--doc_stride 128 \\\n",
        "--version_2_with_negative \\\n",
        "--output_dir /tmp/debug_squad/ \\\n",
        "--overwrite_output"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-22 03:42:04.484006: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "04/22/2021 03:42:06 - WARNING - __main__ -   Process rank: -1, device: cpu, n_gpu: 0distributed training: False, 16-bits training: False\n",
            "04/22/2021 03:42:06 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(output_dir=/tmp/debug_squad/, overwrite_output_dir=True, do_train=True, do_eval=True, do_predict=True, evaluation_strategy=IntervalStrategy.NO, prediction_loss_only=False, per_device_train_batch_size=8, per_device_eval_batch_size=8, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=3e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=3.0, max_steps=-1, lr_scheduler_type=SchedulerType.LINEAR, warmup_ratio=0.0, warmup_steps=0, logging_dir=runs/Apr22_03-42-06_a44dfd0e8363, logging_strategy=IntervalStrategy.STEPS, logging_first_step=False, logging_steps=500, save_strategy=IntervalStrategy.STEPS, save_steps=500, save_total_limit=None, no_cuda=False, seed=42, fp16=False, fp16_opt_level=O1, fp16_backend=auto, fp16_full_eval=False, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=500, dataloader_num_workers=0, past_index=-1, run_name=/tmp/debug_squad/, disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, length_column_name=length, report_to=['tensorboard'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, skip_memory_metrics=False, use_legacy_prediction_loop=False, _n_gpu=0, mp_parameters=)\n",
            "04/22/2021 03:42:06 - WARNING - datasets.builder -   Using custom data configuration default-1be269b19d662c81\n",
            "Downloading and preparing dataset json/default (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/json/default-1be269b19d662c81/0.0.0/83d5b3a2f62630efc6b5315f00f20209b4ad91a00ac586597caee3a4da0bef02...\n",
            "Dataset json downloaded and prepared to /root/.cache/huggingface/datasets/json/default-1be269b19d662c81/0.0.0/83d5b3a2f62630efc6b5315f00f20209b4ad91a00ac586597caee3a4da0bef02. Subsequent calls will reuse this data.\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:42:07,230 >> https://huggingface.co/distilbert-base-uncased/resolve/main/config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpdx6pwdjg\n",
            "Downloading: 100% 442/442 [00:00<00:00, 323kB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:42:07,332 >> storing https://huggingface.co/distilbert-base-uncased/resolve/main/config.json in cache at /root/.cache/huggingface/transformers/23454919702d26495337f3da04d1655c7ee010d5ec9d77bdb9e399e00302c0a1.d423bdf2f58dc8b77d5f5d18028d7ae4a72dcfd8f468e81fe979ada957a8c361\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:42:07,333 >> creating metadata file for /root/.cache/huggingface/transformers/23454919702d26495337f3da04d1655c7ee010d5ec9d77bdb9e399e00302c0a1.d423bdf2f58dc8b77d5f5d18028d7ae4a72dcfd8f468e81fe979ada957a8c361\n",
            "[INFO|configuration_utils.py:491] 2021-04-22 03:42:07,333 >> loading configuration file https://huggingface.co/distilbert-base-uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/23454919702d26495337f3da04d1655c7ee010d5ec9d77bdb9e399e00302c0a1.d423bdf2f58dc8b77d5f5d18028d7ae4a72dcfd8f468e81fe979ada957a8c361\n",
            "[INFO|configuration_utils.py:527] 2021-04-22 03:42:07,334 >> Model config DistilBertConfig {\n",
            "  \"activation\": \"gelu\",\n",
            "  \"architectures\": [\n",
            "    \"DistilBertForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_dropout\": 0.1,\n",
            "  \"dim\": 768,\n",
            "  \"dropout\": 0.1,\n",
            "  \"hidden_dim\": 3072,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"distilbert\",\n",
            "  \"n_heads\": 12,\n",
            "  \"n_layers\": 6,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"qa_dropout\": 0.1,\n",
            "  \"seq_classif_dropout\": 0.2,\n",
            "  \"sinusoidal_pos_embds\": false,\n",
            "  \"tie_weights_\": true,\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"vocab_size\": 30522\n",
            "}\n",
            "\n",
            "[INFO|configuration_utils.py:491] 2021-04-22 03:42:07,437 >> loading configuration file https://huggingface.co/distilbert-base-uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/23454919702d26495337f3da04d1655c7ee010d5ec9d77bdb9e399e00302c0a1.d423bdf2f58dc8b77d5f5d18028d7ae4a72dcfd8f468e81fe979ada957a8c361\n",
            "[INFO|configuration_utils.py:527] 2021-04-22 03:42:07,438 >> Model config DistilBertConfig {\n",
            "  \"activation\": \"gelu\",\n",
            "  \"architectures\": [\n",
            "    \"DistilBertForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_dropout\": 0.1,\n",
            "  \"dim\": 768,\n",
            "  \"dropout\": 0.1,\n",
            "  \"hidden_dim\": 3072,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"distilbert\",\n",
            "  \"n_heads\": 12,\n",
            "  \"n_layers\": 6,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"qa_dropout\": 0.1,\n",
            "  \"seq_classif_dropout\": 0.2,\n",
            "  \"sinusoidal_pos_embds\": false,\n",
            "  \"tie_weights_\": true,\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"vocab_size\": 30522\n",
            "}\n",
            "\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:42:07,540 >> https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpix8pmfca\n",
            "Downloading: 100% 232k/232k [00:00<00:00, 2.05MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:42:07,757 >> storing https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt in cache at /root/.cache/huggingface/transformers/0e1bbfda7f63a99bb52e3915dcf10c3c92122b827d92eb2d34ce94ee79ba486c.d789d64ebfe299b0e416afc4a169632f903f693095b4629a7ea271d5a0cf2c99\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:42:07,757 >> creating metadata file for /root/.cache/huggingface/transformers/0e1bbfda7f63a99bb52e3915dcf10c3c92122b827d92eb2d34ce94ee79ba486c.d789d64ebfe299b0e416afc4a169632f903f693095b4629a7ea271d5a0cf2c99\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:42:07,860 >> https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpbg2z9uuc\n",
            "Downloading: 100% 466k/466k [00:00<00:00, 3.35MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:42:08,106 >> storing https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json in cache at /root/.cache/huggingface/transformers/75abb59d7a06f4f640158a9bfcde005264e59e8d566781ab1415b139d2e4c603.7f2721073f19841be16f41b0a70b600ca6b880c8f3df6f3535cbc704371bdfa4\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:42:08,106 >> creating metadata file for /root/.cache/huggingface/transformers/75abb59d7a06f4f640158a9bfcde005264e59e8d566781ab1415b139d2e4c603.7f2721073f19841be16f41b0a70b600ca6b880c8f3df6f3535cbc704371bdfa4\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:42:08,423 >> https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer_config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpiw1w380p\n",
            "Downloading: 100% 28.0/28.0 [00:00<00:00, 9.60kB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:42:08,530 >> storing https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer_config.json in cache at /root/.cache/huggingface/transformers/8c8624b8ac8aa99c60c912161f8332de003484428c47906d7ff7eb7f73eecdbb.20430bd8e10ef77a7d2977accefe796051e01bc2fc4aa146bc862997a1a15e79\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:42:08,531 >> creating metadata file for /root/.cache/huggingface/transformers/8c8624b8ac8aa99c60c912161f8332de003484428c47906d7ff7eb7f73eecdbb.20430bd8e10ef77a7d2977accefe796051e01bc2fc4aa146bc862997a1a15e79\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:42:08,531 >> loading file https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt from cache at /root/.cache/huggingface/transformers/0e1bbfda7f63a99bb52e3915dcf10c3c92122b827d92eb2d34ce94ee79ba486c.d789d64ebfe299b0e416afc4a169632f903f693095b4629a7ea271d5a0cf2c99\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:42:08,531 >> loading file https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json from cache at /root/.cache/huggingface/transformers/75abb59d7a06f4f640158a9bfcde005264e59e8d566781ab1415b139d2e4c603.7f2721073f19841be16f41b0a70b600ca6b880c8f3df6f3535cbc704371bdfa4\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:42:08,531 >> loading file https://huggingface.co/distilbert-base-uncased/resolve/main/added_tokens.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:42:08,531 >> loading file https://huggingface.co/distilbert-base-uncased/resolve/main/special_tokens_map.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:42:08,531 >> loading file https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer_config.json from cache at /root/.cache/huggingface/transformers/8c8624b8ac8aa99c60c912161f8332de003484428c47906d7ff7eb7f73eecdbb.20430bd8e10ef77a7d2977accefe796051e01bc2fc4aa146bc862997a1a15e79\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:42:08,669 >> https://huggingface.co/distilbert-base-uncased/resolve/main/pytorch_model.bin not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpdq8i3qhm\n",
            "Downloading: 100% 268M/268M [00:18<00:00, 14.1MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:42:27,842 >> storing https://huggingface.co/distilbert-base-uncased/resolve/main/pytorch_model.bin in cache at /root/.cache/huggingface/transformers/9c169103d7e5a73936dd2b627e42851bec0831212b677c637033ee4bce9ab5ee.126183e36667471617ae2f0835fab707baa54b731f991507ebbb55ea85adb12a\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:42:27,843 >> creating metadata file for /root/.cache/huggingface/transformers/9c169103d7e5a73936dd2b627e42851bec0831212b677c637033ee4bce9ab5ee.126183e36667471617ae2f0835fab707baa54b731f991507ebbb55ea85adb12a\n",
            "[INFO|modeling_utils.py:1075] 2021-04-22 03:42:27,843 >> loading weights file https://huggingface.co/distilbert-base-uncased/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/9c169103d7e5a73936dd2b627e42851bec0831212b677c637033ee4bce9ab5ee.126183e36667471617ae2f0835fab707baa54b731f991507ebbb55ea85adb12a\n",
            "[WARNING|modeling_utils.py:1196] 2021-04-22 03:42:29,927 >> Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForQuestionAnswering: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",
            "- This IS expected if you are initializing DistilBertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing DistilBertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "[WARNING|modeling_utils.py:1207] 2021-04-22 03:42:29,927 >> Some weights of DistilBertForQuestionAnswering were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['qa_outputs.weight', 'qa_outputs.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "100% 1/1 [00:00<00:00, 140.07ba/s]\n",
            "100% 1/1 [00:00<00:00, 79.94ba/s]\n",
            "100% 1/1 [00:00<00:00, 82.65ba/s]\n",
            "Downloading: 6.49kB [00:00, 3.48MB/s]       \n",
            "Downloading: 11.3kB [00:00, 6.12MB/s]       \n",
            "[INFO|trainer.py:1107] 2021-04-22 03:42:31,193 >> ***** Running training *****\n",
            "[INFO|trainer.py:1108] 2021-04-22 03:42:31,194 >>   Num examples = 5\n",
            "[INFO|trainer.py:1109] 2021-04-22 03:42:31,194 >>   Num Epochs = 3\n",
            "[INFO|trainer.py:1110] 2021-04-22 03:42:31,194 >>   Instantaneous batch size per device = 8\n",
            "[INFO|trainer.py:1111] 2021-04-22 03:42:31,194 >>   Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "[INFO|trainer.py:1112] 2021-04-22 03:42:31,194 >>   Gradient Accumulation steps = 1\n",
            "[INFO|trainer.py:1113] 2021-04-22 03:42:31,194 >>   Total optimization steps = 3\n",
            "100% 3/3 [00:54<00:00, 18.30s/it][INFO|trainer.py:1302] 2021-04-22 03:43:25,772 >> \n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "{'train_runtime': 54.5786, 'train_samples_per_second': 0.055, 'epoch': 3.0}\n",
            "100% 3/3 [00:54<00:00, 18.19s/it]\n",
            "[INFO|trainer.py:1768] 2021-04-22 03:43:25,917 >> Saving model checkpoint to /tmp/debug_squad/\n",
            "[INFO|configuration_utils.py:329] 2021-04-22 03:43:25,918 >> Configuration saved in /tmp/debug_squad/config.json\n",
            "[INFO|modeling_utils.py:848] 2021-04-22 03:43:27,547 >> Model weights saved in /tmp/debug_squad/pytorch_model.bin\n",
            "[INFO|tokenization_utils_base.py:1918] 2021-04-22 03:43:27,547 >> tokenizer config file saved in /tmp/debug_squad/tokenizer_config.json\n",
            "[INFO|tokenization_utils_base.py:1924] 2021-04-22 03:43:27,548 >> Special tokens file saved in /tmp/debug_squad/special_tokens_map.json\n",
            "[INFO|trainer_pt_utils.py:898] 2021-04-22 03:43:27,617 >> ***** train metrics *****\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:27,617 >>   epoch                      =        3.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:27,618 >>   init_mem_cpu_alloc_delta   =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:27,618 >>   init_mem_cpu_peaked_delta  =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:27,618 >>   train_mem_cpu_alloc_delta  =     2012MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:27,618 >>   train_mem_cpu_peaked_delta =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:27,618 >>   train_runtime              = 0:00:54.57\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:27,618 >>   train_samples              =          5\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:27,618 >>   train_samples_per_second   =      0.055\n",
            "04/22/2021 03:43:27 - INFO - __main__ -   *** Evaluate ***\n",
            "[INFO|trainer.py:497] 2021-04-22 03:43:27,619 >> The following columns in the evaluation set  don't have a corresponding argument in `DistilBertForQuestionAnswering.forward` and have been ignored: offset_mapping, example_id.\n",
            "[INFO|trainer.py:2324] 2021-04-22 03:43:27,622 >> ***** Running Evaluation *****\n",
            "[INFO|trainer.py:2325] 2021-04-22 03:43:27,622 >>   Num examples = 5\n",
            "[INFO|trainer.py:2326] 2021-04-22 03:43:27,622 >>   Batch size = 8\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/trainer_pt_utils.py:367: FutureWarning: DistributedTensorGatherer is deprecated and will be removed in v5 of Tranformers.\n",
            "  FutureWarning,\n",
            "  0% 0/1 [00:00<?, ?it/s]04/22/2021 03:43:30 - INFO - utils_qa -   Post-processing 5 example predictions split into 5 features.\n",
            "\n",
            "100% 5/5 [00:00<00:00, 405.85it/s]\n",
            "04/22/2021 03:43:30 - INFO - utils_qa -   Saving predictions to /tmp/debug_squad/eval_predictions.json.\n",
            "04/22/2021 03:43:30 - INFO - utils_qa -   Saving nbest_preds to /tmp/debug_squad/eval_nbest_predictions.json.\n",
            "04/22/2021 03:43:30 - INFO - utils_qa -   Saving null_odds to /tmp/debug_squad/eval_null_odds.json.\n",
            "100% 1/1 [00:00<00:00, 10.63it/s]\n",
            "[INFO|trainer_pt_utils.py:898] 2021-04-22 03:43:30,880 >> ***** eval metrics *****\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:30,880 >>   HasAns_exact      = 33.3333\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:30,880 >>   HasAns_f1         = 41.6667\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:30,880 >>   HasAns_total      =       3\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:30,880 >>   NoAns_exact       =     0.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:30,880 >>   NoAns_f1          =     0.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:30,880 >>   NoAns_total       =       2\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:30,881 >>   best_exact        =    60.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:30,881 >>   best_exact_thresh =     0.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:30,881 >>   best_f1           =    65.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:30,881 >>   best_f1_thresh    =     0.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:30,881 >>   epoch             =     3.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:30,881 >>   eval_samples      =       5\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:30,881 >>   exact             =    20.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:30,881 >>   f1                =    25.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:30,881 >>   total             =       5\n",
            "04/22/2021 03:43:30 - INFO - __main__ -   *** Predict ***\n",
            "[INFO|trainer.py:497] 2021-04-22 03:43:30,882 >> The following columns in the test set  don't have a corresponding argument in `DistilBertForQuestionAnswering.forward` and have been ignored: offset_mapping, example_id.\n",
            "[INFO|trainer.py:2324] 2021-04-22 03:43:30,884 >> ***** Running Evaluation *****\n",
            "[INFO|trainer.py:2325] 2021-04-22 03:43:30,885 >>   Num examples = 5\n",
            "[INFO|trainer.py:2326] 2021-04-22 03:43:30,885 >>   Batch size = 8\n",
            "  0% 0/1 [00:00<?, ?it/s]04/22/2021 03:43:34 - INFO - utils_qa -   Post-processing 5 example predictions split into 5 features.\n",
            "\n",
            "100% 5/5 [00:00<00:00, 435.84it/s]\n",
            "04/22/2021 03:43:34 - INFO - utils_qa -   Saving predictions to /tmp/debug_squad/test_predictions.json.\n",
            "04/22/2021 03:43:34 - INFO - utils_qa -   Saving nbest_preds to /tmp/debug_squad/test_nbest_predictions.json.\n",
            "04/22/2021 03:43:34 - INFO - utils_qa -   Saving null_odds to /tmp/debug_squad/test_null_odds.json.\n",
            "[INFO|trainer_pt_utils.py:898] 2021-04-22 03:43:34,088 >> ***** test metrics *****\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:34,088 >>   HasAns_exact      = 33.3333\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:34,088 >>   HasAns_f1         = 41.6667\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:34,088 >>   HasAns_total      =       3\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:34,089 >>   NoAns_exact       =     0.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:34,089 >>   NoAns_f1          =     0.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:34,089 >>   NoAns_total       =       2\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:34,089 >>   best_exact        =    60.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:34,089 >>   best_exact_thresh =     0.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:34,089 >>   best_f1           =    65.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:34,089 >>   best_f1_thresh    =     0.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:34,089 >>   exact             =    20.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:34,089 >>   f1                =    25.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:34,089 >>   test_samples      =       5\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 03:43:34,089 >>   total             =       5\n",
            "100% 1/1 [00:00<00:00,  6.32it/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J7I5mXl4kKl7",
        "outputId": "3b4287e8-9691-45a5-9a1a-f69254d30a96"
      },
      "source": [
        "!python transformers/examples/pytorch/question-answering/run_qa_beam_search.py \\\n",
        "--model_name_or_path xlnet-large-cased \\\n",
        "--train_file transformers/tests/fixtures/tests_samples/SQUAD/sample.json \\\n",
        "--validation_file transformers/tests/fixtures/tests_samples/SQUAD/sample.json \\\n",
        "--test_file transformers/tests/fixtures/tests_samples/SQUAD/sample.json \\\n",
        "--do_train \\\n",
        "--do_eval \\\n",
        "--do_predict \\\n",
        "--max_train_samples 5 \\\n",
        "--max_val_samples 5 \\\n",
        "--max_test_samples 5 \\\n",
        "--learning_rate 3e-5 \\\n",
        "--max_seq_length 384 \\\n",
        "--doc_stride 128 \\\n",
        "--version_2_with_negative \\\n",
        "--output_dir /tmp/debug_squad/ \\\n",
        "--overwrite_output"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-22 03:48:35.281391: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "04/22/2021 03:48:37 - WARNING - __main__ -   Process rank: -1, device: cpu, n_gpu: 0distributed training: False, 16-bits training: False\n",
            "04/22/2021 03:48:37 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(output_dir=/tmp/debug_squad/, overwrite_output_dir=True, do_train=True, do_eval=True, do_predict=True, evaluation_strategy=IntervalStrategy.NO, prediction_loss_only=False, per_device_train_batch_size=8, per_device_eval_batch_size=8, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=3e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=3.0, max_steps=-1, lr_scheduler_type=SchedulerType.LINEAR, warmup_ratio=0.0, warmup_steps=0, logging_dir=runs/Apr22_03-48-37_a44dfd0e8363, logging_strategy=IntervalStrategy.STEPS, logging_first_step=False, logging_steps=500, save_strategy=IntervalStrategy.STEPS, save_steps=500, save_total_limit=None, no_cuda=False, seed=42, fp16=False, fp16_opt_level=O1, fp16_backend=auto, fp16_full_eval=False, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=500, dataloader_num_workers=0, past_index=-1, run_name=/tmp/debug_squad/, disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, length_column_name=length, report_to=['tensorboard'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, skip_memory_metrics=False, use_legacy_prediction_loop=False, _n_gpu=0, mp_parameters=)\n",
            "04/22/2021 03:48:37 - WARNING - datasets.builder -   Using custom data configuration default-1be269b19d662c81\n",
            "04/22/2021 03:48:37 - WARNING - datasets.builder -   Reusing dataset json (/root/.cache/huggingface/datasets/json/default-1be269b19d662c81/0.0.0/83d5b3a2f62630efc6b5315f00f20209b4ad91a00ac586597caee3a4da0bef02)\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:48:37,567 >> https://huggingface.co/xlnet-large-cased/resolve/main/config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpc_moa42o\n",
            "Downloading: 100% 761/761 [00:00<00:00, 496kB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:48:37,673 >> storing https://huggingface.co/xlnet-large-cased/resolve/main/config.json in cache at /root/.cache/huggingface/transformers/1f0d5fc4143aa8fe332810bac98d442fed5483549adcd9656e5709cd470003a0.a0945cddd1ef8f9d9c40c35c36bad4908625533057baeeafc6d26a9550f18c60\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:48:37,673 >> creating metadata file for /root/.cache/huggingface/transformers/1f0d5fc4143aa8fe332810bac98d442fed5483549adcd9656e5709cd470003a0.a0945cddd1ef8f9d9c40c35c36bad4908625533057baeeafc6d26a9550f18c60\n",
            "[INFO|configuration_utils.py:491] 2021-04-22 03:48:37,673 >> loading configuration file https://huggingface.co/xlnet-large-cased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/1f0d5fc4143aa8fe332810bac98d442fed5483549adcd9656e5709cd470003a0.a0945cddd1ef8f9d9c40c35c36bad4908625533057baeeafc6d26a9550f18c60\n",
            "[INFO|configuration_utils.py:527] 2021-04-22 03:48:37,674 >> Model config XLNetConfig {\n",
            "  \"architectures\": [\n",
            "    \"XLNetLMHeadModel\"\n",
            "  ],\n",
            "  \"attn_type\": \"bi\",\n",
            "  \"bi_data\": false,\n",
            "  \"bos_token_id\": 1,\n",
            "  \"clamp_len\": -1,\n",
            "  \"d_head\": 64,\n",
            "  \"d_inner\": 4096,\n",
            "  \"d_model\": 1024,\n",
            "  \"dropout\": 0.1,\n",
            "  \"end_n_top\": 5,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"ff_activation\": \"gelu\",\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"mem_len\": null,\n",
            "  \"model_type\": \"xlnet\",\n",
            "  \"n_head\": 16,\n",
            "  \"n_layer\": 24,\n",
            "  \"pad_token_id\": 5,\n",
            "  \"reuse_len\": null,\n",
            "  \"same_length\": false,\n",
            "  \"start_n_top\": 5,\n",
            "  \"summary_activation\": \"tanh\",\n",
            "  \"summary_last_dropout\": 0.1,\n",
            "  \"summary_type\": \"last\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 250\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"untie_r\": true,\n",
            "  \"use_mems_eval\": true,\n",
            "  \"use_mems_train\": false,\n",
            "  \"vocab_size\": 32000\n",
            "}\n",
            "\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:48:37,776 >> https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmphdj5utq2\n",
            "Downloading: 100% 798k/798k [00:00<00:00, 5.31MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:48:38,043 >> storing https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model in cache at /root/.cache/huggingface/transformers/3af982b422f8bb8c510fdd1112afe6f5ec3f3219ef859edcf4c3826bec14832e.d93497120e3a865e2970f26abdf7bf375896f97fde8b874b70909592a6c785c9\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:48:38,043 >> creating metadata file for /root/.cache/huggingface/transformers/3af982b422f8bb8c510fdd1112afe6f5ec3f3219ef859edcf4c3826bec14832e.d93497120e3a865e2970f26abdf7bf375896f97fde8b874b70909592a6c785c9\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:48:38,146 >> https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpk94j24q6\n",
            "Downloading: 100% 1.38M/1.38M [00:00<00:00, 7.44MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:48:38,446 >> storing https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json in cache at /root/.cache/huggingface/transformers/6a4afd4829edeea0c7fe7735eccea233e66e79729e574966cfd9ec47f81d269a.2a683f915238b4f560dab0c724066cf0a7de9a851e96b0fb3a1e7f0881552f53\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:48:38,447 >> creating metadata file for /root/.cache/huggingface/transformers/6a4afd4829edeea0c7fe7735eccea233e66e79729e574966cfd9ec47f81d269a.2a683f915238b4f560dab0c724066cf0a7de9a851e96b0fb3a1e7f0881552f53\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:48:38,756 >> loading file https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model from cache at /root/.cache/huggingface/transformers/3af982b422f8bb8c510fdd1112afe6f5ec3f3219ef859edcf4c3826bec14832e.d93497120e3a865e2970f26abdf7bf375896f97fde8b874b70909592a6c785c9\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:48:38,756 >> loading file https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json from cache at /root/.cache/huggingface/transformers/6a4afd4829edeea0c7fe7735eccea233e66e79729e574966cfd9ec47f81d269a.2a683f915238b4f560dab0c724066cf0a7de9a851e96b0fb3a1e7f0881552f53\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:48:38,756 >> loading file https://huggingface.co/xlnet-large-cased/resolve/main/added_tokens.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:48:38,756 >> loading file https://huggingface.co/xlnet-large-cased/resolve/main/special_tokens_map.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 03:48:38,756 >> loading file https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer_config.json from cache at None\n",
            "[INFO|file_utils.py:1395] 2021-04-22 03:48:38,934 >> https://huggingface.co/xlnet-large-cased/resolve/main/pytorch_model.bin not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp5yjakn3b\n",
            "Downloading: 100% 1.44G/1.44G [00:26<00:00, 54.2MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 03:49:05,775 >> storing https://huggingface.co/xlnet-large-cased/resolve/main/pytorch_model.bin in cache at /root/.cache/huggingface/transformers/0c2b00a768ca7c5b3534b75606a47a7e1125b10ce354b217022de5a12029859c.7fff7afe180c24f31dabdb196f95ca2e26a8aa357c1db6137f4fec6430db9776\n",
            "[INFO|file_utils.py:1402] 2021-04-22 03:49:05,775 >> creating metadata file for /root/.cache/huggingface/transformers/0c2b00a768ca7c5b3534b75606a47a7e1125b10ce354b217022de5a12029859c.7fff7afe180c24f31dabdb196f95ca2e26a8aa357c1db6137f4fec6430db9776\n",
            "[INFO|modeling_utils.py:1075] 2021-04-22 03:49:05,776 >> loading weights file https://huggingface.co/xlnet-large-cased/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/0c2b00a768ca7c5b3534b75606a47a7e1125b10ce354b217022de5a12029859c.7fff7afe180c24f31dabdb196f95ca2e26a8aa357c1db6137f4fec6430db9776\n",
            "[WARNING|modeling_utils.py:1196] 2021-04-22 03:49:21,805 >> Some weights of the model checkpoint at xlnet-large-cased were not used when initializing XLNetForQuestionAnswering: ['lm_loss.weight', 'lm_loss.bias']\n",
            "- This IS expected if you are initializing XLNetForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing XLNetForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "[WARNING|modeling_utils.py:1207] 2021-04-22 03:49:21,807 >> Some weights of XLNetForQuestionAnswering were not initialized from the model checkpoint at xlnet-large-cased and are newly initialized: ['start_logits.dense.weight', 'start_logits.dense.bias', 'end_logits.dense_0.weight', 'end_logits.dense_0.bias', 'end_logits.LayerNorm.weight', 'end_logits.LayerNorm.bias', 'end_logits.dense_1.weight', 'end_logits.dense_1.bias', 'answer_class.dense_0.weight', 'answer_class.dense_0.bias', 'answer_class.dense_1.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "100% 1/1 [00:00<00:00, 30.65ba/s]\n",
            "100% 1/1 [00:00<00:00, 33.86ba/s]\n",
            "100% 1/1 [00:00<00:00, 45.47ba/s]\n",
            "[INFO|trainer.py:1107] 2021-04-22 03:49:24,162 >> ***** Running training *****\n",
            "[INFO|trainer.py:1108] 2021-04-22 03:49:24,163 >>   Num examples = 5\n",
            "[INFO|trainer.py:1109] 2021-04-22 03:49:24,163 >>   Num Epochs = 3\n",
            "[INFO|trainer.py:1110] 2021-04-22 03:49:24,163 >>   Instantaneous batch size per device = 8\n",
            "[INFO|trainer.py:1111] 2021-04-22 03:49:24,163 >>   Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "[INFO|trainer.py:1112] 2021-04-22 03:49:24,163 >>   Gradient Accumulation steps = 1\n",
            "[INFO|trainer.py:1113] 2021-04-22 03:49:24,163 >>   Total optimization steps = 3\n",
            " 33% 1/3 [02:51<05:43, 171.95s/it]^C\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CkdfShyWer51"
      },
      "source": [
        "## summarization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QOPnWbhQk5T5",
        "outputId": "8ca21221-bb58-4765-cb9b-364355ce1add"
      },
      "source": [
        "!python transformers/examples/pytorch/summarization/run_summarization.py \\\n",
        "--model_name_or_path t5-small \\\n",
        "--do_train \\\n",
        "--do_eval \\\n",
        "--do_predict \\\n",
        "--max_train_samples 5 \\\n",
        "--max_val_samples 5 \\\n",
        "--max_test_samples 5 \\\n",
        "--dataset_name cnn_dailymail \\\n",
        "--dataset_config \"3.0.0\" \\\n",
        "--source_prefix \"summarize: \" \\\n",
        "--output_dir /tmp/tst-summarization \\\n",
        "--overwrite_output_dir \\\n",
        "--predict_with_generate"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-22 03:53:33.019055: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "04/22/2021 03:53:38 - WARNING - __main__ -   Process rank: -1, device: cpu, n_gpu: 0distributed training: False, 16-bits training: False\n",
            "04/22/2021 03:53:38 - INFO - __main__ -   Training/evaluation parameters Seq2SeqTrainingArguments(output_dir='/tmp/tst-summarization', overwrite_output_dir=True, do_train=True, do_eval=True, do_predict=True, evaluation_strategy=<IntervalStrategy.NO: 'no'>, prediction_loss_only=False, per_device_train_batch_size=8, per_device_eval_batch_size=8, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=5e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=3.0, max_steps=-1, lr_scheduler_type=<SchedulerType.LINEAR: 'linear'>, warmup_ratio=0.0, warmup_steps=0, logging_dir='runs/Apr22_03-53-38_a44dfd0e8363', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=False, logging_steps=500, save_strategy=<IntervalStrategy.STEPS: 'steps'>, save_steps=500, save_total_limit=None, no_cuda=False, seed=42, fp16=False, fp16_opt_level='O1', fp16_backend='auto', fp16_full_eval=False, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=500, dataloader_num_workers=0, past_index=-1, run_name='/tmp/tst-summarization', disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, skip_memory_metrics=False, use_legacy_prediction_loop=False, mp_parameters='', sortish_sampler=False, predict_with_generate=True)\n",
            "Downloading: 9.35kB [00:00, 5.54MB/s]       \n",
            "Downloading: 9.50kB [00:00, 6.21MB/s]       \n",
            "Downloading and preparing dataset cnn_dailymail/3.0.0 (download: 558.32 MiB, generated: 1.28 GiB, post-processed: Unknown size, total: 1.82 GiB) to /root/.cache/huggingface/datasets/cnn_dailymail/3.0.0/3.0.0/3cb851bf7cf5826e45d49db2863f627cba583cbc32342df7349dfe6c38060234...\n",
            "Downloading: 159MB [00:01, 105MB/s]\n",
            "Downloading: 376MB [00:04, 84.4MB/s]\n",
            "Downloading: 2.11MB [00:00, 27.0MB/s]      \n",
            "Downloading: 46.4MB [00:00, 65.2MB/s]\n",
            "Downloading: 2.43MB [00:00, 25.6MB/s]      \n",
            "Dataset cnn_dailymail downloaded and prepared to /root/.cache/huggingface/datasets/cnn_dailymail/3.0.0/3.0.0/3cb851bf7cf5826e45d49db2863f627cba583cbc32342df7349dfe6c38060234. Subsequent calls will reuse this data.\n",
            "https://huggingface.co/t5-small/resolve/main/config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp56m974um\n",
            "Downloading: 100% 1.20k/1.20k [00:00<00:00, 676kB/s]\n",
            "storing https://huggingface.co/t5-small/resolve/main/config.json in cache at /root/.cache/huggingface/transformers/fe501e8fd6425b8ec93df37767fcce78ce626e34cc5edc859c662350cf712e41.406701565c0afd9899544c1cb8b93185a76f00b31e5ce7f6e18bbaef02241985\n",
            "creating metadata file for /root/.cache/huggingface/transformers/fe501e8fd6425b8ec93df37767fcce78ce626e34cc5edc859c662350cf712e41.406701565c0afd9899544c1cb8b93185a76f00b31e5ce7f6e18bbaef02241985\n",
            "loading configuration file https://huggingface.co/t5-small/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/fe501e8fd6425b8ec93df37767fcce78ce626e34cc5edc859c662350cf712e41.406701565c0afd9899544c1cb8b93185a76f00b31e5ce7f6e18bbaef02241985\n",
            "Model config T5Config {\n",
            "  \"architectures\": [\n",
            "    \"T5WithLMHeadModel\"\n",
            "  ],\n",
            "  \"d_ff\": 2048,\n",
            "  \"d_kv\": 64,\n",
            "  \"d_model\": 512,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"dropout_rate\": 0.1,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"feed_forward_proj\": \"relu\",\n",
            "  \"initializer_factor\": 1.0,\n",
            "  \"is_encoder_decoder\": true,\n",
            "  \"layer_norm_epsilon\": 1e-06,\n",
            "  \"model_type\": \"t5\",\n",
            "  \"n_positions\": 512,\n",
            "  \"num_decoder_layers\": 6,\n",
            "  \"num_heads\": 8,\n",
            "  \"num_layers\": 6,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"relative_attention_num_buckets\": 32,\n",
            "  \"task_specific_params\": {\n",
            "    \"summarization\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"length_penalty\": 2.0,\n",
            "      \"max_length\": 200,\n",
            "      \"min_length\": 30,\n",
            "      \"no_repeat_ngram_size\": 3,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"summarize: \"\n",
            "    },\n",
            "    \"translation_en_to_de\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to German: \"\n",
            "    },\n",
            "    \"translation_en_to_fr\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to French: \"\n",
            "    },\n",
            "    \"translation_en_to_ro\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to Romanian: \"\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 32128\n",
            "}\n",
            "\n",
            "loading configuration file https://huggingface.co/t5-small/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/fe501e8fd6425b8ec93df37767fcce78ce626e34cc5edc859c662350cf712e41.406701565c0afd9899544c1cb8b93185a76f00b31e5ce7f6e18bbaef02241985\n",
            "Model config T5Config {\n",
            "  \"architectures\": [\n",
            "    \"T5WithLMHeadModel\"\n",
            "  ],\n",
            "  \"d_ff\": 2048,\n",
            "  \"d_kv\": 64,\n",
            "  \"d_model\": 512,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"dropout_rate\": 0.1,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"feed_forward_proj\": \"relu\",\n",
            "  \"initializer_factor\": 1.0,\n",
            "  \"is_encoder_decoder\": true,\n",
            "  \"layer_norm_epsilon\": 1e-06,\n",
            "  \"model_type\": \"t5\",\n",
            "  \"n_positions\": 512,\n",
            "  \"num_decoder_layers\": 6,\n",
            "  \"num_heads\": 8,\n",
            "  \"num_layers\": 6,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"relative_attention_num_buckets\": 32,\n",
            "  \"task_specific_params\": {\n",
            "    \"summarization\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"length_penalty\": 2.0,\n",
            "      \"max_length\": 200,\n",
            "      \"min_length\": 30,\n",
            "      \"no_repeat_ngram_size\": 3,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"summarize: \"\n",
            "    },\n",
            "    \"translation_en_to_de\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to German: \"\n",
            "    },\n",
            "    \"translation_en_to_fr\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to French: \"\n",
            "    },\n",
            "    \"translation_en_to_ro\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to Romanian: \"\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 32128\n",
            "}\n",
            "\n",
            "https://huggingface.co/t5-small/resolve/main/spiece.model not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp0k49sokq\n",
            "Downloading: 100% 792k/792k [00:00<00:00, 5.12MB/s]\n",
            "storing https://huggingface.co/t5-small/resolve/main/spiece.model in cache at /root/.cache/huggingface/transformers/65fc04e21f45f61430aea0c4fedffac16a4d20d78b8e6601d8d996ebefefecd2.3b69006860e7b5d0a63ffdddc01ddcd6b7c318a6f4fd793596552c741734c62d\n",
            "creating metadata file for /root/.cache/huggingface/transformers/65fc04e21f45f61430aea0c4fedffac16a4d20d78b8e6601d8d996ebefefecd2.3b69006860e7b5d0a63ffdddc01ddcd6b7c318a6f4fd793596552c741734c62d\n",
            "https://huggingface.co/t5-small/resolve/main/tokenizer.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpfbq65ndb\n",
            "Downloading: 100% 1.39M/1.39M [00:00<00:00, 7.70MB/s]\n",
            "storing https://huggingface.co/t5-small/resolve/main/tokenizer.json in cache at /root/.cache/huggingface/transformers/06779097c78e12f47ef67ecb728810c2ae757ee0a9efe9390c6419783d99382d.8627f1bd5d270a9fd2e5a51c8bec3223896587cc3cfe13edeabb0992ab43c529\n",
            "creating metadata file for /root/.cache/huggingface/transformers/06779097c78e12f47ef67ecb728810c2ae757ee0a9efe9390c6419783d99382d.8627f1bd5d270a9fd2e5a51c8bec3223896587cc3cfe13edeabb0992ab43c529\n",
            "loading file https://huggingface.co/t5-small/resolve/main/spiece.model from cache at /root/.cache/huggingface/transformers/65fc04e21f45f61430aea0c4fedffac16a4d20d78b8e6601d8d996ebefefecd2.3b69006860e7b5d0a63ffdddc01ddcd6b7c318a6f4fd793596552c741734c62d\n",
            "loading file https://huggingface.co/t5-small/resolve/main/tokenizer.json from cache at /root/.cache/huggingface/transformers/06779097c78e12f47ef67ecb728810c2ae757ee0a9efe9390c6419783d99382d.8627f1bd5d270a9fd2e5a51c8bec3223896587cc3cfe13edeabb0992ab43c529\n",
            "loading file https://huggingface.co/t5-small/resolve/main/added_tokens.json from cache at None\n",
            "loading file https://huggingface.co/t5-small/resolve/main/special_tokens_map.json from cache at None\n",
            "loading file https://huggingface.co/t5-small/resolve/main/tokenizer_config.json from cache at None\n",
            "https://huggingface.co/t5-small/resolve/main/pytorch_model.bin not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpg3fo2mpg\n",
            "Downloading: 100% 242M/242M [00:04<00:00, 52.9MB/s]\n",
            "storing https://huggingface.co/t5-small/resolve/main/pytorch_model.bin in cache at /root/.cache/huggingface/transformers/fee5a3a0ae379232608b6eed45d2d7a0d2966b9683728838412caccc41b4b0ed.ddacdc89ec88482db20c676f0861a336f3d0409f94748c209847b49529d73885\n",
            "creating metadata file for /root/.cache/huggingface/transformers/fee5a3a0ae379232608b6eed45d2d7a0d2966b9683728838412caccc41b4b0ed.ddacdc89ec88482db20c676f0861a336f3d0409f94748c209847b49529d73885\n",
            "loading weights file https://huggingface.co/t5-small/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/fee5a3a0ae379232608b6eed45d2d7a0d2966b9683728838412caccc41b4b0ed.ddacdc89ec88482db20c676f0861a336f3d0409f94748c209847b49529d73885\n",
            "All model checkpoint weights were used when initializing T5ForConditionalGeneration.\n",
            "\n",
            "All the weights of T5ForConditionalGeneration were initialized from the model checkpoint at t5-small.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use T5ForConditionalGeneration for predictions without further training.\n",
            "100% 1/1 [00:00<00:00, 36.46ba/s]\n",
            "100% 1/1 [00:00<00:00, 58.91ba/s]\n",
            "100% 1/1 [00:00<00:00, 53.59ba/s]\n",
            "Downloading: 5.61kB [00:00, 3.21MB/s]       \n",
            "***** Running training *****\n",
            "  Num examples = 5\n",
            "  Num Epochs = 3\n",
            "  Instantaneous batch size per device = 8\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3\n",
            "100% 3/3 [02:09<00:00, 43.76s/it]\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "{'train_runtime': 129.0523, 'train_samples_per_second': 0.023, 'epoch': 3.0}\n",
            "100% 3/3 [02:09<00:00, 43.02s/it]\n",
            "Saving model checkpoint to /tmp/tst-summarization\n",
            "Configuration saved in /tmp/tst-summarization/config.json\n",
            "Model weights saved in /tmp/tst-summarization/pytorch_model.bin\n",
            "tokenizer config file saved in /tmp/tst-summarization/tokenizer_config.json\n",
            "Special tokens file saved in /tmp/tst-summarization/special_tokens_map.json\n",
            "Copy vocab file to /tmp/tst-summarization/spiece.model\n",
            "***** train metrics *****\n",
            "  epoch                      =        3.0\n",
            "  init_mem_cpu_alloc_delta   =        0MB\n",
            "  init_mem_cpu_peaked_delta  =        0MB\n",
            "  train_mem_cpu_alloc_delta  =     6802MB\n",
            "  train_mem_cpu_peaked_delta =        0MB\n",
            "  train_runtime              = 0:02:09.05\n",
            "  train_samples              =          5\n",
            "  train_samples_per_second   =      0.023\n",
            "04/22/2021 03:58:43 - INFO - __main__ -   *** Evaluate ***\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 5\n",
            "  Batch size = 8\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/generation_utils.py:965: UserWarning: `max_length` is deprecated in this function, use `stopping_criteria=StoppingCriteriaList(MaxLengthCriteria(max_length=max_length))` instead.\n",
            "  UserWarning,\n",
            "100% 1/1 [00:00<00:00,  1.96it/s]\n",
            "***** eval metrics *****\n",
            "  epoch                     =        3.0\n",
            "  eval_gen_len              =       60.6\n",
            "  eval_loss                 =     2.5957\n",
            "  eval_mem_cpu_alloc_delta  =       20MB\n",
            "  eval_mem_cpu_peaked_delta =        0MB\n",
            "  eval_rouge1               =    22.6667\n",
            "  eval_rouge2               =     9.0685\n",
            "  eval_rougeL               =    20.1569\n",
            "  eval_rougeLsum            =       22.0\n",
            "  eval_runtime              = 0:00:43.68\n",
            "  eval_samples              =          5\n",
            "  eval_samples_per_second   =      0.114\n",
            "04/22/2021 03:59:27 - INFO - __main__ -   *** Test ***\n",
            "***** Running Prediction *****\n",
            "  Num examples = 5\n",
            "  Batch size = 8\n",
            "  0% 0/1 [00:00<?, ?it/s]***** test metrics *****\n",
            "  test_gen_len              =       72.2\n",
            "  test_loss                 =     2.2007\n",
            "  test_mem_cpu_alloc_delta  =        1MB\n",
            "  test_mem_cpu_peaked_delta =        0MB\n",
            "  test_rouge1               =    26.3319\n",
            "  test_rouge2               =     8.1449\n",
            "  test_rougeL               =    17.8706\n",
            "  test_rougeLsum            =    23.0485\n",
            "  test_runtime              = 0:00:47.77\n",
            "  test_samples              =          5\n",
            "  test_samples_per_second   =      0.105\n",
            "100% 1/1 [00:00<00:00,  1.58it/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sMuDiD48etuW"
      },
      "source": [
        "## text-classification"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1MEiv6RAocUH",
        "outputId": "9db8c274-e7f6-4cfe-8843-cc0dce2b9d22"
      },
      "source": [
        "!python transformers/examples/pytorch/text-classification/run_glue.py \\\n",
        "--model_name_or_path distilbert-base-cased \\\n",
        "--task_name mrpc \\\n",
        "--do_train \\\n",
        "--do_eval \\\n",
        "--do_predict \\\n",
        "--max_train_samples 5 \\\n",
        "--max_val_samples 5 \\\n",
        "--max_test_samples 5 \\\n",
        "--max_seq_length 128 \\\n",
        "--learning_rate 2e-5 \\\n",
        "--output_dir /tmp/mrpc/"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-22 04:05:51.595960: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "04/22/2021 04:05:56 - WARNING - __main__ -   Process rank: -1, device: cpu, n_gpu: 0distributed training: False, 16-bits training: False\n",
            "04/22/2021 04:05:56 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(output_dir=/tmp/mrpc/, overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=True, evaluation_strategy=IntervalStrategy.NO, prediction_loss_only=False, per_device_train_batch_size=8, per_device_eval_batch_size=8, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=2e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=3.0, max_steps=-1, lr_scheduler_type=SchedulerType.LINEAR, warmup_ratio=0.0, warmup_steps=0, logging_dir=runs/Apr22_04-05-56_a44dfd0e8363, logging_strategy=IntervalStrategy.STEPS, logging_first_step=False, logging_steps=500, save_strategy=IntervalStrategy.STEPS, save_steps=500, save_total_limit=None, no_cuda=False, seed=42, fp16=False, fp16_opt_level=O1, fp16_backend=auto, fp16_full_eval=False, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=500, dataloader_num_workers=0, past_index=-1, run_name=/tmp/mrpc/, disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, length_column_name=length, report_to=['tensorboard'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, skip_memory_metrics=False, use_legacy_prediction_loop=False, _n_gpu=0, mp_parameters=)\n",
            "Downloading: 28.8kB [00:00, 12.6MB/s]       \n",
            "Downloading: 28.7kB [00:00, 15.5MB/s]       \n",
            "Downloading and preparing dataset glue/mrpc (download: 1.43 MiB, generated: 1.43 MiB, post-processed: Unknown size, total: 2.85 MiB) to /root/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad...\n",
            "Downloading: 6.22kB [00:00, 1.37MB/s]\n",
            "Downloading: 1.05MB [00:00, 3.45MB/s]\n",
            "Downloading: 441kB [00:00, 1.87MB/s]\n",
            "Dataset glue downloaded and prepared to /root/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad. Subsequent calls will reuse this data.\n",
            "[INFO|configuration_utils.py:491] 2021-04-22 04:06:00,985 >> loading configuration file https://huggingface.co/distilbert-base-cased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/ebe1ea24d11aa664488b8de5b21e33989008ca78f207d4e30ec6350b693f073f.302bfd1b5e031cc1b17796e0b6e5b242ba2045d31d00f97589e12b458ebff27a\n",
            "[INFO|configuration_utils.py:527] 2021-04-22 04:06:00,986 >> Model config DistilBertConfig {\n",
            "  \"activation\": \"gelu\",\n",
            "  \"attention_dropout\": 0.1,\n",
            "  \"dim\": 768,\n",
            "  \"dropout\": 0.1,\n",
            "  \"finetuning_task\": \"mrpc\",\n",
            "  \"hidden_dim\": 3072,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"distilbert\",\n",
            "  \"n_heads\": 12,\n",
            "  \"n_layers\": 6,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"qa_dropout\": 0.1,\n",
            "  \"seq_classif_dropout\": 0.2,\n",
            "  \"sinusoidal_pos_embds\": false,\n",
            "  \"tie_weights_\": true,\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"vocab_size\": 28996\n",
            "}\n",
            "\n",
            "[INFO|configuration_utils.py:491] 2021-04-22 04:06:01,091 >> loading configuration file https://huggingface.co/distilbert-base-cased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/ebe1ea24d11aa664488b8de5b21e33989008ca78f207d4e30ec6350b693f073f.302bfd1b5e031cc1b17796e0b6e5b242ba2045d31d00f97589e12b458ebff27a\n",
            "[INFO|configuration_utils.py:527] 2021-04-22 04:06:01,092 >> Model config DistilBertConfig {\n",
            "  \"activation\": \"gelu\",\n",
            "  \"attention_dropout\": 0.1,\n",
            "  \"dim\": 768,\n",
            "  \"dropout\": 0.1,\n",
            "  \"hidden_dim\": 3072,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"distilbert\",\n",
            "  \"n_heads\": 12,\n",
            "  \"n_layers\": 6,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"qa_dropout\": 0.1,\n",
            "  \"seq_classif_dropout\": 0.2,\n",
            "  \"sinusoidal_pos_embds\": false,\n",
            "  \"tie_weights_\": true,\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"vocab_size\": 28996\n",
            "}\n",
            "\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 04:06:01,615 >> loading file https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt from cache at /root/.cache/huggingface/transformers/ba377304984dc63e3ede0e23a938bbbf04d5c3835b66d5bb48343aecca188429.437aa611e89f6fc6675a049d2b5545390adbc617e7d655286421c191d2be2791\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 04:06:01,615 >> loading file https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json from cache at /root/.cache/huggingface/transformers/acb5c2138c1f8c84f074b86dafce3631667fccd6efcb1a7ea1320cf75c386a36.3dab63143af66769bbb35e3811f75f7e16b2320e12b7935e216bd6159ce6d9a6\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 04:06:01,615 >> loading file https://huggingface.co/distilbert-base-cased/resolve/main/added_tokens.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 04:06:01,615 >> loading file https://huggingface.co/distilbert-base-cased/resolve/main/special_tokens_map.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 04:06:01,615 >> loading file https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer_config.json from cache at /root/.cache/huggingface/transformers/81e970e5e6ec68be12da0f8f3b2f2469c78d579282299a2ea65b4b7441719107.ec5c189f89475aac7d8cbd243960a0655cfadc3d0474da8ff2ed0bf1699c2a5f\n",
            "[INFO|modeling_utils.py:1075] 2021-04-22 04:06:01,772 >> loading weights file https://huggingface.co/distilbert-base-cased/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/9c9f39769dba4c5fe379b4bc82973eb01297bd607954621434eb9f1bc85a23a0.06b428c87335c1bb22eae46fdab31c8286efa0aa09e898a7ac42ddf5c3f5dc19\n",
            "[WARNING|modeling_utils.py:1196] 2021-04-22 04:06:08,787 >> Some weights of the model checkpoint at distilbert-base-cased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",
            "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "[WARNING|modeling_utils.py:1207] 2021-04-22 04:06:08,787 >> Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-cased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "100% 4/4 [00:00<00:00,  7.12ba/s]\n",
            "100% 1/1 [00:00<00:00, 17.23ba/s]\n",
            "100% 2/2 [00:00<00:00,  8.04ba/s]\n",
            "04/22/2021 04:06:09 - INFO - __main__ -   Sample 0 of the training set: {'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'idx': 0, 'input_ids': [101, 7277, 2180, 5303, 4806, 1117, 1711, 117, 2292, 1119, 1270, 107, 1103, 7737, 107, 117, 1104, 9938, 4267, 12223, 21811, 1117, 2554, 119, 102, 11336, 6732, 3384, 1106, 1140, 1112, 1178, 107, 1103, 7737, 107, 117, 7277, 2180, 5303, 4806, 1117, 1711, 1104, 9938, 4267, 12223, 21811, 1117, 2554, 119, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'label': 1, 'sentence1': 'Amrozi accused his brother , whom he called \" the witness \" , of deliberately distorting his evidence .', 'sentence2': 'Referring to him as only \" the witness \" , Amrozi accused his brother of deliberately distorting his evidence .'}.\n",
            "04/22/2021 04:06:09 - INFO - __main__ -   Sample 4 of the training set: {'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'idx': 4, 'input_ids': [101, 1109, 4482, 3152, 109, 123, 119, 1429, 117, 1137, 1164, 1429, 3029, 117, 1106, 1601, 5286, 1120, 109, 1626, 119, 4062, 1113, 1103, 1203, 1365, 9924, 7855, 119, 102, 153, 2349, 111, 142, 13619, 119, 6117, 4874, 109, 122, 119, 5519, 1137, 129, 3029, 1106, 109, 1626, 119, 5347, 1113, 1103, 1203, 1365, 9924, 7855, 1113, 5286, 119, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'label': 1, 'sentence1': 'The stock rose $ 2.11 , or about 11 percent , to close Friday at $ 21.51 on the New York Stock Exchange .', 'sentence2': 'PG & E Corp. shares jumped $ 1.63 or 8 percent to $ 21.03 on the New York Stock Exchange on Friday .'}.\n",
            "04/22/2021 04:06:09 - INFO - __main__ -   Sample 2 of the training set: {'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'idx': 2, 'input_ids': [101, 1220, 1125, 1502, 1126, 16355, 1113, 1103, 4639, 1113, 1340, 1275, 117, 4733, 1103, 6527, 1111, 4688, 117, 1119, 1896, 119, 102, 1212, 1340, 1275, 117, 1103, 2062, 112, 188, 5032, 1125, 1502, 1126, 16355, 1113, 1103, 4639, 117, 4733, 1103, 16454, 1111, 4688, 119, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'label': 1, 'sentence1': 'They had published an advertisement on the Internet on June 10 , offering the cargo for sale , he added .', 'sentence2': \"On June 10 , the ship 's owners had published an advertisement on the Internet , offering the explosives for sale .\"}.\n",
            "Downloading: 5.75kB [00:00, 3.42MB/s]       \n",
            "[INFO|trainer.py:497] 2021-04-22 04:06:10,642 >> The following columns in the training set  don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: sentence1, idx, sentence2.\n",
            "[INFO|trainer.py:1107] 2021-04-22 04:06:10,652 >> ***** Running training *****\n",
            "[INFO|trainer.py:1108] 2021-04-22 04:06:10,652 >>   Num examples = 5\n",
            "[INFO|trainer.py:1109] 2021-04-22 04:06:10,652 >>   Num Epochs = 3\n",
            "[INFO|trainer.py:1110] 2021-04-22 04:06:10,652 >>   Instantaneous batch size per device = 8\n",
            "[INFO|trainer.py:1111] 2021-04-22 04:06:10,652 >>   Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "[INFO|trainer.py:1112] 2021-04-22 04:06:10,652 >>   Gradient Accumulation steps = 1\n",
            "[INFO|trainer.py:1113] 2021-04-22 04:06:10,652 >>   Total optimization steps = 3\n",
            "100% 3/3 [00:21<00:00,  7.11s/it][INFO|trainer.py:1302] 2021-04-22 04:06:31,686 >> \n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "{'train_runtime': 21.0341, 'train_samples_per_second': 0.143, 'epoch': 3.0}\n",
            "100% 3/3 [00:21<00:00,  7.00s/it]\n",
            "[INFO|trainer.py:1768] 2021-04-22 04:06:31,834 >> Saving model checkpoint to /tmp/mrpc/\n",
            "[INFO|configuration_utils.py:329] 2021-04-22 04:06:31,835 >> Configuration saved in /tmp/mrpc/config.json\n",
            "[INFO|modeling_utils.py:848] 2021-04-22 04:06:32,842 >> Model weights saved in /tmp/mrpc/pytorch_model.bin\n",
            "[INFO|tokenization_utils_base.py:1918] 2021-04-22 04:06:32,843 >> tokenizer config file saved in /tmp/mrpc/tokenizer_config.json\n",
            "[INFO|tokenization_utils_base.py:1924] 2021-04-22 04:06:32,843 >> Special tokens file saved in /tmp/mrpc/special_tokens_map.json\n",
            "[INFO|trainer_pt_utils.py:898] 2021-04-22 04:06:32,922 >> ***** train metrics *****\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:32,923 >>   epoch                      =        3.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:32,923 >>   init_mem_cpu_alloc_delta   =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:32,923 >>   init_mem_cpu_peaked_delta  =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:32,923 >>   train_mem_cpu_alloc_delta  =      835MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:32,923 >>   train_mem_cpu_peaked_delta =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:32,923 >>   train_runtime              = 0:00:21.03\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:32,923 >>   train_samples              =          5\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:32,923 >>   train_samples_per_second   =      0.143\n",
            "04/22/2021 04:06:32 - INFO - __main__ -   *** Evaluate ***\n",
            "[INFO|trainer.py:497] 2021-04-22 04:06:33,077 >> The following columns in the evaluation set  don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: sentence1, idx, sentence2.\n",
            "[INFO|trainer.py:1988] 2021-04-22 04:06:33,081 >> ***** Running Evaluation *****\n",
            "[INFO|trainer.py:1990] 2021-04-22 04:06:33,081 >>   Num examples = 5\n",
            "[INFO|trainer.py:1993] 2021-04-22 04:06:33,081 >>   Batch size = 8\n",
            "100% 1/1 [00:00<00:00, 26.77it/s]\n",
            "[INFO|trainer_pt_utils.py:898] 2021-04-22 04:06:35,088 >> ***** eval metrics *****\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:35,089 >>   epoch                     =        3.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:35,089 >>   eval_accuracy             =        0.6\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:35,089 >>   eval_combined_score       =        0.3\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:35,089 >>   eval_f1                   =        0.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:35,089 >>   eval_loss                 =     0.6851\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:35,089 >>   eval_mem_cpu_alloc_delta  =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:35,089 >>   eval_mem_cpu_peaked_delta =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:35,089 >>   eval_runtime              = 0:00:01.85\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:35,089 >>   eval_samples              =          5\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:06:35,089 >>   eval_samples_per_second   =      2.692\n",
            "04/22/2021 04:06:35 - INFO - __main__ -   *** Test ***\n",
            "transformers/examples/pytorch/text-classification/run_glue.py:504: FutureWarning: remove_columns_ is deprecated and will be removed in the next major version of datasets. Use Dataset.remove_columns instead.\n",
            "  test_dataset.remove_columns_(\"label\")\n",
            "[INFO|trainer.py:497] 2021-04-22 04:06:35,230 >> The following columns in the test set  don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: sentence1, idx, sentence2.\n",
            "[INFO|trainer.py:1988] 2021-04-22 04:06:35,234 >> ***** Running Prediction *****\n",
            "[INFO|trainer.py:1990] 2021-04-22 04:06:35,234 >>   Num examples = 5\n",
            "[INFO|trainer.py:1993] 2021-04-22 04:06:35,235 >>   Batch size = 8\n",
            "  0% 0/1 [00:00<?, ?it/s]04/22/2021 04:06:37 - INFO - __main__ -   ***** Test results mrpc *****\n",
            "100% 1/1 [00:00<00:00,  7.18it/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ot6lcfpDevM_"
      },
      "source": [
        "## text-translation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PF6SSJnCrHLT",
        "outputId": "f95cc5c0-83fc-43dd-9e21-08b1db22458c"
      },
      "source": [
        "!python transformers/examples/pytorch/translation/run_translation.py \\\n",
        "--model_name_or_path Helsinki-NLP/opus-mt-en-ro \\\n",
        "--do_train \\\n",
        "--do_eval \\\n",
        "--do_predict \\\n",
        "--max_train_samples 5 \\\n",
        "--max_val_samples 5 \\\n",
        "--max_test_samples 5 \\\n",
        "--source_lang en \\\n",
        "--target_lang ro \\\n",
        "--dataset_name wmt16 \\\n",
        "--dataset_config_name ro-en \\\n",
        "--output_dir /tmp/tst-translation \\\n",
        "--overwrite_output_dir \\\n",
        "--predict_with_generate"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-22 04:06:51.378155: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "04/22/2021 04:06:53 - WARNING - __main__ -   Process rank: -1, device: cpu, n_gpu: 0distributed training: False, 16-bits training: False\n",
            "04/22/2021 04:06:53 - INFO - __main__ -   Training/evaluation parameters Seq2SeqTrainingArguments(output_dir='/tmp/tst-translation', overwrite_output_dir=True, do_train=True, do_eval=True, do_predict=True, evaluation_strategy=<IntervalStrategy.NO: 'no'>, prediction_loss_only=False, per_device_train_batch_size=8, per_device_eval_batch_size=8, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=5e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=3.0, max_steps=-1, lr_scheduler_type=<SchedulerType.LINEAR: 'linear'>, warmup_ratio=0.0, warmup_steps=0, logging_dir='runs/Apr22_04-06-53_a44dfd0e8363', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=False, logging_steps=500, save_strategy=<IntervalStrategy.STEPS: 'steps'>, save_steps=500, save_total_limit=None, no_cuda=False, seed=42, fp16=False, fp16_opt_level='O1', fp16_backend='auto', fp16_full_eval=False, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=500, dataloader_num_workers=0, past_index=-1, run_name='/tmp/tst-translation', disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, skip_memory_metrics=False, use_legacy_prediction_loop=False, mp_parameters='', sortish_sampler=False, predict_with_generate=True)\n",
            "Downloading: 2.81kB [00:00, 1.80MB/s]       \n",
            "Downloading: 3.19kB [00:00, 2.12MB/s]       \n",
            "Downloading: 41.0kB [00:00, 19.8MB/s]       \n",
            "Downloading and preparing dataset wmt16/ro-en (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/wmt16/ro-en/1.0.0/0d9fb3e814712c785176ad8cdb9f465fbe6479000ee6546725db30ad8a8b5f8a...\n",
            "Downloading: 100% 225M/225M [00:04<00:00, 52.5MB/s]\n",
            "Downloading: 100% 23.5M/23.5M [00:02<00:00, 9.73MB/s]\n",
            "Downloading: 100% 38.7M/38.7M [00:00<00:00, 50.1MB/s]\n",
            "Dataset wmt16 downloaded and prepared to /root/.cache/huggingface/datasets/wmt16/ro-en/1.0.0/0d9fb3e814712c785176ad8cdb9f465fbe6479000ee6546725db30ad8a8b5f8a. Subsequent calls will reuse this data.\n",
            "https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpjraejix7\n",
            "Downloading: 100% 1.13k/1.13k [00:00<00:00, 682kB/s]\n",
            "storing https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/config.json in cache at /root/.cache/huggingface/transformers/c9aa21082ce9a9811f9545a0fc0b441e82444d82f3b2571462c42fb470eec36e.9b192a33701c4f94ad3145ff0cdda62ca61214951101372f2ddaa47cf4f4aa25\n",
            "creating metadata file for /root/.cache/huggingface/transformers/c9aa21082ce9a9811f9545a0fc0b441e82444d82f3b2571462c42fb470eec36e.9b192a33701c4f94ad3145ff0cdda62ca61214951101372f2ddaa47cf4f4aa25\n",
            "loading configuration file https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/c9aa21082ce9a9811f9545a0fc0b441e82444d82f3b2571462c42fb470eec36e.9b192a33701c4f94ad3145ff0cdda62ca61214951101372f2ddaa47cf4f4aa25\n",
            "Model config MarianConfig {\n",
            "  \"_num_labels\": 3,\n",
            "  \"activation_dropout\": 0.0,\n",
            "  \"activation_function\": \"swish\",\n",
            "  \"add_bias_logits\": false,\n",
            "  \"add_final_layer_norm\": false,\n",
            "  \"architectures\": [\n",
            "    \"MarianMTModel\"\n",
            "  ],\n",
            "  \"attention_dropout\": 0.0,\n",
            "  \"bad_words_ids\": [\n",
            "    [\n",
            "      59542\n",
            "    ]\n",
            "  ],\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classif_dropout\": 0.0,\n",
            "  \"classifier_dropout\": 0.0,\n",
            "  \"d_model\": 512,\n",
            "  \"decoder_attention_heads\": 8,\n",
            "  \"decoder_ffn_dim\": 2048,\n",
            "  \"decoder_layerdrop\": 0.0,\n",
            "  \"decoder_layers\": 6,\n",
            "  \"decoder_start_token_id\": 59542,\n",
            "  \"dropout\": 0.1,\n",
            "  \"encoder_attention_heads\": 8,\n",
            "  \"encoder_ffn_dim\": 2048,\n",
            "  \"encoder_layerdrop\": 0.0,\n",
            "  \"encoder_layers\": 6,\n",
            "  \"eos_token_id\": 0,\n",
            "  \"forced_eos_token_id\": 0,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\"\n",
            "  },\n",
            "  \"init_std\": 0.02,\n",
            "  \"is_encoder_decoder\": true,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2\n",
            "  },\n",
            "  \"max_length\": 512,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"marian\",\n",
            "  \"normalize_before\": false,\n",
            "  \"normalize_embedding\": false,\n",
            "  \"num_beams\": 4,\n",
            "  \"num_hidden_layers\": 6,\n",
            "  \"pad_token_id\": 59542,\n",
            "  \"scale_embedding\": true,\n",
            "  \"static_position_embeddings\": true,\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 59543\n",
            "}\n",
            "\n",
            "loading configuration file https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/c9aa21082ce9a9811f9545a0fc0b441e82444d82f3b2571462c42fb470eec36e.9b192a33701c4f94ad3145ff0cdda62ca61214951101372f2ddaa47cf4f4aa25\n",
            "Model config MarianConfig {\n",
            "  \"_num_labels\": 3,\n",
            "  \"activation_dropout\": 0.0,\n",
            "  \"activation_function\": \"swish\",\n",
            "  \"add_bias_logits\": false,\n",
            "  \"add_final_layer_norm\": false,\n",
            "  \"architectures\": [\n",
            "    \"MarianMTModel\"\n",
            "  ],\n",
            "  \"attention_dropout\": 0.0,\n",
            "  \"bad_words_ids\": [\n",
            "    [\n",
            "      59542\n",
            "    ]\n",
            "  ],\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classif_dropout\": 0.0,\n",
            "  \"classifier_dropout\": 0.0,\n",
            "  \"d_model\": 512,\n",
            "  \"decoder_attention_heads\": 8,\n",
            "  \"decoder_ffn_dim\": 2048,\n",
            "  \"decoder_layerdrop\": 0.0,\n",
            "  \"decoder_layers\": 6,\n",
            "  \"decoder_start_token_id\": 59542,\n",
            "  \"dropout\": 0.1,\n",
            "  \"encoder_attention_heads\": 8,\n",
            "  \"encoder_ffn_dim\": 2048,\n",
            "  \"encoder_layerdrop\": 0.0,\n",
            "  \"encoder_layers\": 6,\n",
            "  \"eos_token_id\": 0,\n",
            "  \"forced_eos_token_id\": 0,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\"\n",
            "  },\n",
            "  \"init_std\": 0.02,\n",
            "  \"is_encoder_decoder\": true,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2\n",
            "  },\n",
            "  \"max_length\": 512,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"marian\",\n",
            "  \"normalize_before\": false,\n",
            "  \"normalize_embedding\": false,\n",
            "  \"num_beams\": 4,\n",
            "  \"num_hidden_layers\": 6,\n",
            "  \"pad_token_id\": 59542,\n",
            "  \"scale_embedding\": true,\n",
            "  \"static_position_embeddings\": true,\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 59543\n",
            "}\n",
            "\n",
            "https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/source.spm not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp4fq64tap\n",
            "Downloading: 100% 789k/789k [00:00<00:00, 5.21MB/s]\n",
            "storing https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/source.spm in cache at /root/.cache/huggingface/transformers/1954839338efdfd7d7cd66bb676626f3fa5404a96e4c3fd9e0d777f08fb2f7fb.2a52a89183278824d0afa78f92263dc46ca0c520a60df7071b6ae4e00fc5bef7\n",
            "creating metadata file for /root/.cache/huggingface/transformers/1954839338efdfd7d7cd66bb676626f3fa5404a96e4c3fd9e0d777f08fb2f7fb.2a52a89183278824d0afa78f92263dc46ca0c520a60df7071b6ae4e00fc5bef7\n",
            "https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/target.spm not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmptonph4z2\n",
            "Downloading: 100% 817k/817k [00:00<00:00, 5.41MB/s]\n",
            "storing https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/target.spm in cache at /root/.cache/huggingface/transformers/1e4b193c82cb1b7fe168ad6685f3a053a2b1ab3482d2d12c306282e03ff379f0.afd22f83f2170348e28be1f817974819f568155c58acd1bb3800cd7937db36a1\n",
            "creating metadata file for /root/.cache/huggingface/transformers/1e4b193c82cb1b7fe168ad6685f3a053a2b1ab3482d2d12c306282e03ff379f0.afd22f83f2170348e28be1f817974819f568155c58acd1bb3800cd7937db36a1\n",
            "https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/vocab.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmppekq8rpy\n",
            "Downloading: 100% 1.39M/1.39M [00:00<00:00, 7.73MB/s]\n",
            "storing https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/vocab.json in cache at /root/.cache/huggingface/transformers/bffa5c5398d82c200e0eeee10da8e37228615dc7fbabd429567d6ae242f5a615.5e73795abff6c04c6739872ed86c111983aa47f92d9648ac33f2d9d3643e1357\n",
            "creating metadata file for /root/.cache/huggingface/transformers/bffa5c5398d82c200e0eeee10da8e37228615dc7fbabd429567d6ae242f5a615.5e73795abff6c04c6739872ed86c111983aa47f92d9648ac33f2d9d3643e1357\n",
            "https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/tokenizer_config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpd_mmecr0\n",
            "Downloading: 100% 42.0/42.0 [00:00<00:00, 28.7kB/s]\n",
            "storing https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/tokenizer_config.json in cache at /root/.cache/huggingface/transformers/e05a8eb93de5526dd8224c840b38615d3f79b0bf9636f74ade2807ec16ae347f.09f8e480ed5ba39c95bf16472d22478f2d1870f222433394dd0fee0ac9f1b907\n",
            "creating metadata file for /root/.cache/huggingface/transformers/e05a8eb93de5526dd8224c840b38615d3f79b0bf9636f74ade2807ec16ae347f.09f8e480ed5ba39c95bf16472d22478f2d1870f222433394dd0fee0ac9f1b907\n",
            "loading file https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/source.spm from cache at /root/.cache/huggingface/transformers/1954839338efdfd7d7cd66bb676626f3fa5404a96e4c3fd9e0d777f08fb2f7fb.2a52a89183278824d0afa78f92263dc46ca0c520a60df7071b6ae4e00fc5bef7\n",
            "loading file https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/target.spm from cache at /root/.cache/huggingface/transformers/1e4b193c82cb1b7fe168ad6685f3a053a2b1ab3482d2d12c306282e03ff379f0.afd22f83f2170348e28be1f817974819f568155c58acd1bb3800cd7937db36a1\n",
            "loading file https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/vocab.json from cache at /root/.cache/huggingface/transformers/bffa5c5398d82c200e0eeee10da8e37228615dc7fbabd429567d6ae242f5a615.5e73795abff6c04c6739872ed86c111983aa47f92d9648ac33f2d9d3643e1357\n",
            "loading file https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/tokenizer_config.json from cache at /root/.cache/huggingface/transformers/e05a8eb93de5526dd8224c840b38615d3f79b0bf9636f74ade2807ec16ae347f.09f8e480ed5ba39c95bf16472d22478f2d1870f222433394dd0fee0ac9f1b907\n",
            "loading file https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/added_tokens.json from cache at None\n",
            "loading file https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/special_tokens_map.json from cache at None\n",
            "loading file https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/tokenizer.json from cache at None\n",
            "https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/pytorch_model.bin not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp754dp8v_\n",
            "Downloading: 100% 301M/301M [00:08<00:00, 35.0MB/s]\n",
            "storing https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/pytorch_model.bin in cache at /root/.cache/huggingface/transformers/ddea6571c8d435d3a95a0d3b003ce4ed9cd74380314ce1510adc1a2b80ec6541.ea59afa9489e25f746dc0446b0ef6874b3b9e704d9d077620684ec467f86be9d\n",
            "creating metadata file for /root/.cache/huggingface/transformers/ddea6571c8d435d3a95a0d3b003ce4ed9cd74380314ce1510adc1a2b80ec6541.ea59afa9489e25f746dc0446b0ef6874b3b9e704d9d077620684ec467f86be9d\n",
            "loading weights file https://huggingface.co/Helsinki-NLP/opus-mt-en-ro/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/ddea6571c8d435d3a95a0d3b003ce4ed9cd74380314ce1510adc1a2b80ec6541.ea59afa9489e25f746dc0446b0ef6874b3b9e704d9d077620684ec467f86be9d\n",
            "All model checkpoint weights were used when initializing MarianMTModel.\n",
            "\n",
            "All the weights of MarianMTModel were initialized from the model checkpoint at Helsinki-NLP/opus-mt-en-ro.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use MarianMTModel for predictions without further training.\n",
            "100% 1/1 [00:00<00:00, 276.92ba/s]\n",
            "100% 1/1 [00:00<00:00, 211.28ba/s]\n",
            "100% 1/1 [00:00<00:00, 245.07ba/s]\n",
            "Downloading: 5.40kB [00:00, 3.28MB/s]       \n",
            "***** Running training *****\n",
            "  Num examples = 5\n",
            "  Num Epochs = 3\n",
            "  Instantaneous batch size per device = 8\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3\n",
            "100% 3/3 [00:07<00:00,  2.66s/it]\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "{'train_runtime': 7.8219, 'train_samples_per_second': 0.384, 'epoch': 3.0}\n",
            "100% 3/3 [00:07<00:00,  2.60s/it]\n",
            "Saving model checkpoint to /tmp/tst-translation\n",
            "Configuration saved in /tmp/tst-translation/config.json\n",
            "Model weights saved in /tmp/tst-translation/pytorch_model.bin\n",
            "tokenizer config file saved in /tmp/tst-translation/tokenizer_config.json\n",
            "Special tokens file saved in /tmp/tst-translation/special_tokens_map.json\n",
            "***** train metrics *****\n",
            "  epoch                      =        3.0\n",
            "  init_mem_cpu_alloc_delta   =        0MB\n",
            "  init_mem_cpu_peaked_delta  =        0MB\n",
            "  train_mem_cpu_alloc_delta  =     1028MB\n",
            "  train_mem_cpu_peaked_delta =        0MB\n",
            "  train_runtime              = 0:00:07.82\n",
            "  train_samples              =          5\n",
            "  train_samples_per_second   =      0.384\n",
            "04/22/2021 04:08:04 - INFO - __main__ -   *** Evaluate ***\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 5\n",
            "  Batch size = 8\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/generation_utils.py:965: UserWarning: `max_length` is deprecated in this function, use `stopping_criteria=StoppingCriteriaList(MaxLengthCriteria(max_length=max_length))` instead.\n",
            "  UserWarning,\n",
            "100% 1/1 [00:00<00:00, 31.58it/s]\n",
            "***** eval metrics *****\n",
            "  epoch                     =        3.0\n",
            "  eval_bleu                 =    26.6404\n",
            "  eval_gen_len              =       38.2\n",
            "  eval_loss                 =     0.9509\n",
            "  eval_mem_cpu_alloc_delta  =        1MB\n",
            "  eval_mem_cpu_peaked_delta =        0MB\n",
            "  eval_runtime              = 0:00:14.02\n",
            "  eval_samples              =          5\n",
            "  eval_samples_per_second   =      0.356\n",
            "04/22/2021 04:08:19 - INFO - __main__ -   *** Test ***\n",
            "***** Running Prediction *****\n",
            "  Num examples = 5\n",
            "  Batch size = 8\n",
            "  0% 0/1 [00:00<?, ?it/s]***** test metrics *****\n",
            "  test_bleu                 =    31.1522\n",
            "  test_gen_len              =       40.0\n",
            "  test_loss                 =     1.3919\n",
            "  test_mem_cpu_alloc_delta  =        0MB\n",
            "  test_mem_cpu_peaked_delta =        0MB\n",
            "  test_runtime              = 0:00:14.01\n",
            "  test_samples              =          5\n",
            "  test_samples_per_second   =      0.357\n",
            "100% 1/1 [00:00<00:00,  5.48it/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w0jyisZ2ewm1"
      },
      "source": [
        "## token-classification"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-BEeAVLaensq",
        "outputId": "cf3d8f6c-0fa7-4d09-f8bd-fab573c5c5ba"
      },
      "source": [
        "!python transformers/examples/pytorch/token-classification/run_ner.py \\\n",
        "--model_name_or_path bert-base-uncased \\\n",
        "--dataset_name conll2003 \\\n",
        "--output_dir /tmp/test-ner \\\n",
        "--do_train \\\n",
        "--do_eval \\\n",
        "--do_predict \\\n",
        "--max_train_samples 5 \\\n",
        "--max_val_samples 5 \\\n",
        "--max_test_samples 5 \\"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-22 04:09:03.987898: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "04/22/2021 04:09:06 - WARNING - __main__ -   Process rank: -1, device: cpu, n_gpu: 0distributed training: False, 16-bits training: False\n",
            "04/22/2021 04:09:06 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(output_dir=/tmp/test-ner, overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=True, evaluation_strategy=IntervalStrategy.NO, prediction_loss_only=False, per_device_train_batch_size=8, per_device_eval_batch_size=8, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=5e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=3.0, max_steps=-1, lr_scheduler_type=SchedulerType.LINEAR, warmup_ratio=0.0, warmup_steps=0, logging_dir=runs/Apr22_04-09-06_a44dfd0e8363, logging_strategy=IntervalStrategy.STEPS, logging_first_step=False, logging_steps=500, save_strategy=IntervalStrategy.STEPS, save_steps=500, save_total_limit=None, no_cuda=False, seed=42, fp16=False, fp16_opt_level=O1, fp16_backend=auto, fp16_full_eval=False, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=500, dataloader_num_workers=0, past_index=-1, run_name=/tmp/test-ner, disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, length_column_name=length, report_to=['tensorboard'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, skip_memory_metrics=False, use_legacy_prediction_loop=False, _n_gpu=0, mp_parameters=)\n",
            "Downloading: 9.52kB [00:00, 5.73MB/s]       \n",
            "Downloading: 4.18kB [00:00, 2.60MB/s]       \n",
            "Downloading and preparing dataset conll2003/conll2003 (download: 4.63 MiB, generated: 9.78 MiB, post-processed: Unknown size, total: 14.41 MiB) to /root/.cache/huggingface/datasets/conll2003/conll2003/1.0.0/40e7cb6bcc374f7c349c83acd1e9352a4f09474eb691f64f364ee62eb65d0ca6...\n",
            "Downloading: 3.28MB [00:00, 49.0MB/s]      \n",
            "Downloading: 827kB [00:00, 31.1MB/s]       \n",
            "Downloading: 748kB [00:00, 31.9MB/s]       \n",
            "Dataset conll2003 downloaded and prepared to /root/.cache/huggingface/datasets/conll2003/conll2003/1.0.0/40e7cb6bcc374f7c349c83acd1e9352a4f09474eb691f64f364ee62eb65d0ca6. Subsequent calls will reuse this data.\n",
            "[INFO|file_utils.py:1395] 2021-04-22 04:09:13,749 >> https://huggingface.co/bert-base-uncased/resolve/main/config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp60ups_3j\n",
            "Downloading: 100% 433/433 [00:00<00:00, 270kB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 04:09:13,853 >> storing https://huggingface.co/bert-base-uncased/resolve/main/config.json in cache at /root/.cache/huggingface/transformers/3c61d016573b14f7f008c02c4e51a366c67ab274726fe2910691e2a761acf43e.637c6035640bacb831febcc2b7f7bee0a96f9b30c2d7e9ef84082d9f252f3170\n",
            "[INFO|file_utils.py:1402] 2021-04-22 04:09:13,853 >> creating metadata file for /root/.cache/huggingface/transformers/3c61d016573b14f7f008c02c4e51a366c67ab274726fe2910691e2a761acf43e.637c6035640bacb831febcc2b7f7bee0a96f9b30c2d7e9ef84082d9f252f3170\n",
            "[INFO|configuration_utils.py:491] 2021-04-22 04:09:13,854 >> loading configuration file https://huggingface.co/bert-base-uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/3c61d016573b14f7f008c02c4e51a366c67ab274726fe2910691e2a761acf43e.637c6035640bacb831febcc2b7f7bee0a96f9b30c2d7e9ef84082d9f252f3170\n",
            "[INFO|configuration_utils.py:527] 2021-04-22 04:09:13,855 >> Model config BertConfig {\n",
            "  \"architectures\": [\n",
            "    \"BertForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"finetuning_task\": \"ner\",\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"bert\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 30522\n",
            "}\n",
            "\n",
            "[INFO|configuration_utils.py:491] 2021-04-22 04:09:13,958 >> loading configuration file https://huggingface.co/bert-base-uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/3c61d016573b14f7f008c02c4e51a366c67ab274726fe2910691e2a761acf43e.637c6035640bacb831febcc2b7f7bee0a96f9b30c2d7e9ef84082d9f252f3170\n",
            "[INFO|configuration_utils.py:527] 2021-04-22 04:09:13,959 >> Model config BertConfig {\n",
            "  \"architectures\": [\n",
            "    \"BertForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"bert\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.6.0.dev0\",\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 30522\n",
            "}\n",
            "\n",
            "[INFO|file_utils.py:1395] 2021-04-22 04:09:14,064 >> https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp_kzo061j\n",
            "Downloading: 100% 232k/232k [00:00<00:00, 2.05MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 04:09:14,281 >> storing https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt in cache at /root/.cache/huggingface/transformers/45c3f7a79a80e1cf0a489e5c62b43f173c15db47864303a55d623bb3c96f72a5.d789d64ebfe299b0e416afc4a169632f903f693095b4629a7ea271d5a0cf2c99\n",
            "[INFO|file_utils.py:1402] 2021-04-22 04:09:14,281 >> creating metadata file for /root/.cache/huggingface/transformers/45c3f7a79a80e1cf0a489e5c62b43f173c15db47864303a55d623bb3c96f72a5.d789d64ebfe299b0e416afc4a169632f903f693095b4629a7ea271d5a0cf2c99\n",
            "[INFO|file_utils.py:1395] 2021-04-22 04:09:14,384 >> https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpq0pei56a\n",
            "Downloading: 100% 466k/466k [00:00<00:00, 3.34MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 04:09:14,630 >> storing https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json in cache at /root/.cache/huggingface/transformers/534479488c54aeaf9c3406f647aa2ec13648c06771ffe269edabebd4c412da1d.7f2721073f19841be16f41b0a70b600ca6b880c8f3df6f3535cbc704371bdfa4\n",
            "[INFO|file_utils.py:1402] 2021-04-22 04:09:14,631 >> creating metadata file for /root/.cache/huggingface/transformers/534479488c54aeaf9c3406f647aa2ec13648c06771ffe269edabebd4c412da1d.7f2721073f19841be16f41b0a70b600ca6b880c8f3df6f3535cbc704371bdfa4\n",
            "[INFO|file_utils.py:1395] 2021-04-22 04:09:14,939 >> https://huggingface.co/bert-base-uncased/resolve/main/tokenizer_config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpj7foicg_\n",
            "Downloading: 100% 28.0/28.0 [00:00<00:00, 17.6kB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 04:09:15,044 >> storing https://huggingface.co/bert-base-uncased/resolve/main/tokenizer_config.json in cache at /root/.cache/huggingface/transformers/c1d7f0a763fb63861cc08553866f1fc3e5a6f4f07621be277452d26d71303b7e.20430bd8e10ef77a7d2977accefe796051e01bc2fc4aa146bc862997a1a15e79\n",
            "[INFO|file_utils.py:1402] 2021-04-22 04:09:15,044 >> creating metadata file for /root/.cache/huggingface/transformers/c1d7f0a763fb63861cc08553866f1fc3e5a6f4f07621be277452d26d71303b7e.20430bd8e10ef77a7d2977accefe796051e01bc2fc4aa146bc862997a1a15e79\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 04:09:15,045 >> loading file https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt from cache at /root/.cache/huggingface/transformers/45c3f7a79a80e1cf0a489e5c62b43f173c15db47864303a55d623bb3c96f72a5.d789d64ebfe299b0e416afc4a169632f903f693095b4629a7ea271d5a0cf2c99\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 04:09:15,045 >> loading file https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json from cache at /root/.cache/huggingface/transformers/534479488c54aeaf9c3406f647aa2ec13648c06771ffe269edabebd4c412da1d.7f2721073f19841be16f41b0a70b600ca6b880c8f3df6f3535cbc704371bdfa4\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 04:09:15,045 >> loading file https://huggingface.co/bert-base-uncased/resolve/main/added_tokens.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 04:09:15,045 >> loading file https://huggingface.co/bert-base-uncased/resolve/main/special_tokens_map.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1713] 2021-04-22 04:09:15,045 >> loading file https://huggingface.co/bert-base-uncased/resolve/main/tokenizer_config.json from cache at /root/.cache/huggingface/transformers/c1d7f0a763fb63861cc08553866f1fc3e5a6f4f07621be277452d26d71303b7e.20430bd8e10ef77a7d2977accefe796051e01bc2fc4aa146bc862997a1a15e79\n",
            "[INFO|file_utils.py:1395] 2021-04-22 04:09:15,182 >> https://huggingface.co/bert-base-uncased/resolve/main/pytorch_model.bin not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpgux_si9z\n",
            "Downloading: 100% 440M/440M [00:12<00:00, 33.9MB/s]\n",
            "[INFO|file_utils.py:1399] 2021-04-22 04:09:28,388 >> storing https://huggingface.co/bert-base-uncased/resolve/main/pytorch_model.bin in cache at /root/.cache/huggingface/transformers/a8041bf617d7f94ea26d15e218abd04afc2004805632abc0ed2066aa16d50d04.faf6ea826ae9c5867d12b22257f9877e6b8367890837bd60f7c54a29633f7f2f\n",
            "[INFO|file_utils.py:1402] 2021-04-22 04:09:28,388 >> creating metadata file for /root/.cache/huggingface/transformers/a8041bf617d7f94ea26d15e218abd04afc2004805632abc0ed2066aa16d50d04.faf6ea826ae9c5867d12b22257f9877e6b8367890837bd60f7c54a29633f7f2f\n",
            "[INFO|modeling_utils.py:1075] 2021-04-22 04:09:28,388 >> loading weights file https://huggingface.co/bert-base-uncased/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/a8041bf617d7f94ea26d15e218abd04afc2004805632abc0ed2066aa16d50d04.faf6ea826ae9c5867d12b22257f9877e6b8367890837bd60f7c54a29633f7f2f\n",
            "[WARNING|modeling_utils.py:1196] 2021-04-22 04:09:32,416 >> Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "[WARNING|modeling_utils.py:1207] 2021-04-22 04:09:32,416 >> Some weights of BertForTokenClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "100% 1/1 [00:00<00:00, 198.97ba/s]\n",
            "100% 1/1 [00:00<00:00, 229.89ba/s]\n",
            "100% 1/1 [00:00<00:00, 277.64ba/s]\n",
            "Downloading: 6.34kB [00:00, 3.36MB/s]       \n",
            "[INFO|trainer.py:497] 2021-04-22 04:09:33,536 >> The following columns in the training set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: id, pos_tags, ner_tags, chunk_tags, tokens.\n",
            "[INFO|trainer.py:1107] 2021-04-22 04:09:33,545 >> ***** Running training *****\n",
            "[INFO|trainer.py:1108] 2021-04-22 04:09:33,545 >>   Num examples = 5\n",
            "[INFO|trainer.py:1109] 2021-04-22 04:09:33,545 >>   Num Epochs = 3\n",
            "[INFO|trainer.py:1110] 2021-04-22 04:09:33,545 >>   Instantaneous batch size per device = 8\n",
            "[INFO|trainer.py:1111] 2021-04-22 04:09:33,545 >>   Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "[INFO|trainer.py:1112] 2021-04-22 04:09:33,545 >>   Gradient Accumulation steps = 1\n",
            "[INFO|trainer.py:1113] 2021-04-22 04:09:33,546 >>   Total optimization steps = 3\n",
            "100% 3/3 [00:15<00:00,  5.25s/it][INFO|trainer.py:1302] 2021-04-22 04:09:48,880 >> \n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "{'train_runtime': 15.3349, 'train_samples_per_second': 0.196, 'epoch': 3.0}\n",
            "100% 3/3 [00:15<00:00,  5.11s/it]\n",
            "[INFO|trainer.py:1768] 2021-04-22 04:09:49,031 >> Saving model checkpoint to /tmp/test-ner\n",
            "[INFO|configuration_utils.py:329] 2021-04-22 04:09:49,032 >> Configuration saved in /tmp/test-ner/config.json\n",
            "[INFO|modeling_utils.py:848] 2021-04-22 04:09:50,331 >> Model weights saved in /tmp/test-ner/pytorch_model.bin\n",
            "[INFO|tokenization_utils_base.py:1918] 2021-04-22 04:09:50,332 >> tokenizer config file saved in /tmp/test-ner/tokenizer_config.json\n",
            "[INFO|tokenization_utils_base.py:1924] 2021-04-22 04:09:50,332 >> Special tokens file saved in /tmp/test-ner/special_tokens_map.json\n",
            "[INFO|trainer_pt_utils.py:898] 2021-04-22 04:09:50,390 >> ***** train metrics *****\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:50,390 >>   epoch                      =        3.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:50,391 >>   init_mem_cpu_alloc_delta   =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:50,391 >>   init_mem_cpu_peaked_delta  =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:50,391 >>   train_mem_cpu_alloc_delta  =     1009MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:50,391 >>   train_mem_cpu_peaked_delta =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:50,391 >>   train_runtime              = 0:00:15.33\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:50,391 >>   train_samples              =          5\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:50,391 >>   train_samples_per_second   =      0.196\n",
            "04/22/2021 04:09:50 - INFO - __main__ -   *** Evaluate ***\n",
            "[INFO|trainer.py:497] 2021-04-22 04:09:50,542 >> The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: id, pos_tags, ner_tags, chunk_tags, tokens.\n",
            "[INFO|trainer.py:1988] 2021-04-22 04:09:50,547 >> ***** Running Evaluation *****\n",
            "[INFO|trainer.py:1990] 2021-04-22 04:09:50,548 >>   Num examples = 5\n",
            "[INFO|trainer.py:1993] 2021-04-22 04:09:50,548 >>   Batch size = 8\n",
            "  0% 0/1 [00:00<?, ?it/s]/usr/local/lib/python3.7/dist-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "100% 1/1 [00:00<00:00, 56.38it/s]\n",
            "[INFO|trainer_pt_utils.py:898] 2021-04-22 04:09:51,944 >> ***** eval metrics *****\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:51,944 >>   epoch                     =        3.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:51,944 >>   eval_accuracy             =     0.7561\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:51,944 >>   eval_f1                   =        0.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:51,944 >>   eval_loss                 =      1.669\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:51,944 >>   eval_mem_cpu_alloc_delta  =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:51,944 >>   eval_mem_cpu_peaked_delta =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:51,944 >>   eval_precision            =        0.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:51,944 >>   eval_recall               =        0.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:51,944 >>   eval_runtime              = 0:00:01.25\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:51,944 >>   eval_samples              =          5\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:51,944 >>   eval_samples_per_second   =        4.0\n",
            "04/22/2021 04:09:51 - INFO - __main__ -   *** Predict ***\n",
            "[INFO|trainer.py:497] 2021-04-22 04:09:52,098 >> The following columns in the test set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: id, pos_tags, ner_tags, chunk_tags, tokens.\n",
            "[INFO|trainer.py:1988] 2021-04-22 04:09:52,102 >> ***** Running Prediction *****\n",
            "[INFO|trainer.py:1990] 2021-04-22 04:09:52,103 >>   Num examples = 5\n",
            "[INFO|trainer.py:1993] 2021-04-22 04:09:52,103 >>   Batch size = 8\n",
            "  0% 0/1 [00:00<?, ?it/s]/usr/local/lib/python3.7/dist-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "[INFO|trainer_pt_utils.py:898] 2021-04-22 04:09:53,152 >> ***** test metrics *****\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:53,152 >>   test_accuracy             =     0.7857\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:53,152 >>   test_f1                   =        0.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:53,152 >>   test_loss                 =     1.4666\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:53,152 >>   test_mem_cpu_alloc_delta  =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:53,152 >>   test_mem_cpu_peaked_delta =        0MB\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:53,152 >>   test_precision            =        0.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:53,152 >>   test_recall               =        0.0\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:53,152 >>   test_runtime              = 0:00:00.89\n",
            "[INFO|trainer_pt_utils.py:903] 2021-04-22 04:09:53,152 >>   test_samples_per_second   =      5.594\n",
            "100% 1/1 [00:00<00:00,  5.90it/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eKpCx5dusRNO"
      },
      "source": [
        ""
      ],
      "execution_count": 12,
      "outputs": []
    }
  ]
}