iwebBert_v02.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "75fa69a2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['corpora/wikipedia.v03.txt']\n",
      "\n",
      "\n",
      "\n",
      "CPU times: user 8min 21s, sys: 1min 7s, total: 9min 28s\n",
      "Wall time: 49.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "from pathlib import Path\n",
    "from tokenizers import ByteLevelBPETokenizer\n",
    "\n",
    "paths = [str(x) for x in Path(\".\").glob(\"corpora/wikipedia.v03.txt\")]\n",
    "\n",
    "print(paths)\n",
    "\n",
    "tokenizer= ByteLevelBPETokenizer()\n",
    "tokenizer.train(files=paths, vocab_size = 64_000, min_frequency=2, special_tokens=[\"<p>\",\"</p>\",\"<s>\",\"<pad>\",\"</s>\",\"<unk>\",\"<mask>\",])\n",
    "\n",
    "\n",
    "\n",
    "#from transformers import RobertaTokenizer\n",
    "\n",
    "#tokenizer = RobertaTokenizer.from_pretrained(\"roberta-base\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "660cc6f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/home/jmhuerta/content/iwebBERT_v03/vocab.json',\n",
       " '/home/jmhuerta/content/iwebBERT_v03/merges.txt']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "token_dir = '/home/jmhuerta/content/iwebBERT_v03/'\n",
    "if not os.path.exists(token_dir):\n",
    "    os.makedirs(token_dir)\n",
    "tokenizer.save_model(token_dir)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4eb3e8f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tokenizers.implementations import ByteLevelBPETokenizer\n",
    "from tokenizers.processors import BertProcessing\n",
    "tokenizer = ByteLevelBPETokenizer('/home/jmhuerta/content/iwebBERT_v03/vocab.json', '/home/jmhuerta/content/iwebBERT_v03/merges.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1feb3107",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer._tokenizer.post_processor = BertProcessing((\"</s>\",tokenizer.token_to_id(\"</s>\")),(\"<s>\",tokenizer.token_to_id(\"<s>\")))\n",
    "tokenizer.enable_truncation(max_length=512)\n",
    "                                                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "955f9b04",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Encoding(num_tokens=12, attributes=[ids, type_ids, tokens, offsets, attention_mask, special_tokens_mask, overflowing])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.encode(\"The Critique of Pure Reason. And practical reason.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c9f8daf9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU available True\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "print(\"GPU available\", torch.cuda.is_available())\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6ad91eb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import RobertaConfig\n",
    "config = RobertaConfig(\n",
    "    vocab_size=64_000,\n",
    "    max_position_embedding=512,\n",
    "    num_attention_heads=12, \n",
    "    num_hidden_layers=6,\n",
    "    type_vocab_size=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d6db1217",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import RobertaTokenizer\n",
    "\n",
    "tokenizer = RobertaTokenizer.from_pretrained('/home/jmhuerta/content/iwebBERT_v03/',max_length=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "16fc2117",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RobertaForMaskedLM(\n",
      "  (roberta): RobertaModel(\n",
      "    (embeddings): RobertaEmbeddings(\n",
      "      (word_embeddings): Embedding(64000, 768, padding_idx=1)\n",
      "      (position_embeddings): Embedding(512, 768, padding_idx=1)\n",
      "      (token_type_embeddings): Embedding(1, 768)\n",
      "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "      (dropout): Dropout(p=0.1, inplace=False)\n",
      "    )\n",
      "    (encoder): RobertaEncoder(\n",
      "      (layer): ModuleList(\n",
      "        (0): RobertaLayer(\n",
      "          (attention): RobertaAttention(\n",
      "            (self): RobertaSelfAttention(\n",
      "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): RobertaSelfOutput(\n",
      "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): RobertaIntermediate(\n",
      "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): RobertaOutput(\n",
      "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (1): RobertaLayer(\n",
      "          (attention): RobertaAttention(\n",
      "            (self): RobertaSelfAttention(\n",
      "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): RobertaSelfOutput(\n",
      "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): RobertaIntermediate(\n",
      "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): RobertaOutput(\n",
      "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (2): RobertaLayer(\n",
      "          (attention): RobertaAttention(\n",
      "            (self): RobertaSelfAttention(\n",
      "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): RobertaSelfOutput(\n",
      "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): RobertaIntermediate(\n",
      "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): RobertaOutput(\n",
      "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (3): RobertaLayer(\n",
      "          (attention): RobertaAttention(\n",
      "            (self): RobertaSelfAttention(\n",
      "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): RobertaSelfOutput(\n",
      "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): RobertaIntermediate(\n",
      "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): RobertaOutput(\n",
      "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (4): RobertaLayer(\n",
      "          (attention): RobertaAttention(\n",
      "            (self): RobertaSelfAttention(\n",
      "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): RobertaSelfOutput(\n",
      "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): RobertaIntermediate(\n",
      "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): RobertaOutput(\n",
      "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (5): RobertaLayer(\n",
      "          (attention): RobertaAttention(\n",
      "            (self): RobertaSelfAttention(\n",
      "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "            (output): RobertaSelfOutput(\n",
      "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): RobertaIntermediate(\n",
      "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            (intermediate_act_fn): GELUActivation()\n",
      "          )\n",
      "          (output): RobertaOutput(\n",
      "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (lm_head): RobertaLMHead(\n",
      "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "    (layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "    (decoder): Linear(in_features=768, out_features=64000, bias=True)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "from transformers import RobertaForMaskedLM\n",
    "model = RobertaForMaskedLM(config=config).to(device)\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1fa7d6f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "92730880\n"
     ]
    }
   ],
   "source": [
    "print(model.num_parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e10601a6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jmhuerta/anaconda3/envs/pytorch/lib/python3.10/site-packages/transformers/data/datasets/language_modeling.py:121: FutureWarning: This dataset will be removed from the library soon, preprocessing should be handled with the 🤗 Datasets library. You can have a look at this example script for pointers: https://github.com/huggingface/transformers/blob/main/examples/pytorch/language-modeling/run_mlm.py\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 9min 9s, sys: 13.2 s, total: 9min 22s\n",
      "Wall time: 9min 25s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from transformers import LineByLineTextDataset\n",
    "dataset = LineByLineTextDataset(tokenizer=tokenizer, file_path =\"./corpora/wikipedia.v03.txt\", block_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7463f247",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import DataCollatorForLanguageModeling\n",
    "data_collator = DataCollatorForLanguageModeling (\n",
    "    tokenizer= tokenizer, mlm=True, mlm_probability =0.15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "97b5a01a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#@title Step 12: Initializing the Trainer\n",
    "from transformers import Trainer, TrainingArguments\n",
    "\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./iwebBERT\",\n",
    "    overwrite_output_dir = True,\n",
    "    num_train_epochs=1,\n",
    "    per_device_train_batch_size=32,\n",
    "    save_steps=10_000,\n",
    "    save_total_limit=2,\n",
    ")\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    data_collator=data_collator,\n",
    "    train_dataset=dataset,\n",
    ")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1e3760ea",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jmhuerta/anaconda3/envs/pytorch/lib/python3.10/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 2000000\n",
      "  Num Epochs = 1\n",
      "  Instantaneous batch size per device = 32\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 32\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 62500\n",
      "  Number of trainable parameters = 92730880\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='62500' max='62500' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [62500/62500 14:04:24, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>7.645500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>7.022200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>6.810100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>6.661500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>6.556300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>6.347400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3500</td>\n",
       "      <td>6.115200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4000</td>\n",
       "      <td>5.873800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4500</td>\n",
       "      <td>5.669500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5000</td>\n",
       "      <td>5.502100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5500</td>\n",
       "      <td>5.376900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6000</td>\n",
       "      <td>5.258300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6500</td>\n",
       "      <td>5.175900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7000</td>\n",
       "      <td>5.118200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7500</td>\n",
       "      <td>5.037700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8000</td>\n",
       "      <td>4.993100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8500</td>\n",
       "      <td>4.906300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9000</td>\n",
       "      <td>4.867500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9500</td>\n",
       "      <td>4.750900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10000</td>\n",
       "      <td>4.739400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10500</td>\n",
       "      <td>4.704800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11000</td>\n",
       "      <td>4.686600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11500</td>\n",
       "      <td>4.645200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12000</td>\n",
       "      <td>4.600300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12500</td>\n",
       "      <td>4.560500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13000</td>\n",
       "      <td>4.522000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13500</td>\n",
       "      <td>4.490800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14000</td>\n",
       "      <td>4.463000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14500</td>\n",
       "      <td>4.408900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15000</td>\n",
       "      <td>4.428000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15500</td>\n",
       "      <td>4.376700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16000</td>\n",
       "      <td>4.361200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16500</td>\n",
       "      <td>4.321800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17000</td>\n",
       "      <td>4.330900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17500</td>\n",
       "      <td>4.301000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18000</td>\n",
       "      <td>4.271200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18500</td>\n",
       "      <td>4.246600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19000</td>\n",
       "      <td>4.247100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19500</td>\n",
       "      <td>4.187800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20000</td>\n",
       "      <td>4.209300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20500</td>\n",
       "      <td>4.157600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21000</td>\n",
       "      <td>4.156300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21500</td>\n",
       "      <td>4.153900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22000</td>\n",
       "      <td>4.106300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22500</td>\n",
       "      <td>4.100900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23000</td>\n",
       "      <td>4.083400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23500</td>\n",
       "      <td>4.062600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24000</td>\n",
       "      <td>4.043100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24500</td>\n",
       "      <td>4.051600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25000</td>\n",
       "      <td>4.000800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25500</td>\n",
       "      <td>4.009800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26000</td>\n",
       "      <td>3.984100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26500</td>\n",
       "      <td>3.999500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27000</td>\n",
       "      <td>3.957900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27500</td>\n",
       "      <td>3.950900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28000</td>\n",
       "      <td>3.928900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28500</td>\n",
       "      <td>3.933600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29000</td>\n",
       "      <td>3.908200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29500</td>\n",
       "      <td>3.893800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30000</td>\n",
       "      <td>3.901800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30500</td>\n",
       "      <td>3.903200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31000</td>\n",
       "      <td>3.868000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31500</td>\n",
       "      <td>3.868200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32000</td>\n",
       "      <td>3.858900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32500</td>\n",
       "      <td>3.850200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33000</td>\n",
       "      <td>3.848600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33500</td>\n",
       "      <td>3.869300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34000</td>\n",
       "      <td>3.814800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34500</td>\n",
       "      <td>3.811000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35000</td>\n",
       "      <td>3.790900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35500</td>\n",
       "      <td>3.801900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36000</td>\n",
       "      <td>3.789200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36500</td>\n",
       "      <td>3.768100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37000</td>\n",
       "      <td>3.780100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37500</td>\n",
       "      <td>3.773000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38000</td>\n",
       "      <td>3.736300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38500</td>\n",
       "      <td>3.744700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39000</td>\n",
       "      <td>3.725600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39500</td>\n",
       "      <td>3.709800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40000</td>\n",
       "      <td>3.721400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40500</td>\n",
       "      <td>3.717500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41000</td>\n",
       "      <td>3.713800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41500</td>\n",
       "      <td>3.696600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42000</td>\n",
       "      <td>3.677000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42500</td>\n",
       "      <td>3.682800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43000</td>\n",
       "      <td>3.683800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43500</td>\n",
       "      <td>3.696500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44000</td>\n",
       "      <td>3.691300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44500</td>\n",
       "      <td>3.659900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45000</td>\n",
       "      <td>3.666300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45500</td>\n",
       "      <td>3.640200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46000</td>\n",
       "      <td>3.637400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46500</td>\n",
       "      <td>3.626000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47000</td>\n",
       "      <td>3.633200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47500</td>\n",
       "      <td>3.605000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48000</td>\n",
       "      <td>3.641400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48500</td>\n",
       "      <td>3.624900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49000</td>\n",
       "      <td>3.597500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49500</td>\n",
       "      <td>3.595900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50000</td>\n",
       "      <td>3.576400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50500</td>\n",
       "      <td>3.576700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51000</td>\n",
       "      <td>3.605900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51500</td>\n",
       "      <td>3.606000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>52000</td>\n",
       "      <td>3.600000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>52500</td>\n",
       "      <td>3.558600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>53000</td>\n",
       "      <td>3.571200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>53500</td>\n",
       "      <td>3.574300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>54000</td>\n",
       "      <td>3.594600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>54500</td>\n",
       "      <td>3.576300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>55000</td>\n",
       "      <td>3.564200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>55500</td>\n",
       "      <td>3.562100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>56000</td>\n",
       "      <td>3.548300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>56500</td>\n",
       "      <td>3.547000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>57000</td>\n",
       "      <td>3.554800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>57500</td>\n",
       "      <td>3.536700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58000</td>\n",
       "      <td>3.529900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58500</td>\n",
       "      <td>3.535600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59000</td>\n",
       "      <td>3.540100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59500</td>\n",
       "      <td>3.533600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60000</td>\n",
       "      <td>3.551800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60500</td>\n",
       "      <td>3.528500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>61000</td>\n",
       "      <td>3.523500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>61500</td>\n",
       "      <td>3.552100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>62000</td>\n",
       "      <td>3.534900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>62500</td>\n",
       "      <td>3.539700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to ./iwebBERT/checkpoint-10000\n",
      "Configuration saved in ./iwebBERT/checkpoint-10000/config.json\n",
      "Model weights saved in ./iwebBERT/checkpoint-10000/pytorch_model.bin\n",
      "Deleting older checkpoint [iwebBERT/checkpoint-20000] due to args.save_total_limit\n",
      "Saving model checkpoint to ./iwebBERT/checkpoint-20000\n",
      "Configuration saved in ./iwebBERT/checkpoint-20000/config.json\n",
      "Model weights saved in ./iwebBERT/checkpoint-20000/pytorch_model.bin\n",
      "Deleting older checkpoint [iwebBERT/checkpoint-30000] due to args.save_total_limit\n",
      "Saving model checkpoint to ./iwebBERT/checkpoint-30000\n",
      "Configuration saved in ./iwebBERT/checkpoint-30000/config.json\n",
      "Model weights saved in ./iwebBERT/checkpoint-30000/pytorch_model.bin\n",
      "Deleting older checkpoint [iwebBERT/checkpoint-10000] due to args.save_total_limit\n",
      "Saving model checkpoint to ./iwebBERT/checkpoint-40000\n",
      "Configuration saved in ./iwebBERT/checkpoint-40000/config.json\n",
      "Model weights saved in ./iwebBERT/checkpoint-40000/pytorch_model.bin\n",
      "Deleting older checkpoint [iwebBERT/checkpoint-20000] due to args.save_total_limit\n",
      "Saving model checkpoint to ./iwebBERT/checkpoint-50000\n",
      "Configuration saved in ./iwebBERT/checkpoint-50000/config.json\n",
      "Model weights saved in ./iwebBERT/checkpoint-50000/pytorch_model.bin\n",
      "Deleting older checkpoint [iwebBERT/checkpoint-30000] due to args.save_total_limit\n",
      "Saving model checkpoint to ./iwebBERT/checkpoint-60000\n",
      "Configuration saved in ./iwebBERT/checkpoint-60000/config.json\n",
      "Model weights saved in ./iwebBERT/checkpoint-60000/pytorch_model.bin\n",
      "Deleting older checkpoint [iwebBERT/checkpoint-40000] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3h 5min 21s, sys: 6min 28s, total: 3h 11min 49s\n",
      "Wall time: 14h 4min 27s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=62500, training_loss=4.169751466796875, metrics={'train_runtime': 50667.42, 'train_samples_per_second': 39.473, 'train_steps_per_second': 1.234, 'total_flos': 6.633070678199501e+16, 'train_loss': 4.169751466796875, 'epoch': 1.0})"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "%%time\n",
    "\n",
    "trainer.train()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "99bebbc2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to ./iwebBERT/\n",
      "Configuration saved in ./iwebBERT/config.json\n",
      "Model weights saved in ./iwebBERT/pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "trainer.save_model(\"./iwebBERT/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a936cbcd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba91f20820e74a76a35625d927d62216",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "notebook_login()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e0d9b1eb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Configuration saved in /tmp/tmpba7d_umd/config.json\n",
      "Model weights saved in /tmp/tmpba7d_umd/pytorch_model.bin\n",
      "Uploading the following files to jmhuerta/iwebBERT_2M: pytorch_model.bin,config.json\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc3db3b45d4b4c99a80d7f0b03bc2ea3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload 1 LFS files:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03d26be721b14dc381badd7be1aa8505",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model.bin:   0%|          | 0.00/371M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/jmhuerta/iwebBERT_2M/commit/9fea52089e76864fd679807102af7b518317ee70', commit_message='Upload RobertaForMaskedLM', commit_description='', oid='9fea52089e76864fd679807102af7b518317ee70', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from huggingface_hub import create_repo\n",
    "#create_repo(\"iwebBERT\")\n",
    "model.push_to_hub(\"iwebBERT_2M\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f5c80291",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file ./iwebBERT/config.json\n",
      "Model config RobertaConfig {\n",
      "  \"_name_or_path\": \"./iwebBERT\",\n",
      "  \"architectures\": [\n",
      "    \"RobertaForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embedding\": 512,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"roberta\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 6,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"torch_dtype\": \"float32\",\n",
      "  \"transformers_version\": \"4.24.0\",\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 64000\n",
      "}\n",
      "\n",
      "loading configuration file ./iwebBERT/config.json\n",
      "Model config RobertaConfig {\n",
      "  \"_name_or_path\": \"./iwebBERT\",\n",
      "  \"architectures\": [\n",
      "    \"RobertaForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embedding\": 512,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"roberta\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 6,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"torch_dtype\": \"float32\",\n",
      "  \"transformers_version\": \"4.24.0\",\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 64000\n",
      "}\n",
      "\n",
      "loading weights file ./iwebBERT/pytorch_model.bin\n",
      "All model checkpoint weights were used when initializing RobertaForMaskedLM.\n",
      "\n",
      "All the weights of RobertaForMaskedLM were initialized from the model checkpoint at ./iwebBERT.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use RobertaForMaskedLM for predictions without further training.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'score': 0.01324025820940733,\n",
       "  'token': 2942,\n",
       "  'token_str': ' action',\n",
       "  'sequence': 'Human thinking involves human action.'},\n",
       " {'score': 0.012830071151256561,\n",
       "  'token': 1224,\n",
       "  'token_str': ' life',\n",
       "  'sequence': 'Human thinking involves human life.'},\n",
       " {'score': 0.01282454188913107,\n",
       "  'token': 9054,\n",
       "  'token_str': ' thinking',\n",
       "  'sequence': 'Human thinking involves human thinking.'},\n",
       " {'score': 0.011552060022950172,\n",
       "  'token': 5400,\n",
       "  'token_str': ' behavior',\n",
       "  'sequence': 'Human thinking involves human behavior.'},\n",
       " {'score': 0.010932030156254768,\n",
       "  'token': 831,\n",
       "  'token_str': 'ism',\n",
       "  'sequence': 'Human thinking involves humanism.'}]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "fill_mask = pipeline(\n",
    "    \"fill-mask\",\n",
    "    model=\"./iwebBERT\",\n",
    "    tokenizer=tokenizer)\n",
    "fill_mask(\"Human thinking involves human <mask> .\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b4267427",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'score': 0.024219484999775887,\n",
       "  'token': 4886,\n",
       "  'token_str': ' alternative',\n",
       "  'sequence': 'Autism has a strong alternative basis '},\n",
       " {'score': 0.017878247424960136,\n",
       "  'token': 3182,\n",
       "  'token_str': ' legal',\n",
       "  'sequence': 'Autism has a strong legal basis '},\n",
       " {'score': 0.015577989630401134,\n",
       "  'token': 1612,\n",
       "  'token_str': ' political',\n",
       "  'sequence': 'Autism has a strong political basis '},\n",
       " {'score': 0.012330393306910992,\n",
       "  'token': 3030,\n",
       "  'token_str': ' independent',\n",
       "  'sequence': 'Autism has a strong independent basis '},\n",
       " {'score': 0.009066910482943058,\n",
       "  'token': 3078,\n",
       "  'token_str': ' related',\n",
       "  'sequence': 'Autism has a strong related basis '}]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fill_mask(\"Autism has a strong <mask> basis \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "0f61e3e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'score': 0.03342326357960701,\n",
       "  'token': 2634,\n",
       "  'token_str': ' replaced',\n",
       "  'sequence': 'Lance Armstrong cycling career was replaced by doping allegations.'},\n",
       " {'score': 0.03025027923285961,\n",
       "  'token': 2264,\n",
       "  'token_str': ' followed',\n",
       "  'sequence': 'Lance Armstrong cycling career was followed by doping allegations.'},\n",
       " {'score': 0.02391374297440052,\n",
       "  'token': 3222,\n",
       "  'token_str': ' supported',\n",
       "  'sequence': 'Lance Armstrong cycling career was supported by doping allegations.'},\n",
       " {'score': 0.020141808316111565,\n",
       "  'token': 6531,\n",
       "  'token_str': ' criticized',\n",
       "  'sequence': 'Lance Armstrong cycling career was criticized by doping allegations.'},\n",
       " {'score': 0.015271670185029507,\n",
       "  'token': 4375,\n",
       "  'token_str': ' influenced',\n",
       "  'sequence': 'Lance Armstrong cycling career was influenced by doping allegations.'}]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fill_mask(\"Lance Armstrong cycling career was <mask> by doping allegations.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a186e49d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'score': 0.5893740653991699,\n",
       "  'token': 3477,\n",
       "  'token_str': ' Second',\n",
       "  'sequence': 'The Second World War was started by an incident in the Balkans.'},\n",
       " {'score': 0.23300762474536896,\n",
       "  'token': 2768,\n",
       "  'token_str': ' First',\n",
       "  'sequence': 'The First World War was started by an incident in the Balkans.'},\n",
       " {'score': 0.01540341041982174,\n",
       "  'token': 876,\n",
       "  'token_str': ' American',\n",
       "  'sequence': 'The American World War was started by an incident in the Balkans.'},\n",
       " {'score': 0.006034450139850378,\n",
       "  'token': 1053,\n",
       "  'token_str': ' early',\n",
       "  'sequence': 'The early World War was started by an incident in the Balkans.'},\n",
       " {'score': 0.004087221343070269,\n",
       "  'token': 2237,\n",
       "  'token_str': ' Great',\n",
       "  'sequence': 'The Great World War was started by an incident in the Balkans.'}]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fill_mask(\"The <mask> World War was started by an incident in the Balkans.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e234a865",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'score': 0.019469674676656723,\n",
       "  'token': 1314,\n",
       "  'token_str': ' common',\n",
       "  'sequence': 'In his Critique of Practical Reason, Aristotle argues that systematic division is a common principle.'},\n",
       " {'score': 0.018620166927576065,\n",
       "  'token': 2799,\n",
       "  'token_str': ' particular',\n",
       "  'sequence': 'In his Critique of Practical Reason, Aristotle argues that systematic division is a particular principle.'},\n",
       " {'score': 0.0163904819637537,\n",
       "  'token': 1846,\n",
       "  'token_str': ' similar',\n",
       "  'sequence': 'In his Critique of Practical Reason, Aristotle argues that systematic division is a similar principle.'},\n",
       " {'score': 0.013411641120910645,\n",
       "  'token': 4917,\n",
       "  'token_str': ' positive',\n",
       "  'sequence': 'In his Critique of Practical Reason, Aristotle argues that systematic division is a positive principle.'},\n",
       " {'score': 0.01327616535127163,\n",
       "  'token': 1203,\n",
       "  'token_str': ' general',\n",
       "  'sequence': 'In his Critique of Practical Reason, Aristotle argues that systematic division is a general principle.'}]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fill_mask(\"In his Critique of Practical Reason, Aristotle argues that systematic division is a <mask> principle.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "4d3f6782",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'score': 0.04419669508934021,\n",
       "  'token': 265,\n",
       "  'token_str': ' a',\n",
       "  'sequence': 'Godwin is generally regarded as the founder of the school of thought known as a anarchism.'},\n",
       " {'score': 0.030851706862449646,\n",
       "  'token': 2265,\n",
       "  'token_str': ' Christian',\n",
       "  'sequence': 'Godwin is generally regarded as the founder of the school of thought known as Christian anarchism.'},\n",
       " {'score': 0.028029263019561768,\n",
       "  'token': 268,\n",
       "  'token_str': ' the',\n",
       "  'sequence': 'Godwin is generally regarded as the founder of the school of thought known as the anarchism.'},\n",
       " {'score': 0.02710946463048458,\n",
       "  'token': 427,\n",
       "  'token_str': ' his',\n",
       "  'sequence': 'Godwin is generally regarded as the founder of the school of thought known as his anarchism.'},\n",
       " {'score': 0.015148647129535675,\n",
       "  'token': 1612,\n",
       "  'token_str': ' political',\n",
       "  'sequence': 'Godwin is generally regarded as the founder of the school of thought known as political anarchism.'}]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fill_mask(\"Godwin is generally regarded as the founder of the school of thought known as <mask> anarchism.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "1c590f98",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'score': 0.4361608326435089,\n",
       "  'token': 569,\n",
       "  'token_str': ' He',\n",
       "  'sequence': ' He is generally regarded as the founder of the school of thought known as philosophical anarchism.'},\n",
       " {'score': 0.10161503404378891,\n",
       "  'token': 574,\n",
       "  'token_str': ' It',\n",
       "  'sequence': ' It is generally regarded as the founder of the school of thought known as philosophical anarchism.'},\n",
       " {'score': 0.010055812075734138,\n",
       "  'token': 744,\n",
       "  'token_str': ' This',\n",
       "  'sequence': ' This is generally regarded as the founder of the school of thought known as philosophical anarchism.'},\n",
       " {'score': 0.005643005482852459,\n",
       "  'token': 1294,\n",
       "  'token_str': ' She',\n",
       "  'sequence': ' She is generally regarded as the founder of the school of thought known as philosophical anarchism.'},\n",
       " {'score': 0.0032359748147428036,\n",
       "  'token': 402,\n",
       "  'token_str': ' it',\n",
       "  'sequence': ' it is generally regarded as the founder of the school of thought known as philosophical anarchism.'}]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fill_mask(\"<mask> is generally regarded as the founder of the school of thought known as philosophical anarchism.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "6d188729",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'score': 0.030602358281612396,\n",
       "  'token': 8821,\n",
       "  'token_str': ' vector',\n",
       "  'sequence': 'A Markov random field, also known as a Markov network, is a vector over an undirected graph.'},\n",
       " {'score': 0.024418938905000687,\n",
       "  'token': 1883,\n",
       "  'token_str': ' field',\n",
       "  'sequence': 'A Markov random field, also known as a Markov network, is a field over an undirected graph.'},\n",
       " {'score': 0.018840661272406578,\n",
       "  'token': 4707,\n",
       "  'token_str': ' graph',\n",
       "  'sequence': 'A Markov random field, also known as a Markov network, is a graph over an undirected graph.'},\n",
       " {'score': 0.015372563153505325,\n",
       "  'token': 2493,\n",
       "  'token_str': ' function',\n",
       "  'sequence': 'A Markov random field, also known as a Markov network, is a function over an undirected graph.'},\n",
       " {'score': 0.013797575607895851,\n",
       "  'token': 900,\n",
       "  'token_str': ' set',\n",
       "  'sequence': 'A Markov random field, also known as a Markov network, is a set over an undirected graph.'}]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fill_mask(\"A Markov random field, also known as a Markov network, is a <mask> over an undirected graph.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "4e68a7a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'score': 0.036914240568876266,\n",
       "  'token': 12429,\n",
       "  'token_str': ' infinite',\n",
       "  'sequence': 'A Markov random field, also known as a Markov network, is a model over an infinite graph.'},\n",
       " {'score': 0.022134175524115562,\n",
       "  'token': 14446,\n",
       "  'token_str': ' arbitrary',\n",
       "  'sequence': 'A Markov random field, also known as a Markov network, is a model over an arbitrary graph.'},\n",
       " {'score': 0.01123829185962677,\n",
       "  'token': 10027,\n",
       "  'token_str': ' integral',\n",
       "  'sequence': 'A Markov random field, also known as a Markov network, is a model over an integral graph.'},\n",
       " {'score': 0.01084906142205,\n",
       "  'token': 13225,\n",
       "  'token_str': ' algebraic',\n",
       "  'sequence': 'A Markov random field, also known as a Markov network, is a model over an algebraic graph.'},\n",
       " {'score': 0.008688466623425484,\n",
       "  'token': 7019,\n",
       "  'token_str': ' input',\n",
       "  'sequence': 'A Markov random field, also known as a Markov network, is a model over an input graph.'}]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fill_mask(\"A Markov random field, also known as a Markov network, is a model over an <mask> graph.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9baab96",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}